xref: /linux/fs/ext4/inode.c (revision ab93e0dd72c37d378dd936f031ffb83ff2bd87ce)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  linux/fs/ext4/inode.c
4  *
5  * Copyright (C) 1992, 1993, 1994, 1995
6  * Remy Card (card@masi.ibp.fr)
7  * Laboratoire MASI - Institut Blaise Pascal
8  * Universite Pierre et Marie Curie (Paris VI)
9  *
10  *  from
11  *
12  *  linux/fs/minix/inode.c
13  *
14  *  Copyright (C) 1991, 1992  Linus Torvalds
15  *
16  *  64-bit file support on 64-bit platforms by Jakub Jelinek
17  *	(jj@sunsite.ms.mff.cuni.cz)
18  *
19  *  Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000
20  */
21 
22 #include <linux/fs.h>
23 #include <linux/mount.h>
24 #include <linux/time.h>
25 #include <linux/highuid.h>
26 #include <linux/pagemap.h>
27 #include <linux/dax.h>
28 #include <linux/quotaops.h>
29 #include <linux/string.h>
30 #include <linux/buffer_head.h>
31 #include <linux/writeback.h>
32 #include <linux/pagevec.h>
33 #include <linux/mpage.h>
34 #include <linux/rmap.h>
35 #include <linux/namei.h>
36 #include <linux/uio.h>
37 #include <linux/bio.h>
38 #include <linux/workqueue.h>
39 #include <linux/kernel.h>
40 #include <linux/printk.h>
41 #include <linux/slab.h>
42 #include <linux/bitops.h>
43 #include <linux/iomap.h>
44 #include <linux/iversion.h>
45 
46 #include "ext4_jbd2.h"
47 #include "xattr.h"
48 #include "acl.h"
49 #include "truncate.h"
50 
51 #include <trace/events/ext4.h>
52 
53 static void ext4_journalled_zero_new_buffers(handle_t *handle,
54 					    struct inode *inode,
55 					    struct folio *folio,
56 					    unsigned from, unsigned to);
57 
ext4_inode_csum(struct inode * inode,struct ext4_inode * raw,struct ext4_inode_info * ei)58 static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
59 			      struct ext4_inode_info *ei)
60 {
61 	__u32 csum;
62 	__u16 dummy_csum = 0;
63 	int offset = offsetof(struct ext4_inode, i_checksum_lo);
64 	unsigned int csum_size = sizeof(dummy_csum);
65 
66 	csum = ext4_chksum(ei->i_csum_seed, (__u8 *)raw, offset);
67 	csum = ext4_chksum(csum, (__u8 *)&dummy_csum, csum_size);
68 	offset += csum_size;
69 	csum = ext4_chksum(csum, (__u8 *)raw + offset,
70 			   EXT4_GOOD_OLD_INODE_SIZE - offset);
71 
72 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
73 		offset = offsetof(struct ext4_inode, i_checksum_hi);
74 		csum = ext4_chksum(csum, (__u8 *)raw + EXT4_GOOD_OLD_INODE_SIZE,
75 				   offset - EXT4_GOOD_OLD_INODE_SIZE);
76 		if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {
77 			csum = ext4_chksum(csum, (__u8 *)&dummy_csum,
78 					   csum_size);
79 			offset += csum_size;
80 		}
81 		csum = ext4_chksum(csum, (__u8 *)raw + offset,
82 				   EXT4_INODE_SIZE(inode->i_sb) - offset);
83 	}
84 
85 	return csum;
86 }
87 
ext4_inode_csum_verify(struct inode * inode,struct ext4_inode * raw,struct ext4_inode_info * ei)88 static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,
89 				  struct ext4_inode_info *ei)
90 {
91 	__u32 provided, calculated;
92 
93 	if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
94 	    cpu_to_le32(EXT4_OS_LINUX) ||
95 	    !ext4_has_feature_metadata_csum(inode->i_sb))
96 		return 1;
97 
98 	provided = le16_to_cpu(raw->i_checksum_lo);
99 	calculated = ext4_inode_csum(inode, raw, ei);
100 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
101 	    EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
102 		provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;
103 	else
104 		calculated &= 0xFFFF;
105 
106 	return provided == calculated;
107 }
108 
ext4_inode_csum_set(struct inode * inode,struct ext4_inode * raw,struct ext4_inode_info * ei)109 void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,
110 			 struct ext4_inode_info *ei)
111 {
112 	__u32 csum;
113 
114 	if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
115 	    cpu_to_le32(EXT4_OS_LINUX) ||
116 	    !ext4_has_feature_metadata_csum(inode->i_sb))
117 		return;
118 
119 	csum = ext4_inode_csum(inode, raw, ei);
120 	raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);
121 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
122 	    EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
123 		raw->i_checksum_hi = cpu_to_le16(csum >> 16);
124 }
125 
ext4_begin_ordered_truncate(struct inode * inode,loff_t new_size)126 static inline int ext4_begin_ordered_truncate(struct inode *inode,
127 					      loff_t new_size)
128 {
129 	trace_ext4_begin_ordered_truncate(inode, new_size);
130 	/*
131 	 * If jinode is zero, then we never opened the file for
132 	 * writing, so there's no need to call
133 	 * jbd2_journal_begin_ordered_truncate() since there's no
134 	 * outstanding writes we need to flush.
135 	 */
136 	if (!EXT4_I(inode)->jinode)
137 		return 0;
138 	return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
139 						   EXT4_I(inode)->jinode,
140 						   new_size);
141 }
142 
143 /*
144  * Test whether an inode is a fast symlink.
145  * A fast symlink has its symlink data stored in ext4_inode_info->i_data.
146  */
ext4_inode_is_fast_symlink(struct inode * inode)147 int ext4_inode_is_fast_symlink(struct inode *inode)
148 {
149 	if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
150 		int ea_blocks = EXT4_I(inode)->i_file_acl ?
151 				EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;
152 
153 		if (ext4_has_inline_data(inode))
154 			return 0;
155 
156 		return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
157 	}
158 	return S_ISLNK(inode->i_mode) && inode->i_size &&
159 	       (inode->i_size < EXT4_N_BLOCKS * 4);
160 }
161 
162 /*
163  * Called at the last iput() if i_nlink is zero.
164  */
ext4_evict_inode(struct inode * inode)165 void ext4_evict_inode(struct inode *inode)
166 {
167 	handle_t *handle;
168 	int err;
169 	/*
170 	 * Credits for final inode cleanup and freeing:
171 	 * sb + inode (ext4_orphan_del()), block bitmap, group descriptor
172 	 * (xattr block freeing), bitmap, group descriptor (inode freeing)
173 	 */
174 	int extra_credits = 6;
175 	struct ext4_xattr_inode_array *ea_inode_array = NULL;
176 	bool freeze_protected = false;
177 
178 	trace_ext4_evict_inode(inode);
179 
180 	dax_break_layout_final(inode);
181 
182 	if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)
183 		ext4_evict_ea_inode(inode);
184 	if (inode->i_nlink) {
185 		truncate_inode_pages_final(&inode->i_data);
186 
187 		goto no_delete;
188 	}
189 
190 	if (is_bad_inode(inode))
191 		goto no_delete;
192 	dquot_initialize(inode);
193 
194 	if (ext4_should_order_data(inode))
195 		ext4_begin_ordered_truncate(inode, 0);
196 	truncate_inode_pages_final(&inode->i_data);
197 
198 	/*
199 	 * For inodes with journalled data, transaction commit could have
200 	 * dirtied the inode. And for inodes with dioread_nolock, unwritten
201 	 * extents converting worker could merge extents and also have dirtied
202 	 * the inode. Flush worker is ignoring it because of I_FREEING flag but
203 	 * we still need to remove the inode from the writeback lists.
204 	 */
205 	if (!list_empty_careful(&inode->i_io_list))
206 		inode_io_list_del(inode);
207 
208 	/*
209 	 * Protect us against freezing - iput() caller didn't have to have any
210 	 * protection against it. When we are in a running transaction though,
211 	 * we are already protected against freezing and we cannot grab further
212 	 * protection due to lock ordering constraints.
213 	 */
214 	if (!ext4_journal_current_handle()) {
215 		sb_start_intwrite(inode->i_sb);
216 		freeze_protected = true;
217 	}
218 
219 	if (!IS_NOQUOTA(inode))
220 		extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb);
221 
222 	/*
223 	 * Block bitmap, group descriptor, and inode are accounted in both
224 	 * ext4_blocks_for_truncate() and extra_credits. So subtract 3.
225 	 */
226 	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,
227 			 ext4_blocks_for_truncate(inode) + extra_credits - 3);
228 	if (IS_ERR(handle)) {
229 		ext4_std_error(inode->i_sb, PTR_ERR(handle));
230 		/*
231 		 * If we're going to skip the normal cleanup, we still need to
232 		 * make sure that the in-core orphan linked list is properly
233 		 * cleaned up.
234 		 */
235 		ext4_orphan_del(NULL, inode);
236 		if (freeze_protected)
237 			sb_end_intwrite(inode->i_sb);
238 		goto no_delete;
239 	}
240 
241 	if (IS_SYNC(inode))
242 		ext4_handle_sync(handle);
243 
244 	/*
245 	 * Set inode->i_size to 0 before calling ext4_truncate(). We need
246 	 * special handling of symlinks here because i_size is used to
247 	 * determine whether ext4_inode_info->i_data contains symlink data or
248 	 * block mappings. Setting i_size to 0 will remove its fast symlink
249 	 * status. Erase i_data so that it becomes a valid empty block map.
250 	 */
251 	if (ext4_inode_is_fast_symlink(inode))
252 		memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));
253 	inode->i_size = 0;
254 	err = ext4_mark_inode_dirty(handle, inode);
255 	if (err) {
256 		ext4_warning(inode->i_sb,
257 			     "couldn't mark inode dirty (err %d)", err);
258 		goto stop_handle;
259 	}
260 	if (inode->i_blocks) {
261 		err = ext4_truncate(inode);
262 		if (err) {
263 			ext4_error_err(inode->i_sb, -err,
264 				       "couldn't truncate inode %lu (err %d)",
265 				       inode->i_ino, err);
266 			goto stop_handle;
267 		}
268 	}
269 
270 	/* Remove xattr references. */
271 	err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array,
272 				      extra_credits);
273 	if (err) {
274 		ext4_warning(inode->i_sb, "xattr delete (err %d)", err);
275 stop_handle:
276 		ext4_journal_stop(handle);
277 		ext4_orphan_del(NULL, inode);
278 		if (freeze_protected)
279 			sb_end_intwrite(inode->i_sb);
280 		ext4_xattr_inode_array_free(ea_inode_array);
281 		goto no_delete;
282 	}
283 
284 	/*
285 	 * Kill off the orphan record which ext4_truncate created.
286 	 * AKPM: I think this can be inside the above `if'.
287 	 * Note that ext4_orphan_del() has to be able to cope with the
288 	 * deletion of a non-existent orphan - this is because we don't
289 	 * know if ext4_truncate() actually created an orphan record.
290 	 * (Well, we could do this if we need to, but heck - it works)
291 	 */
292 	ext4_orphan_del(handle, inode);
293 	EXT4_I(inode)->i_dtime	= (__u32)ktime_get_real_seconds();
294 
295 	/*
296 	 * One subtle ordering requirement: if anything has gone wrong
297 	 * (transaction abort, IO errors, whatever), then we can still
298 	 * do these next steps (the fs will already have been marked as
299 	 * having errors), but we can't free the inode if the mark_dirty
300 	 * fails.
301 	 */
302 	if (ext4_mark_inode_dirty(handle, inode))
303 		/* If that failed, just do the required in-core inode clear. */
304 		ext4_clear_inode(inode);
305 	else
306 		ext4_free_inode(handle, inode);
307 	ext4_journal_stop(handle);
308 	if (freeze_protected)
309 		sb_end_intwrite(inode->i_sb);
310 	ext4_xattr_inode_array_free(ea_inode_array);
311 	return;
312 no_delete:
313 	/*
314 	 * Check out some where else accidentally dirty the evicting inode,
315 	 * which may probably cause inode use-after-free issues later.
316 	 */
317 	WARN_ON_ONCE(!list_empty_careful(&inode->i_io_list));
318 
319 	if (!list_empty(&EXT4_I(inode)->i_fc_list))
320 		ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM, NULL);
321 	ext4_clear_inode(inode);	/* We must guarantee clearing of inode... */
322 }
323 
324 #ifdef CONFIG_QUOTA
ext4_get_reserved_space(struct inode * inode)325 qsize_t *ext4_get_reserved_space(struct inode *inode)
326 {
327 	return &EXT4_I(inode)->i_reserved_quota;
328 }
329 #endif
330 
331 /*
332  * Called with i_data_sem down, which is important since we can call
333  * ext4_discard_preallocations() from here.
334  */
ext4_da_update_reserve_space(struct inode * inode,int used,int quota_claim)335 void ext4_da_update_reserve_space(struct inode *inode,
336 					int used, int quota_claim)
337 {
338 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
339 	struct ext4_inode_info *ei = EXT4_I(inode);
340 
341 	spin_lock(&ei->i_block_reservation_lock);
342 	trace_ext4_da_update_reserve_space(inode, used, quota_claim);
343 	if (unlikely(used > ei->i_reserved_data_blocks)) {
344 		ext4_warning(inode->i_sb, "%s: ino %lu, used %d "
345 			 "with only %d reserved data blocks",
346 			 __func__, inode->i_ino, used,
347 			 ei->i_reserved_data_blocks);
348 		WARN_ON(1);
349 		used = ei->i_reserved_data_blocks;
350 	}
351 
352 	/* Update per-inode reservations */
353 	ei->i_reserved_data_blocks -= used;
354 	percpu_counter_sub(&sbi->s_dirtyclusters_counter, used);
355 
356 	spin_unlock(&ei->i_block_reservation_lock);
357 
358 	/* Update quota subsystem for data blocks */
359 	if (quota_claim)
360 		dquot_claim_block(inode, EXT4_C2B(sbi, used));
361 	else {
362 		/*
363 		 * We did fallocate with an offset that is already delayed
364 		 * allocated. So on delayed allocated writeback we should
365 		 * not re-claim the quota for fallocated blocks.
366 		 */
367 		dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
368 	}
369 
370 	/*
371 	 * If we have done all the pending block allocations and if
372 	 * there aren't any writers on the inode, we can discard the
373 	 * inode's preallocations.
374 	 */
375 	if ((ei->i_reserved_data_blocks == 0) &&
376 	    !inode_is_open_for_write(inode))
377 		ext4_discard_preallocations(inode);
378 }
379 
__check_block_validity(struct inode * inode,const char * func,unsigned int line,struct ext4_map_blocks * map)380 static int __check_block_validity(struct inode *inode, const char *func,
381 				unsigned int line,
382 				struct ext4_map_blocks *map)
383 {
384 	journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
385 
386 	if (journal && inode == journal->j_inode)
387 		return 0;
388 
389 	if (!ext4_inode_block_valid(inode, map->m_pblk, map->m_len)) {
390 		ext4_error_inode(inode, func, line, map->m_pblk,
391 				 "lblock %lu mapped to illegal pblock %llu "
392 				 "(length %d)", (unsigned long) map->m_lblk,
393 				 map->m_pblk, map->m_len);
394 		return -EFSCORRUPTED;
395 	}
396 	return 0;
397 }
398 
ext4_issue_zeroout(struct inode * inode,ext4_lblk_t lblk,ext4_fsblk_t pblk,ext4_lblk_t len)399 int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
400 		       ext4_lblk_t len)
401 {
402 	int ret;
403 
404 	if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))
405 		return fscrypt_zeroout_range(inode, lblk, pblk, len);
406 
407 	ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);
408 	if (ret > 0)
409 		ret = 0;
410 
411 	return ret;
412 }
413 
414 /*
415  * For generic regular files, when updating the extent tree, Ext4 should
416  * hold the i_rwsem and invalidate_lock exclusively. This ensures
417  * exclusion against concurrent page faults, as well as reads and writes.
418  */
419 #ifdef CONFIG_EXT4_DEBUG
ext4_check_map_extents_env(struct inode * inode)420 void ext4_check_map_extents_env(struct inode *inode)
421 {
422 	if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
423 		return;
424 
425 	if (!S_ISREG(inode->i_mode) ||
426 	    IS_NOQUOTA(inode) || IS_VERITY(inode) ||
427 	    is_special_ino(inode->i_sb, inode->i_ino) ||
428 	    (inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW)) ||
429 	    ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE) ||
430 	    ext4_verity_in_progress(inode))
431 		return;
432 
433 	WARN_ON_ONCE(!inode_is_locked(inode) &&
434 		     !rwsem_is_locked(&inode->i_mapping->invalidate_lock));
435 }
436 #else
ext4_check_map_extents_env(struct inode * inode)437 void ext4_check_map_extents_env(struct inode *inode) {}
438 #endif
439 
440 #define check_block_validity(inode, map)	\
441 	__check_block_validity((inode), __func__, __LINE__, (map))
442 
443 #ifdef ES_AGGRESSIVE_TEST
ext4_map_blocks_es_recheck(handle_t * handle,struct inode * inode,struct ext4_map_blocks * es_map,struct ext4_map_blocks * map,int flags)444 static void ext4_map_blocks_es_recheck(handle_t *handle,
445 				       struct inode *inode,
446 				       struct ext4_map_blocks *es_map,
447 				       struct ext4_map_blocks *map,
448 				       int flags)
449 {
450 	int retval;
451 
452 	map->m_flags = 0;
453 	/*
454 	 * There is a race window that the result is not the same.
455 	 * e.g. xfstests #223 when dioread_nolock enables.  The reason
456 	 * is that we lookup a block mapping in extent status tree with
457 	 * out taking i_data_sem.  So at the time the unwritten extent
458 	 * could be converted.
459 	 */
460 	down_read(&EXT4_I(inode)->i_data_sem);
461 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
462 		retval = ext4_ext_map_blocks(handle, inode, map, 0);
463 	} else {
464 		retval = ext4_ind_map_blocks(handle, inode, map, 0);
465 	}
466 	up_read((&EXT4_I(inode)->i_data_sem));
467 
468 	/*
469 	 * We don't check m_len because extent will be collpased in status
470 	 * tree.  So the m_len might not equal.
471 	 */
472 	if (es_map->m_lblk != map->m_lblk ||
473 	    es_map->m_flags != map->m_flags ||
474 	    es_map->m_pblk != map->m_pblk) {
475 		printk("ES cache assertion failed for inode: %lu "
476 		       "es_cached ex [%d/%d/%llu/%x] != "
477 		       "found ex [%d/%d/%llu/%x] retval %d flags %x\n",
478 		       inode->i_ino, es_map->m_lblk, es_map->m_len,
479 		       es_map->m_pblk, es_map->m_flags, map->m_lblk,
480 		       map->m_len, map->m_pblk, map->m_flags,
481 		       retval, flags);
482 	}
483 }
484 #endif /* ES_AGGRESSIVE_TEST */
485 
ext4_map_query_blocks_next_in_leaf(handle_t * handle,struct inode * inode,struct ext4_map_blocks * map,unsigned int orig_mlen)486 static int ext4_map_query_blocks_next_in_leaf(handle_t *handle,
487 			struct inode *inode, struct ext4_map_blocks *map,
488 			unsigned int orig_mlen)
489 {
490 	struct ext4_map_blocks map2;
491 	unsigned int status, status2;
492 	int retval;
493 
494 	status = map->m_flags & EXT4_MAP_UNWRITTEN ?
495 		EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
496 
497 	WARN_ON_ONCE(!(map->m_flags & EXT4_MAP_QUERY_LAST_IN_LEAF));
498 	WARN_ON_ONCE(orig_mlen <= map->m_len);
499 
500 	/* Prepare map2 for lookup in next leaf block */
501 	map2.m_lblk = map->m_lblk + map->m_len;
502 	map2.m_len = orig_mlen - map->m_len;
503 	map2.m_flags = 0;
504 	retval = ext4_ext_map_blocks(handle, inode, &map2, 0);
505 
506 	if (retval <= 0) {
507 		ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
508 				      map->m_pblk, status, false);
509 		return map->m_len;
510 	}
511 
512 	if (unlikely(retval != map2.m_len)) {
513 		ext4_warning(inode->i_sb,
514 			     "ES len assertion failed for inode "
515 			     "%lu: retval %d != map->m_len %d",
516 			     inode->i_ino, retval, map2.m_len);
517 		WARN_ON(1);
518 	}
519 
520 	status2 = map2.m_flags & EXT4_MAP_UNWRITTEN ?
521 		EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
522 
523 	/*
524 	 * If map2 is contiguous with map, then let's insert it as a single
525 	 * extent in es cache and return the combined length of both the maps.
526 	 */
527 	if (map->m_pblk + map->m_len == map2.m_pblk &&
528 			status == status2) {
529 		ext4_es_insert_extent(inode, map->m_lblk,
530 				      map->m_len + map2.m_len, map->m_pblk,
531 				      status, false);
532 		map->m_len += map2.m_len;
533 	} else {
534 		ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
535 				      map->m_pblk, status, false);
536 	}
537 
538 	return map->m_len;
539 }
540 
ext4_map_query_blocks(handle_t * handle,struct inode * inode,struct ext4_map_blocks * map,int flags)541 static int ext4_map_query_blocks(handle_t *handle, struct inode *inode,
542 				 struct ext4_map_blocks *map, int flags)
543 {
544 	unsigned int status;
545 	int retval;
546 	unsigned int orig_mlen = map->m_len;
547 
548 	flags &= EXT4_EX_QUERY_FILTER;
549 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
550 		retval = ext4_ext_map_blocks(handle, inode, map, flags);
551 	else
552 		retval = ext4_ind_map_blocks(handle, inode, map, flags);
553 
554 	if (retval <= 0)
555 		return retval;
556 
557 	if (unlikely(retval != map->m_len)) {
558 		ext4_warning(inode->i_sb,
559 			     "ES len assertion failed for inode "
560 			     "%lu: retval %d != map->m_len %d",
561 			     inode->i_ino, retval, map->m_len);
562 		WARN_ON(1);
563 	}
564 
565 	/*
566 	 * No need to query next in leaf:
567 	 * - if returned extent is not last in leaf or
568 	 * - if the last in leaf is the full requested range
569 	 */
570 	if (!(map->m_flags & EXT4_MAP_QUERY_LAST_IN_LEAF) ||
571 			map->m_len == orig_mlen) {
572 		status = map->m_flags & EXT4_MAP_UNWRITTEN ?
573 				EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
574 		ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
575 				      map->m_pblk, status, false);
576 		return retval;
577 	}
578 
579 	return ext4_map_query_blocks_next_in_leaf(handle, inode, map,
580 						  orig_mlen);
581 }
582 
ext4_map_create_blocks(handle_t * handle,struct inode * inode,struct ext4_map_blocks * map,int flags)583 static int ext4_map_create_blocks(handle_t *handle, struct inode *inode,
584 				  struct ext4_map_blocks *map, int flags)
585 {
586 	struct extent_status es;
587 	unsigned int status;
588 	int err, retval = 0;
589 
590 	/*
591 	 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE
592 	 * indicates that the blocks and quotas has already been
593 	 * checked when the data was copied into the page cache.
594 	 */
595 	if (map->m_flags & EXT4_MAP_DELAYED)
596 		flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
597 
598 	/*
599 	 * Here we clear m_flags because after allocating an new extent,
600 	 * it will be set again.
601 	 */
602 	map->m_flags &= ~EXT4_MAP_FLAGS;
603 
604 	/*
605 	 * We need to check for EXT4 here because migrate could have
606 	 * changed the inode type in between.
607 	 */
608 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
609 		retval = ext4_ext_map_blocks(handle, inode, map, flags);
610 	} else {
611 		retval = ext4_ind_map_blocks(handle, inode, map, flags);
612 
613 		/*
614 		 * We allocated new blocks which will result in i_data's
615 		 * format changing. Force the migrate to fail by clearing
616 		 * migrate flags.
617 		 */
618 		if (retval > 0 && map->m_flags & EXT4_MAP_NEW)
619 			ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
620 	}
621 	if (retval <= 0)
622 		return retval;
623 
624 	if (unlikely(retval != map->m_len)) {
625 		ext4_warning(inode->i_sb,
626 			     "ES len assertion failed for inode %lu: "
627 			     "retval %d != map->m_len %d",
628 			     inode->i_ino, retval, map->m_len);
629 		WARN_ON(1);
630 	}
631 
632 	/*
633 	 * We have to zeroout blocks before inserting them into extent
634 	 * status tree. Otherwise someone could look them up there and
635 	 * use them before they are really zeroed. We also have to
636 	 * unmap metadata before zeroing as otherwise writeback can
637 	 * overwrite zeros with stale data from block device.
638 	 */
639 	if (flags & EXT4_GET_BLOCKS_ZERO &&
640 	    map->m_flags & EXT4_MAP_MAPPED && map->m_flags & EXT4_MAP_NEW) {
641 		err = ext4_issue_zeroout(inode, map->m_lblk, map->m_pblk,
642 					 map->m_len);
643 		if (err)
644 			return err;
645 	}
646 
647 	/*
648 	 * If the extent has been zeroed out, we don't need to update
649 	 * extent status tree.
650 	 */
651 	if (flags & EXT4_GET_BLOCKS_PRE_IO &&
652 	    ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {
653 		if (ext4_es_is_written(&es))
654 			return retval;
655 	}
656 
657 	status = map->m_flags & EXT4_MAP_UNWRITTEN ?
658 			EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
659 	ext4_es_insert_extent(inode, map->m_lblk, map->m_len, map->m_pblk,
660 			      status, flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE);
661 
662 	return retval;
663 }
664 
665 /*
666  * The ext4_map_blocks() function tries to look up the requested blocks,
667  * and returns if the blocks are already mapped.
668  *
669  * Otherwise it takes the write lock of the i_data_sem and allocate blocks
670  * and store the allocated blocks in the result buffer head and mark it
671  * mapped.
672  *
673  * If file type is extents based, it will call ext4_ext_map_blocks(),
674  * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping
675  * based files
676  *
677  * On success, it returns the number of blocks being mapped or allocated.
678  * If flags doesn't contain EXT4_GET_BLOCKS_CREATE the blocks are
679  * pre-allocated and unwritten, the resulting @map is marked as unwritten.
680  * If the flags contain EXT4_GET_BLOCKS_CREATE, it will mark @map as mapped.
681  *
682  * It returns 0 if plain look up failed (blocks have not been allocated), in
683  * that case, @map is returned as unmapped but we still do fill map->m_len to
684  * indicate the length of a hole starting at map->m_lblk.
685  *
686  * It returns the error in case of allocation failure.
687  */
ext4_map_blocks(handle_t * handle,struct inode * inode,struct ext4_map_blocks * map,int flags)688 int ext4_map_blocks(handle_t *handle, struct inode *inode,
689 		    struct ext4_map_blocks *map, int flags)
690 {
691 	struct extent_status es;
692 	int retval;
693 	int ret = 0;
694 	unsigned int orig_mlen = map->m_len;
695 #ifdef ES_AGGRESSIVE_TEST
696 	struct ext4_map_blocks orig_map;
697 
698 	memcpy(&orig_map, map, sizeof(*map));
699 #endif
700 
701 	map->m_flags = 0;
702 	ext_debug(inode, "flag 0x%x, max_blocks %u, logical block %lu\n",
703 		  flags, map->m_len, (unsigned long) map->m_lblk);
704 
705 	/*
706 	 * ext4_map_blocks returns an int, and m_len is an unsigned int
707 	 */
708 	if (unlikely(map->m_len > INT_MAX))
709 		map->m_len = INT_MAX;
710 
711 	/* We can handle the block number less than EXT_MAX_BLOCKS */
712 	if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))
713 		return -EFSCORRUPTED;
714 
715 	/*
716 	 * Callers from the context of data submission are the only exceptions
717 	 * for regular files that do not hold the i_rwsem or invalidate_lock.
718 	 * However, caching unrelated ranges is not permitted.
719 	 */
720 	if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
721 		WARN_ON_ONCE(!(flags & EXT4_EX_NOCACHE));
722 	else
723 		ext4_check_map_extents_env(inode);
724 
725 	/* Lookup extent status tree firstly */
726 	if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {
727 		if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) {
728 			map->m_pblk = ext4_es_pblock(&es) +
729 					map->m_lblk - es.es_lblk;
730 			map->m_flags |= ext4_es_is_written(&es) ?
731 					EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;
732 			retval = es.es_len - (map->m_lblk - es.es_lblk);
733 			if (retval > map->m_len)
734 				retval = map->m_len;
735 			map->m_len = retval;
736 		} else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) {
737 			map->m_pblk = 0;
738 			map->m_flags |= ext4_es_is_delayed(&es) ?
739 					EXT4_MAP_DELAYED : 0;
740 			retval = es.es_len - (map->m_lblk - es.es_lblk);
741 			if (retval > map->m_len)
742 				retval = map->m_len;
743 			map->m_len = retval;
744 			retval = 0;
745 		} else {
746 			BUG();
747 		}
748 
749 		if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT)
750 			return retval;
751 #ifdef ES_AGGRESSIVE_TEST
752 		ext4_map_blocks_es_recheck(handle, inode, map,
753 					   &orig_map, flags);
754 #endif
755 		if (!(flags & EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF) ||
756 				orig_mlen == map->m_len)
757 			goto found;
758 
759 		map->m_len = orig_mlen;
760 	}
761 	/*
762 	 * In the query cache no-wait mode, nothing we can do more if we
763 	 * cannot find extent in the cache.
764 	 */
765 	if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT)
766 		return 0;
767 
768 	/*
769 	 * Try to see if we can get the block without requesting a new
770 	 * file system block.
771 	 */
772 	down_read(&EXT4_I(inode)->i_data_sem);
773 	retval = ext4_map_query_blocks(handle, inode, map, flags);
774 	up_read((&EXT4_I(inode)->i_data_sem));
775 
776 found:
777 	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
778 		ret = check_block_validity(inode, map);
779 		if (ret != 0)
780 			return ret;
781 	}
782 
783 	/* If it is only a block(s) look up */
784 	if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
785 		return retval;
786 
787 	/*
788 	 * Returns if the blocks have already allocated
789 	 *
790 	 * Note that if blocks have been preallocated
791 	 * ext4_ext_map_blocks() returns with buffer head unmapped
792 	 */
793 	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
794 		/*
795 		 * If we need to convert extent to unwritten
796 		 * we continue and do the actual work in
797 		 * ext4_ext_map_blocks()
798 		 */
799 		if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))
800 			return retval;
801 
802 
803 	ext4_fc_track_inode(handle, inode);
804 	/*
805 	 * New blocks allocate and/or writing to unwritten extent
806 	 * will possibly result in updating i_data, so we take
807 	 * the write lock of i_data_sem, and call get_block()
808 	 * with create == 1 flag.
809 	 */
810 	down_write(&EXT4_I(inode)->i_data_sem);
811 	retval = ext4_map_create_blocks(handle, inode, map, flags);
812 	up_write((&EXT4_I(inode)->i_data_sem));
813 	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
814 		ret = check_block_validity(inode, map);
815 		if (ret != 0)
816 			return ret;
817 
818 		/*
819 		 * Inodes with freshly allocated blocks where contents will be
820 		 * visible after transaction commit must be on transaction's
821 		 * ordered data list.
822 		 */
823 		if (map->m_flags & EXT4_MAP_NEW &&
824 		    !(map->m_flags & EXT4_MAP_UNWRITTEN) &&
825 		    !(flags & EXT4_GET_BLOCKS_ZERO) &&
826 		    !ext4_is_quota_file(inode) &&
827 		    ext4_should_order_data(inode)) {
828 			loff_t start_byte =
829 				(loff_t)map->m_lblk << inode->i_blkbits;
830 			loff_t length = (loff_t)map->m_len << inode->i_blkbits;
831 
832 			if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
833 				ret = ext4_jbd2_inode_add_wait(handle, inode,
834 						start_byte, length);
835 			else
836 				ret = ext4_jbd2_inode_add_write(handle, inode,
837 						start_byte, length);
838 			if (ret)
839 				return ret;
840 		}
841 	}
842 	if (retval > 0 && (map->m_flags & EXT4_MAP_UNWRITTEN ||
843 				map->m_flags & EXT4_MAP_MAPPED))
844 		ext4_fc_track_range(handle, inode, map->m_lblk,
845 					map->m_lblk + map->m_len - 1);
846 	if (retval < 0)
847 		ext_debug(inode, "failed with err %d\n", retval);
848 	return retval;
849 }
850 
851 /*
852  * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages
853  * we have to be careful as someone else may be manipulating b_state as well.
854  */
ext4_update_bh_state(struct buffer_head * bh,unsigned long flags)855 static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)
856 {
857 	unsigned long old_state;
858 	unsigned long new_state;
859 
860 	flags &= EXT4_MAP_FLAGS;
861 
862 	/* Dummy buffer_head? Set non-atomically. */
863 	if (!bh->b_folio) {
864 		bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;
865 		return;
866 	}
867 	/*
868 	 * Someone else may be modifying b_state. Be careful! This is ugly but
869 	 * once we get rid of using bh as a container for mapping information
870 	 * to pass to / from get_block functions, this can go away.
871 	 */
872 	old_state = READ_ONCE(bh->b_state);
873 	do {
874 		new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;
875 	} while (unlikely(!try_cmpxchg(&bh->b_state, &old_state, new_state)));
876 }
877 
878 /*
879  * Make sure that the current journal transaction has enough credits to map
880  * one extent. Return -EAGAIN if it cannot extend the current running
881  * transaction.
882  */
ext4_journal_ensure_extent_credits(handle_t * handle,struct inode * inode)883 static inline int ext4_journal_ensure_extent_credits(handle_t *handle,
884 						     struct inode *inode)
885 {
886 	int credits;
887 	int ret;
888 
889 	/* Called from ext4_da_write_begin() which has no handle started? */
890 	if (!handle)
891 		return 0;
892 
893 	credits = ext4_chunk_trans_blocks(inode, 1);
894 	ret = __ext4_journal_ensure_credits(handle, credits, credits, 0);
895 	return ret <= 0 ? ret : -EAGAIN;
896 }
897 
_ext4_get_block(struct inode * inode,sector_t iblock,struct buffer_head * bh,int flags)898 static int _ext4_get_block(struct inode *inode, sector_t iblock,
899 			   struct buffer_head *bh, int flags)
900 {
901 	struct ext4_map_blocks map;
902 	int ret = 0;
903 
904 	if (ext4_has_inline_data(inode))
905 		return -ERANGE;
906 
907 	map.m_lblk = iblock;
908 	map.m_len = bh->b_size >> inode->i_blkbits;
909 
910 	ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map,
911 			      flags);
912 	if (ret > 0) {
913 		map_bh(bh, inode->i_sb, map.m_pblk);
914 		ext4_update_bh_state(bh, map.m_flags);
915 		bh->b_size = inode->i_sb->s_blocksize * map.m_len;
916 		ret = 0;
917 	} else if (ret == 0) {
918 		/* hole case, need to fill in bh->b_size */
919 		bh->b_size = inode->i_sb->s_blocksize * map.m_len;
920 	}
921 	return ret;
922 }
923 
ext4_get_block(struct inode * inode,sector_t iblock,struct buffer_head * bh,int create)924 int ext4_get_block(struct inode *inode, sector_t iblock,
925 		   struct buffer_head *bh, int create)
926 {
927 	return _ext4_get_block(inode, iblock, bh,
928 			       create ? EXT4_GET_BLOCKS_CREATE : 0);
929 }
930 
931 /*
932  * Get block function used when preparing for buffered write if we require
933  * creating an unwritten extent if blocks haven't been allocated.  The extent
934  * will be converted to written after the IO is complete.
935  */
ext4_get_block_unwritten(struct inode * inode,sector_t iblock,struct buffer_head * bh_result,int create)936 int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
937 			     struct buffer_head *bh_result, int create)
938 {
939 	int ret = 0;
940 
941 	ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n",
942 		   inode->i_ino, create);
943 	ret = _ext4_get_block(inode, iblock, bh_result,
944 			       EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT);
945 
946 	/*
947 	 * If the buffer is marked unwritten, mark it as new to make sure it is
948 	 * zeroed out correctly in case of partial writes. Otherwise, there is
949 	 * a chance of stale data getting exposed.
950 	 */
951 	if (ret == 0 && buffer_unwritten(bh_result))
952 		set_buffer_new(bh_result);
953 
954 	return ret;
955 }
956 
957 /* Maximum number of blocks we map for direct IO at once. */
958 #define DIO_MAX_BLOCKS 4096
959 
960 /*
961  * `handle' can be NULL if create is zero
962  */
ext4_getblk(handle_t * handle,struct inode * inode,ext4_lblk_t block,int map_flags)963 struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
964 				ext4_lblk_t block, int map_flags)
965 {
966 	struct ext4_map_blocks map;
967 	struct buffer_head *bh;
968 	int create = map_flags & EXT4_GET_BLOCKS_CREATE;
969 	bool nowait = map_flags & EXT4_GET_BLOCKS_CACHED_NOWAIT;
970 	int err;
971 
972 	ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
973 		    || handle != NULL || create == 0);
974 	ASSERT(create == 0 || !nowait);
975 
976 	map.m_lblk = block;
977 	map.m_len = 1;
978 	err = ext4_map_blocks(handle, inode, &map, map_flags);
979 
980 	if (err == 0)
981 		return create ? ERR_PTR(-ENOSPC) : NULL;
982 	if (err < 0)
983 		return ERR_PTR(err);
984 
985 	if (nowait)
986 		return sb_find_get_block(inode->i_sb, map.m_pblk);
987 
988 	/*
989 	 * Since bh could introduce extra ref count such as referred by
990 	 * journal_head etc. Try to avoid using __GFP_MOVABLE here
991 	 * as it may fail the migration when journal_head remains.
992 	 */
993 	bh = getblk_unmovable(inode->i_sb->s_bdev, map.m_pblk,
994 				inode->i_sb->s_blocksize);
995 
996 	if (unlikely(!bh))
997 		return ERR_PTR(-ENOMEM);
998 	if (map.m_flags & EXT4_MAP_NEW) {
999 		ASSERT(create != 0);
1000 		ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
1001 			    || (handle != NULL));
1002 
1003 		/*
1004 		 * Now that we do not always journal data, we should
1005 		 * keep in mind whether this should always journal the
1006 		 * new buffer as metadata.  For now, regular file
1007 		 * writes use ext4_get_block instead, so it's not a
1008 		 * problem.
1009 		 */
1010 		lock_buffer(bh);
1011 		BUFFER_TRACE(bh, "call get_create_access");
1012 		err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
1013 						     EXT4_JTR_NONE);
1014 		if (unlikely(err)) {
1015 			unlock_buffer(bh);
1016 			goto errout;
1017 		}
1018 		if (!buffer_uptodate(bh)) {
1019 			memset(bh->b_data, 0, inode->i_sb->s_blocksize);
1020 			set_buffer_uptodate(bh);
1021 		}
1022 		unlock_buffer(bh);
1023 		BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
1024 		err = ext4_handle_dirty_metadata(handle, inode, bh);
1025 		if (unlikely(err))
1026 			goto errout;
1027 	} else
1028 		BUFFER_TRACE(bh, "not a new buffer");
1029 	return bh;
1030 errout:
1031 	brelse(bh);
1032 	return ERR_PTR(err);
1033 }
1034 
ext4_bread(handle_t * handle,struct inode * inode,ext4_lblk_t block,int map_flags)1035 struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
1036 			       ext4_lblk_t block, int map_flags)
1037 {
1038 	struct buffer_head *bh;
1039 	int ret;
1040 
1041 	bh = ext4_getblk(handle, inode, block, map_flags);
1042 	if (IS_ERR(bh))
1043 		return bh;
1044 	if (!bh || ext4_buffer_uptodate(bh))
1045 		return bh;
1046 
1047 	ret = ext4_read_bh_lock(bh, REQ_META | REQ_PRIO, true);
1048 	if (ret) {
1049 		put_bh(bh);
1050 		return ERR_PTR(ret);
1051 	}
1052 	return bh;
1053 }
1054 
1055 /* Read a contiguous batch of blocks. */
ext4_bread_batch(struct inode * inode,ext4_lblk_t block,int bh_count,bool wait,struct buffer_head ** bhs)1056 int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count,
1057 		     bool wait, struct buffer_head **bhs)
1058 {
1059 	int i, err;
1060 
1061 	for (i = 0; i < bh_count; i++) {
1062 		bhs[i] = ext4_getblk(NULL, inode, block + i, 0 /* map_flags */);
1063 		if (IS_ERR(bhs[i])) {
1064 			err = PTR_ERR(bhs[i]);
1065 			bh_count = i;
1066 			goto out_brelse;
1067 		}
1068 	}
1069 
1070 	for (i = 0; i < bh_count; i++)
1071 		/* Note that NULL bhs[i] is valid because of holes. */
1072 		if (bhs[i] && !ext4_buffer_uptodate(bhs[i]))
1073 			ext4_read_bh_lock(bhs[i], REQ_META | REQ_PRIO, false);
1074 
1075 	if (!wait)
1076 		return 0;
1077 
1078 	for (i = 0; i < bh_count; i++)
1079 		if (bhs[i])
1080 			wait_on_buffer(bhs[i]);
1081 
1082 	for (i = 0; i < bh_count; i++) {
1083 		if (bhs[i] && !buffer_uptodate(bhs[i])) {
1084 			err = -EIO;
1085 			goto out_brelse;
1086 		}
1087 	}
1088 	return 0;
1089 
1090 out_brelse:
1091 	for (i = 0; i < bh_count; i++) {
1092 		brelse(bhs[i]);
1093 		bhs[i] = NULL;
1094 	}
1095 	return err;
1096 }
1097 
ext4_walk_page_buffers(handle_t * handle,struct inode * inode,struct buffer_head * head,unsigned from,unsigned to,int * partial,int (* fn)(handle_t * handle,struct inode * inode,struct buffer_head * bh))1098 int ext4_walk_page_buffers(handle_t *handle, struct inode *inode,
1099 			   struct buffer_head *head,
1100 			   unsigned from,
1101 			   unsigned to,
1102 			   int *partial,
1103 			   int (*fn)(handle_t *handle, struct inode *inode,
1104 				     struct buffer_head *bh))
1105 {
1106 	struct buffer_head *bh;
1107 	unsigned block_start, block_end;
1108 	unsigned blocksize = head->b_size;
1109 	int err, ret = 0;
1110 	struct buffer_head *next;
1111 
1112 	for (bh = head, block_start = 0;
1113 	     ret == 0 && (bh != head || !block_start);
1114 	     block_start = block_end, bh = next) {
1115 		next = bh->b_this_page;
1116 		block_end = block_start + blocksize;
1117 		if (block_end <= from || block_start >= to) {
1118 			if (partial && !buffer_uptodate(bh))
1119 				*partial = 1;
1120 			continue;
1121 		}
1122 		err = (*fn)(handle, inode, bh);
1123 		if (!ret)
1124 			ret = err;
1125 	}
1126 	return ret;
1127 }
1128 
1129 /*
1130  * Helper for handling dirtying of journalled data. We also mark the folio as
1131  * dirty so that writeback code knows about this page (and inode) contains
1132  * dirty data. ext4_writepages() then commits appropriate transaction to
1133  * make data stable.
1134  */
ext4_dirty_journalled_data(handle_t * handle,struct buffer_head * bh)1135 static int ext4_dirty_journalled_data(handle_t *handle, struct buffer_head *bh)
1136 {
1137 	struct folio *folio = bh->b_folio;
1138 	struct inode *inode = folio->mapping->host;
1139 
1140 	/* only regular files have a_ops */
1141 	if (S_ISREG(inode->i_mode))
1142 		folio_mark_dirty(folio);
1143 	return ext4_handle_dirty_metadata(handle, NULL, bh);
1144 }
1145 
do_journal_get_write_access(handle_t * handle,struct inode * inode,struct buffer_head * bh)1146 int do_journal_get_write_access(handle_t *handle, struct inode *inode,
1147 				struct buffer_head *bh)
1148 {
1149 	if (!buffer_mapped(bh) || buffer_freed(bh))
1150 		return 0;
1151 	BUFFER_TRACE(bh, "get write access");
1152 	return ext4_journal_get_write_access(handle, inode->i_sb, bh,
1153 					    EXT4_JTR_NONE);
1154 }
1155 
ext4_block_write_begin(handle_t * handle,struct folio * folio,loff_t pos,unsigned len,get_block_t * get_block)1156 int ext4_block_write_begin(handle_t *handle, struct folio *folio,
1157 			   loff_t pos, unsigned len,
1158 			   get_block_t *get_block)
1159 {
1160 	unsigned int from = offset_in_folio(folio, pos);
1161 	unsigned to = from + len;
1162 	struct inode *inode = folio->mapping->host;
1163 	unsigned block_start, block_end;
1164 	sector_t block;
1165 	int err = 0;
1166 	unsigned blocksize = inode->i_sb->s_blocksize;
1167 	unsigned bbits;
1168 	struct buffer_head *bh, *head, *wait[2];
1169 	int nr_wait = 0;
1170 	int i;
1171 	bool should_journal_data = ext4_should_journal_data(inode);
1172 
1173 	BUG_ON(!folio_test_locked(folio));
1174 	BUG_ON(to > folio_size(folio));
1175 	BUG_ON(from > to);
1176 
1177 	head = folio_buffers(folio);
1178 	if (!head)
1179 		head = create_empty_buffers(folio, blocksize, 0);
1180 	bbits = ilog2(blocksize);
1181 	block = (sector_t)folio->index << (PAGE_SHIFT - bbits);
1182 
1183 	for (bh = head, block_start = 0; bh != head || !block_start;
1184 	    block++, block_start = block_end, bh = bh->b_this_page) {
1185 		block_end = block_start + blocksize;
1186 		if (block_end <= from || block_start >= to) {
1187 			if (folio_test_uptodate(folio)) {
1188 				set_buffer_uptodate(bh);
1189 			}
1190 			continue;
1191 		}
1192 		if (WARN_ON_ONCE(buffer_new(bh)))
1193 			clear_buffer_new(bh);
1194 		if (!buffer_mapped(bh)) {
1195 			WARN_ON(bh->b_size != blocksize);
1196 			err = ext4_journal_ensure_extent_credits(handle, inode);
1197 			if (!err)
1198 				err = get_block(inode, block, bh, 1);
1199 			if (err)
1200 				break;
1201 			if (buffer_new(bh)) {
1202 				/*
1203 				 * We may be zeroing partial buffers or all new
1204 				 * buffers in case of failure. Prepare JBD2 for
1205 				 * that.
1206 				 */
1207 				if (should_journal_data)
1208 					do_journal_get_write_access(handle,
1209 								    inode, bh);
1210 				if (folio_test_uptodate(folio)) {
1211 					/*
1212 					 * Unlike __block_write_begin() we leave
1213 					 * dirtying of new uptodate buffers to
1214 					 * ->write_end() time or
1215 					 * folio_zero_new_buffers().
1216 					 */
1217 					set_buffer_uptodate(bh);
1218 					continue;
1219 				}
1220 				if (block_end > to || block_start < from)
1221 					folio_zero_segments(folio, to,
1222 							    block_end,
1223 							    block_start, from);
1224 				continue;
1225 			}
1226 		}
1227 		if (folio_test_uptodate(folio)) {
1228 			set_buffer_uptodate(bh);
1229 			continue;
1230 		}
1231 		if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1232 		    !buffer_unwritten(bh) &&
1233 		    (block_start < from || block_end > to)) {
1234 			ext4_read_bh_lock(bh, 0, false);
1235 			wait[nr_wait++] = bh;
1236 		}
1237 	}
1238 	/*
1239 	 * If we issued read requests, let them complete.
1240 	 */
1241 	for (i = 0; i < nr_wait; i++) {
1242 		wait_on_buffer(wait[i]);
1243 		if (!buffer_uptodate(wait[i]))
1244 			err = -EIO;
1245 	}
1246 	if (unlikely(err)) {
1247 		if (should_journal_data)
1248 			ext4_journalled_zero_new_buffers(handle, inode, folio,
1249 							 from, to);
1250 		else
1251 			folio_zero_new_buffers(folio, from, to);
1252 	} else if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
1253 		for (i = 0; i < nr_wait; i++) {
1254 			int err2;
1255 
1256 			err2 = fscrypt_decrypt_pagecache_blocks(folio,
1257 						blocksize, bh_offset(wait[i]));
1258 			if (err2) {
1259 				clear_buffer_uptodate(wait[i]);
1260 				err = err2;
1261 			}
1262 		}
1263 	}
1264 
1265 	return err;
1266 }
1267 
1268 /*
1269  * To preserve ordering, it is essential that the hole instantiation and
1270  * the data write be encapsulated in a single transaction.  We cannot
1271  * close off a transaction and start a new one between the ext4_get_block()
1272  * and the ext4_write_end().  So doing the jbd2_journal_start at the start of
1273  * ext4_write_begin() is the right place.
1274  */
ext4_write_begin(const struct kiocb * iocb,struct address_space * mapping,loff_t pos,unsigned len,struct folio ** foliop,void ** fsdata)1275 static int ext4_write_begin(const struct kiocb *iocb,
1276 			    struct address_space *mapping,
1277 			    loff_t pos, unsigned len,
1278 			    struct folio **foliop, void **fsdata)
1279 {
1280 	struct inode *inode = mapping->host;
1281 	int ret, needed_blocks;
1282 	handle_t *handle;
1283 	int retries = 0;
1284 	struct folio *folio;
1285 	pgoff_t index;
1286 	unsigned from, to;
1287 
1288 	ret = ext4_emergency_state(inode->i_sb);
1289 	if (unlikely(ret))
1290 		return ret;
1291 
1292 	trace_ext4_write_begin(inode, pos, len);
1293 	/*
1294 	 * Reserve one block more for addition to orphan list in case
1295 	 * we allocate blocks but write fails for some reason
1296 	 */
1297 	needed_blocks = ext4_chunk_trans_extent(inode,
1298 			ext4_journal_blocks_per_folio(inode)) + 1;
1299 	index = pos >> PAGE_SHIFT;
1300 
1301 	if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
1302 		ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,
1303 						    foliop);
1304 		if (ret < 0)
1305 			return ret;
1306 		if (ret == 1)
1307 			return 0;
1308 	}
1309 
1310 	/*
1311 	 * write_begin_get_folio() can take a long time if the
1312 	 * system is thrashing due to memory pressure, or if the folio
1313 	 * is being written back.  So grab it first before we start
1314 	 * the transaction handle.  This also allows us to allocate
1315 	 * the folio (if needed) without using GFP_NOFS.
1316 	 */
1317 retry_grab:
1318 	folio = write_begin_get_folio(iocb, mapping, index, len);
1319 	if (IS_ERR(folio))
1320 		return PTR_ERR(folio);
1321 
1322 	if (pos + len > folio_pos(folio) + folio_size(folio))
1323 		len = folio_pos(folio) + folio_size(folio) - pos;
1324 
1325 	from = offset_in_folio(folio, pos);
1326 	to = from + len;
1327 
1328 	/*
1329 	 * The same as page allocation, we prealloc buffer heads before
1330 	 * starting the handle.
1331 	 */
1332 	if (!folio_buffers(folio))
1333 		create_empty_buffers(folio, inode->i_sb->s_blocksize, 0);
1334 
1335 	folio_unlock(folio);
1336 
1337 retry_journal:
1338 	handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
1339 	if (IS_ERR(handle)) {
1340 		folio_put(folio);
1341 		return PTR_ERR(handle);
1342 	}
1343 
1344 	folio_lock(folio);
1345 	if (folio->mapping != mapping) {
1346 		/* The folio got truncated from under us */
1347 		folio_unlock(folio);
1348 		folio_put(folio);
1349 		ext4_journal_stop(handle);
1350 		goto retry_grab;
1351 	}
1352 	/* In case writeback began while the folio was unlocked */
1353 	folio_wait_stable(folio);
1354 
1355 	if (ext4_should_dioread_nolock(inode))
1356 		ret = ext4_block_write_begin(handle, folio, pos, len,
1357 					     ext4_get_block_unwritten);
1358 	else
1359 		ret = ext4_block_write_begin(handle, folio, pos, len,
1360 					     ext4_get_block);
1361 	if (!ret && ext4_should_journal_data(inode)) {
1362 		ret = ext4_walk_page_buffers(handle, inode,
1363 					     folio_buffers(folio), from, to,
1364 					     NULL, do_journal_get_write_access);
1365 	}
1366 
1367 	if (ret) {
1368 		bool extended = (pos + len > inode->i_size) &&
1369 				!ext4_verity_in_progress(inode);
1370 
1371 		folio_unlock(folio);
1372 		/*
1373 		 * ext4_block_write_begin may have instantiated a few blocks
1374 		 * outside i_size.  Trim these off again. Don't need
1375 		 * i_size_read because we hold i_rwsem.
1376 		 *
1377 		 * Add inode to orphan list in case we crash before
1378 		 * truncate finishes
1379 		 */
1380 		if (extended && ext4_can_truncate(inode))
1381 			ext4_orphan_add(handle, inode);
1382 
1383 		ext4_journal_stop(handle);
1384 		if (extended) {
1385 			ext4_truncate_failed_write(inode);
1386 			/*
1387 			 * If truncate failed early the inode might
1388 			 * still be on the orphan list; we need to
1389 			 * make sure the inode is removed from the
1390 			 * orphan list in that case.
1391 			 */
1392 			if (inode->i_nlink)
1393 				ext4_orphan_del(NULL, inode);
1394 		}
1395 
1396 		if (ret == -EAGAIN ||
1397 		    (ret == -ENOSPC &&
1398 		     ext4_should_retry_alloc(inode->i_sb, &retries)))
1399 			goto retry_journal;
1400 		folio_put(folio);
1401 		return ret;
1402 	}
1403 	*foliop = folio;
1404 	return ret;
1405 }
1406 
1407 /* For write_end() in data=journal mode */
write_end_fn(handle_t * handle,struct inode * inode,struct buffer_head * bh)1408 static int write_end_fn(handle_t *handle, struct inode *inode,
1409 			struct buffer_head *bh)
1410 {
1411 	int ret;
1412 	if (!buffer_mapped(bh) || buffer_freed(bh))
1413 		return 0;
1414 	set_buffer_uptodate(bh);
1415 	ret = ext4_dirty_journalled_data(handle, bh);
1416 	clear_buffer_meta(bh);
1417 	clear_buffer_prio(bh);
1418 	clear_buffer_new(bh);
1419 	return ret;
1420 }
1421 
1422 /*
1423  * We need to pick up the new inode size which generic_commit_write gave us
1424  * `iocb` can be NULL - eg, when called from page_symlink().
1425  *
1426  * ext4 never places buffers on inode->i_mapping->i_private_list.  metadata
1427  * buffers are managed internally.
1428  */
ext4_write_end(const struct kiocb * iocb,struct address_space * mapping,loff_t pos,unsigned len,unsigned copied,struct folio * folio,void * fsdata)1429 static int ext4_write_end(const struct kiocb *iocb,
1430 			  struct address_space *mapping,
1431 			  loff_t pos, unsigned len, unsigned copied,
1432 			  struct folio *folio, void *fsdata)
1433 {
1434 	handle_t *handle = ext4_journal_current_handle();
1435 	struct inode *inode = mapping->host;
1436 	loff_t old_size = inode->i_size;
1437 	int ret = 0, ret2;
1438 	int i_size_changed = 0;
1439 	bool verity = ext4_verity_in_progress(inode);
1440 
1441 	trace_ext4_write_end(inode, pos, len, copied);
1442 
1443 	if (ext4_has_inline_data(inode) &&
1444 	    ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))
1445 		return ext4_write_inline_data_end(inode, pos, len, copied,
1446 						  folio);
1447 
1448 	copied = block_write_end(pos, len, copied, folio);
1449 	/*
1450 	 * it's important to update i_size while still holding folio lock:
1451 	 * page writeout could otherwise come in and zero beyond i_size.
1452 	 *
1453 	 * If FS_IOC_ENABLE_VERITY is running on this inode, then Merkle tree
1454 	 * blocks are being written past EOF, so skip the i_size update.
1455 	 */
1456 	if (!verity)
1457 		i_size_changed = ext4_update_inode_size(inode, pos + copied);
1458 	folio_unlock(folio);
1459 	folio_put(folio);
1460 
1461 	if (old_size < pos && !verity) {
1462 		pagecache_isize_extended(inode, old_size, pos);
1463 		ext4_zero_partial_blocks(handle, inode, old_size, pos - old_size);
1464 	}
1465 	/*
1466 	 * Don't mark the inode dirty under folio lock. First, it unnecessarily
1467 	 * makes the holding time of folio lock longer. Second, it forces lock
1468 	 * ordering of folio lock and transaction start for journaling
1469 	 * filesystems.
1470 	 */
1471 	if (i_size_changed)
1472 		ret = ext4_mark_inode_dirty(handle, inode);
1473 
1474 	if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
1475 		/* if we have allocated more blocks and copied
1476 		 * less. We will have blocks allocated outside
1477 		 * inode->i_size. So truncate them
1478 		 */
1479 		ext4_orphan_add(handle, inode);
1480 
1481 	ret2 = ext4_journal_stop(handle);
1482 	if (!ret)
1483 		ret = ret2;
1484 
1485 	if (pos + len > inode->i_size && !verity) {
1486 		ext4_truncate_failed_write(inode);
1487 		/*
1488 		 * If truncate failed early the inode might still be
1489 		 * on the orphan list; we need to make sure the inode
1490 		 * is removed from the orphan list in that case.
1491 		 */
1492 		if (inode->i_nlink)
1493 			ext4_orphan_del(NULL, inode);
1494 	}
1495 
1496 	return ret ? ret : copied;
1497 }
1498 
1499 /*
1500  * This is a private version of folio_zero_new_buffers() which doesn't
1501  * set the buffer to be dirty, since in data=journalled mode we need
1502  * to call ext4_dirty_journalled_data() instead.
1503  */
ext4_journalled_zero_new_buffers(handle_t * handle,struct inode * inode,struct folio * folio,unsigned from,unsigned to)1504 static void ext4_journalled_zero_new_buffers(handle_t *handle,
1505 					    struct inode *inode,
1506 					    struct folio *folio,
1507 					    unsigned from, unsigned to)
1508 {
1509 	unsigned int block_start = 0, block_end;
1510 	struct buffer_head *head, *bh;
1511 
1512 	bh = head = folio_buffers(folio);
1513 	do {
1514 		block_end = block_start + bh->b_size;
1515 		if (buffer_new(bh)) {
1516 			if (block_end > from && block_start < to) {
1517 				if (!folio_test_uptodate(folio)) {
1518 					unsigned start, size;
1519 
1520 					start = max(from, block_start);
1521 					size = min(to, block_end) - start;
1522 
1523 					folio_zero_range(folio, start, size);
1524 				}
1525 				clear_buffer_new(bh);
1526 				write_end_fn(handle, inode, bh);
1527 			}
1528 		}
1529 		block_start = block_end;
1530 		bh = bh->b_this_page;
1531 	} while (bh != head);
1532 }
1533 
ext4_journalled_write_end(const struct kiocb * iocb,struct address_space * mapping,loff_t pos,unsigned len,unsigned copied,struct folio * folio,void * fsdata)1534 static int ext4_journalled_write_end(const struct kiocb *iocb,
1535 				     struct address_space *mapping,
1536 				     loff_t pos, unsigned len, unsigned copied,
1537 				     struct folio *folio, void *fsdata)
1538 {
1539 	handle_t *handle = ext4_journal_current_handle();
1540 	struct inode *inode = mapping->host;
1541 	loff_t old_size = inode->i_size;
1542 	int ret = 0, ret2;
1543 	int partial = 0;
1544 	unsigned from, to;
1545 	int size_changed = 0;
1546 	bool verity = ext4_verity_in_progress(inode);
1547 
1548 	trace_ext4_journalled_write_end(inode, pos, len, copied);
1549 	from = pos & (PAGE_SIZE - 1);
1550 	to = from + len;
1551 
1552 	BUG_ON(!ext4_handle_valid(handle));
1553 
1554 	if (ext4_has_inline_data(inode))
1555 		return ext4_write_inline_data_end(inode, pos, len, copied,
1556 						  folio);
1557 
1558 	if (unlikely(copied < len) && !folio_test_uptodate(folio)) {
1559 		copied = 0;
1560 		ext4_journalled_zero_new_buffers(handle, inode, folio,
1561 						 from, to);
1562 	} else {
1563 		if (unlikely(copied < len))
1564 			ext4_journalled_zero_new_buffers(handle, inode, folio,
1565 							 from + copied, to);
1566 		ret = ext4_walk_page_buffers(handle, inode,
1567 					     folio_buffers(folio),
1568 					     from, from + copied, &partial,
1569 					     write_end_fn);
1570 		if (!partial)
1571 			folio_mark_uptodate(folio);
1572 	}
1573 	if (!verity)
1574 		size_changed = ext4_update_inode_size(inode, pos + copied);
1575 	EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1576 	folio_unlock(folio);
1577 	folio_put(folio);
1578 
1579 	if (old_size < pos && !verity) {
1580 		pagecache_isize_extended(inode, old_size, pos);
1581 		ext4_zero_partial_blocks(handle, inode, old_size, pos - old_size);
1582 	}
1583 
1584 	if (size_changed) {
1585 		ret2 = ext4_mark_inode_dirty(handle, inode);
1586 		if (!ret)
1587 			ret = ret2;
1588 	}
1589 
1590 	if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
1591 		/* if we have allocated more blocks and copied
1592 		 * less. We will have blocks allocated outside
1593 		 * inode->i_size. So truncate them
1594 		 */
1595 		ext4_orphan_add(handle, inode);
1596 
1597 	ret2 = ext4_journal_stop(handle);
1598 	if (!ret)
1599 		ret = ret2;
1600 	if (pos + len > inode->i_size && !verity) {
1601 		ext4_truncate_failed_write(inode);
1602 		/*
1603 		 * If truncate failed early the inode might still be
1604 		 * on the orphan list; we need to make sure the inode
1605 		 * is removed from the orphan list in that case.
1606 		 */
1607 		if (inode->i_nlink)
1608 			ext4_orphan_del(NULL, inode);
1609 	}
1610 
1611 	return ret ? ret : copied;
1612 }
1613 
1614 /*
1615  * Reserve space for 'nr_resv' clusters
1616  */
ext4_da_reserve_space(struct inode * inode,int nr_resv)1617 static int ext4_da_reserve_space(struct inode *inode, int nr_resv)
1618 {
1619 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1620 	struct ext4_inode_info *ei = EXT4_I(inode);
1621 	int ret;
1622 
1623 	/*
1624 	 * We will charge metadata quota at writeout time; this saves
1625 	 * us from metadata over-estimation, though we may go over by
1626 	 * a small amount in the end.  Here we just reserve for data.
1627 	 */
1628 	ret = dquot_reserve_block(inode, EXT4_C2B(sbi, nr_resv));
1629 	if (ret)
1630 		return ret;
1631 
1632 	spin_lock(&ei->i_block_reservation_lock);
1633 	if (ext4_claim_free_clusters(sbi, nr_resv, 0)) {
1634 		spin_unlock(&ei->i_block_reservation_lock);
1635 		dquot_release_reservation_block(inode, EXT4_C2B(sbi, nr_resv));
1636 		return -ENOSPC;
1637 	}
1638 	ei->i_reserved_data_blocks += nr_resv;
1639 	trace_ext4_da_reserve_space(inode, nr_resv);
1640 	spin_unlock(&ei->i_block_reservation_lock);
1641 
1642 	return 0;       /* success */
1643 }
1644 
ext4_da_release_space(struct inode * inode,int to_free)1645 void ext4_da_release_space(struct inode *inode, int to_free)
1646 {
1647 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1648 	struct ext4_inode_info *ei = EXT4_I(inode);
1649 
1650 	if (!to_free)
1651 		return;		/* Nothing to release, exit */
1652 
1653 	spin_lock(&EXT4_I(inode)->i_block_reservation_lock);
1654 
1655 	trace_ext4_da_release_space(inode, to_free);
1656 	if (unlikely(to_free > ei->i_reserved_data_blocks)) {
1657 		/*
1658 		 * if there aren't enough reserved blocks, then the
1659 		 * counter is messed up somewhere.  Since this
1660 		 * function is called from invalidate page, it's
1661 		 * harmless to return without any action.
1662 		 */
1663 		ext4_warning(inode->i_sb, "ext4_da_release_space: "
1664 			 "ino %lu, to_free %d with only %d reserved "
1665 			 "data blocks", inode->i_ino, to_free,
1666 			 ei->i_reserved_data_blocks);
1667 		WARN_ON(1);
1668 		to_free = ei->i_reserved_data_blocks;
1669 	}
1670 	ei->i_reserved_data_blocks -= to_free;
1671 
1672 	/* update fs dirty data blocks counter */
1673 	percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);
1674 
1675 	spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
1676 
1677 	dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
1678 }
1679 
1680 /*
1681  * Delayed allocation stuff
1682  */
1683 
1684 struct mpage_da_data {
1685 	/* These are input fields for ext4_do_writepages() */
1686 	struct inode *inode;
1687 	struct writeback_control *wbc;
1688 	unsigned int can_map:1;	/* Can writepages call map blocks? */
1689 
1690 	/* These are internal state of ext4_do_writepages() */
1691 	loff_t start_pos;	/* The start pos to write */
1692 	loff_t next_pos;	/* Current pos to examine */
1693 	loff_t end_pos;		/* Last pos to examine */
1694 
1695 	/*
1696 	 * Extent to map - this can be after start_pos because that can be
1697 	 * fully mapped. We somewhat abuse m_flags to store whether the extent
1698 	 * is delalloc or unwritten.
1699 	 */
1700 	struct ext4_map_blocks map;
1701 	struct ext4_io_submit io_submit;	/* IO submission data */
1702 	unsigned int do_map:1;
1703 	unsigned int scanned_until_end:1;
1704 	unsigned int journalled_more_data:1;
1705 };
1706 
mpage_release_unused_pages(struct mpage_da_data * mpd,bool invalidate)1707 static void mpage_release_unused_pages(struct mpage_da_data *mpd,
1708 				       bool invalidate)
1709 {
1710 	unsigned nr, i;
1711 	pgoff_t index, end;
1712 	struct folio_batch fbatch;
1713 	struct inode *inode = mpd->inode;
1714 	struct address_space *mapping = inode->i_mapping;
1715 
1716 	/* This is necessary when next_pos == 0. */
1717 	if (mpd->start_pos >= mpd->next_pos)
1718 		return;
1719 
1720 	mpd->scanned_until_end = 0;
1721 	if (invalidate) {
1722 		ext4_lblk_t start, last;
1723 		start = EXT4_B_TO_LBLK(inode, mpd->start_pos);
1724 		last = mpd->next_pos >> inode->i_blkbits;
1725 
1726 		/*
1727 		 * avoid racing with extent status tree scans made by
1728 		 * ext4_insert_delayed_block()
1729 		 */
1730 		down_write(&EXT4_I(inode)->i_data_sem);
1731 		ext4_es_remove_extent(inode, start, last - start);
1732 		up_write(&EXT4_I(inode)->i_data_sem);
1733 	}
1734 
1735 	folio_batch_init(&fbatch);
1736 	index = mpd->start_pos >> PAGE_SHIFT;
1737 	end = mpd->next_pos >> PAGE_SHIFT;
1738 	while (index < end) {
1739 		nr = filemap_get_folios(mapping, &index, end - 1, &fbatch);
1740 		if (nr == 0)
1741 			break;
1742 		for (i = 0; i < nr; i++) {
1743 			struct folio *folio = fbatch.folios[i];
1744 
1745 			if (folio_pos(folio) < mpd->start_pos)
1746 				continue;
1747 			if (folio_next_index(folio) > end)
1748 				continue;
1749 			BUG_ON(!folio_test_locked(folio));
1750 			BUG_ON(folio_test_writeback(folio));
1751 			if (invalidate) {
1752 				if (folio_mapped(folio))
1753 					folio_clear_dirty_for_io(folio);
1754 				block_invalidate_folio(folio, 0,
1755 						folio_size(folio));
1756 				folio_clear_uptodate(folio);
1757 			}
1758 			folio_unlock(folio);
1759 		}
1760 		folio_batch_release(&fbatch);
1761 	}
1762 }
1763 
ext4_print_free_blocks(struct inode * inode)1764 static void ext4_print_free_blocks(struct inode *inode)
1765 {
1766 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1767 	struct super_block *sb = inode->i_sb;
1768 	struct ext4_inode_info *ei = EXT4_I(inode);
1769 
1770 	ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
1771 	       EXT4_C2B(EXT4_SB(inode->i_sb),
1772 			ext4_count_free_clusters(sb)));
1773 	ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
1774 	ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
1775 	       (long long) EXT4_C2B(EXT4_SB(sb),
1776 		percpu_counter_sum(&sbi->s_freeclusters_counter)));
1777 	ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
1778 	       (long long) EXT4_C2B(EXT4_SB(sb),
1779 		percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
1780 	ext4_msg(sb, KERN_CRIT, "Block reservation details");
1781 	ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
1782 		 ei->i_reserved_data_blocks);
1783 	return;
1784 }
1785 
1786 /*
1787  * Check whether the cluster containing lblk has been allocated or has
1788  * delalloc reservation.
1789  *
1790  * Returns 0 if the cluster doesn't have either, 1 if it has delalloc
1791  * reservation, 2 if it's already been allocated, negative error code on
1792  * failure.
1793  */
ext4_clu_alloc_state(struct inode * inode,ext4_lblk_t lblk)1794 static int ext4_clu_alloc_state(struct inode *inode, ext4_lblk_t lblk)
1795 {
1796 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1797 	int ret;
1798 
1799 	/* Has delalloc reservation? */
1800 	if (ext4_es_scan_clu(inode, &ext4_es_is_delayed, lblk))
1801 		return 1;
1802 
1803 	/* Already been allocated? */
1804 	if (ext4_es_scan_clu(inode, &ext4_es_is_mapped, lblk))
1805 		return 2;
1806 	ret = ext4_clu_mapped(inode, EXT4_B2C(sbi, lblk));
1807 	if (ret < 0)
1808 		return ret;
1809 	if (ret > 0)
1810 		return 2;
1811 
1812 	return 0;
1813 }
1814 
1815 /*
1816  * ext4_insert_delayed_blocks - adds a multiple delayed blocks to the extents
1817  *                              status tree, incrementing the reserved
1818  *                              cluster/block count or making pending
1819  *                              reservations where needed
1820  *
1821  * @inode - file containing the newly added block
1822  * @lblk - start logical block to be added
1823  * @len - length of blocks to be added
1824  *
1825  * Returns 0 on success, negative error code on failure.
1826  */
ext4_insert_delayed_blocks(struct inode * inode,ext4_lblk_t lblk,ext4_lblk_t len)1827 static int ext4_insert_delayed_blocks(struct inode *inode, ext4_lblk_t lblk,
1828 				      ext4_lblk_t len)
1829 {
1830 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1831 	int ret;
1832 	bool lclu_allocated = false;
1833 	bool end_allocated = false;
1834 	ext4_lblk_t resv_clu;
1835 	ext4_lblk_t end = lblk + len - 1;
1836 
1837 	/*
1838 	 * If the cluster containing lblk or end is shared with a delayed,
1839 	 * written, or unwritten extent in a bigalloc file system, it's
1840 	 * already been accounted for and does not need to be reserved.
1841 	 * A pending reservation must be made for the cluster if it's
1842 	 * shared with a written or unwritten extent and doesn't already
1843 	 * have one.  Written and unwritten extents can be purged from the
1844 	 * extents status tree if the system is under memory pressure, so
1845 	 * it's necessary to examine the extent tree if a search of the
1846 	 * extents status tree doesn't get a match.
1847 	 */
1848 	if (sbi->s_cluster_ratio == 1) {
1849 		ret = ext4_da_reserve_space(inode, len);
1850 		if (ret != 0)   /* ENOSPC */
1851 			return ret;
1852 	} else {   /* bigalloc */
1853 		resv_clu = EXT4_B2C(sbi, end) - EXT4_B2C(sbi, lblk) + 1;
1854 
1855 		ret = ext4_clu_alloc_state(inode, lblk);
1856 		if (ret < 0)
1857 			return ret;
1858 		if (ret > 0) {
1859 			resv_clu--;
1860 			lclu_allocated = (ret == 2);
1861 		}
1862 
1863 		if (EXT4_B2C(sbi, lblk) != EXT4_B2C(sbi, end)) {
1864 			ret = ext4_clu_alloc_state(inode, end);
1865 			if (ret < 0)
1866 				return ret;
1867 			if (ret > 0) {
1868 				resv_clu--;
1869 				end_allocated = (ret == 2);
1870 			}
1871 		}
1872 
1873 		if (resv_clu) {
1874 			ret = ext4_da_reserve_space(inode, resv_clu);
1875 			if (ret != 0)   /* ENOSPC */
1876 				return ret;
1877 		}
1878 	}
1879 
1880 	ext4_es_insert_delayed_extent(inode, lblk, len, lclu_allocated,
1881 				      end_allocated);
1882 	return 0;
1883 }
1884 
1885 /*
1886  * Looks up the requested blocks and sets the delalloc extent map.
1887  * First try to look up for the extent entry that contains the requested
1888  * blocks in the extent status tree without i_data_sem, then try to look
1889  * up for the ondisk extent mapping with i_data_sem in read mode,
1890  * finally hold i_data_sem in write mode, looks up again and add a
1891  * delalloc extent entry if it still couldn't find any extent. Pass out
1892  * the mapped extent through @map and return 0 on success.
1893  */
ext4_da_map_blocks(struct inode * inode,struct ext4_map_blocks * map)1894 static int ext4_da_map_blocks(struct inode *inode, struct ext4_map_blocks *map)
1895 {
1896 	struct extent_status es;
1897 	int retval;
1898 #ifdef ES_AGGRESSIVE_TEST
1899 	struct ext4_map_blocks orig_map;
1900 
1901 	memcpy(&orig_map, map, sizeof(*map));
1902 #endif
1903 
1904 	map->m_flags = 0;
1905 	ext_debug(inode, "max_blocks %u, logical block %lu\n", map->m_len,
1906 		  (unsigned long) map->m_lblk);
1907 
1908 	ext4_check_map_extents_env(inode);
1909 
1910 	/* Lookup extent status tree firstly */
1911 	if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {
1912 		map->m_len = min_t(unsigned int, map->m_len,
1913 				   es.es_len - (map->m_lblk - es.es_lblk));
1914 
1915 		if (ext4_es_is_hole(&es))
1916 			goto add_delayed;
1917 
1918 found:
1919 		/*
1920 		 * Delayed extent could be allocated by fallocate.
1921 		 * So we need to check it.
1922 		 */
1923 		if (ext4_es_is_delayed(&es)) {
1924 			map->m_flags |= EXT4_MAP_DELAYED;
1925 			return 0;
1926 		}
1927 
1928 		map->m_pblk = ext4_es_pblock(&es) + map->m_lblk - es.es_lblk;
1929 		if (ext4_es_is_written(&es))
1930 			map->m_flags |= EXT4_MAP_MAPPED;
1931 		else if (ext4_es_is_unwritten(&es))
1932 			map->m_flags |= EXT4_MAP_UNWRITTEN;
1933 		else
1934 			BUG();
1935 
1936 #ifdef ES_AGGRESSIVE_TEST
1937 		ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);
1938 #endif
1939 		return 0;
1940 	}
1941 
1942 	/*
1943 	 * Try to see if we can get the block without requesting a new
1944 	 * file system block.
1945 	 */
1946 	down_read(&EXT4_I(inode)->i_data_sem);
1947 	if (ext4_has_inline_data(inode))
1948 		retval = 0;
1949 	else
1950 		retval = ext4_map_query_blocks(NULL, inode, map, 0);
1951 	up_read(&EXT4_I(inode)->i_data_sem);
1952 	if (retval)
1953 		return retval < 0 ? retval : 0;
1954 
1955 add_delayed:
1956 	down_write(&EXT4_I(inode)->i_data_sem);
1957 	/*
1958 	 * Page fault path (ext4_page_mkwrite does not take i_rwsem)
1959 	 * and fallocate path (no folio lock) can race. Make sure we
1960 	 * lookup the extent status tree here again while i_data_sem
1961 	 * is held in write mode, before inserting a new da entry in
1962 	 * the extent status tree.
1963 	 */
1964 	if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {
1965 		map->m_len = min_t(unsigned int, map->m_len,
1966 				   es.es_len - (map->m_lblk - es.es_lblk));
1967 
1968 		if (!ext4_es_is_hole(&es)) {
1969 			up_write(&EXT4_I(inode)->i_data_sem);
1970 			goto found;
1971 		}
1972 	} else if (!ext4_has_inline_data(inode)) {
1973 		retval = ext4_map_query_blocks(NULL, inode, map, 0);
1974 		if (retval) {
1975 			up_write(&EXT4_I(inode)->i_data_sem);
1976 			return retval < 0 ? retval : 0;
1977 		}
1978 	}
1979 
1980 	map->m_flags |= EXT4_MAP_DELAYED;
1981 	retval = ext4_insert_delayed_blocks(inode, map->m_lblk, map->m_len);
1982 	up_write(&EXT4_I(inode)->i_data_sem);
1983 
1984 	return retval;
1985 }
1986 
1987 /*
1988  * This is a special get_block_t callback which is used by
1989  * ext4_da_write_begin().  It will either return mapped block or
1990  * reserve space for a single block.
1991  *
1992  * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.
1993  * We also have b_blocknr = -1 and b_bdev initialized properly
1994  *
1995  * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.
1996  * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev
1997  * initialized properly.
1998  */
ext4_da_get_block_prep(struct inode * inode,sector_t iblock,struct buffer_head * bh,int create)1999 int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
2000 			   struct buffer_head *bh, int create)
2001 {
2002 	struct ext4_map_blocks map;
2003 	sector_t invalid_block = ~((sector_t) 0xffff);
2004 	int ret = 0;
2005 
2006 	BUG_ON(create == 0);
2007 	BUG_ON(bh->b_size != inode->i_sb->s_blocksize);
2008 
2009 	if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))
2010 		invalid_block = ~0;
2011 
2012 	map.m_lblk = iblock;
2013 	map.m_len = 1;
2014 
2015 	/*
2016 	 * first, we need to know whether the block is allocated already
2017 	 * preallocated blocks are unmapped but should treated
2018 	 * the same as allocated blocks.
2019 	 */
2020 	ret = ext4_da_map_blocks(inode, &map);
2021 	if (ret < 0)
2022 		return ret;
2023 
2024 	if (map.m_flags & EXT4_MAP_DELAYED) {
2025 		map_bh(bh, inode->i_sb, invalid_block);
2026 		set_buffer_new(bh);
2027 		set_buffer_delay(bh);
2028 		return 0;
2029 	}
2030 
2031 	map_bh(bh, inode->i_sb, map.m_pblk);
2032 	ext4_update_bh_state(bh, map.m_flags);
2033 
2034 	if (buffer_unwritten(bh)) {
2035 		/* A delayed write to unwritten bh should be marked
2036 		 * new and mapped.  Mapped ensures that we don't do
2037 		 * get_block multiple times when we write to the same
2038 		 * offset and new ensures that we do proper zero out
2039 		 * for partial write.
2040 		 */
2041 		set_buffer_new(bh);
2042 		set_buffer_mapped(bh);
2043 	}
2044 	return 0;
2045 }
2046 
mpage_folio_done(struct mpage_da_data * mpd,struct folio * folio)2047 static void mpage_folio_done(struct mpage_da_data *mpd, struct folio *folio)
2048 {
2049 	mpd->start_pos += folio_size(folio);
2050 	mpd->wbc->nr_to_write -= folio_nr_pages(folio);
2051 	folio_unlock(folio);
2052 }
2053 
mpage_submit_folio(struct mpage_da_data * mpd,struct folio * folio)2054 static int mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio)
2055 {
2056 	size_t len;
2057 	loff_t size;
2058 	int err;
2059 
2060 	WARN_ON_ONCE(folio_pos(folio) != mpd->start_pos);
2061 	folio_clear_dirty_for_io(folio);
2062 	/*
2063 	 * We have to be very careful here!  Nothing protects writeback path
2064 	 * against i_size changes and the page can be writeably mapped into
2065 	 * page tables. So an application can be growing i_size and writing
2066 	 * data through mmap while writeback runs. folio_clear_dirty_for_io()
2067 	 * write-protects our page in page tables and the page cannot get
2068 	 * written to again until we release folio lock. So only after
2069 	 * folio_clear_dirty_for_io() we are safe to sample i_size for
2070 	 * ext4_bio_write_folio() to zero-out tail of the written page. We rely
2071 	 * on the barrier provided by folio_test_clear_dirty() in
2072 	 * folio_clear_dirty_for_io() to make sure i_size is really sampled only
2073 	 * after page tables are updated.
2074 	 */
2075 	size = i_size_read(mpd->inode);
2076 	len = folio_size(folio);
2077 	if (folio_pos(folio) + len > size &&
2078 	    !ext4_verity_in_progress(mpd->inode))
2079 		len = size & (len - 1);
2080 	err = ext4_bio_write_folio(&mpd->io_submit, folio, len);
2081 
2082 	return err;
2083 }
2084 
2085 #define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay))
2086 
2087 /*
2088  * mballoc gives us at most this number of blocks...
2089  * XXX: That seems to be only a limitation of ext4_mb_normalize_request().
2090  * The rest of mballoc seems to handle chunks up to full group size.
2091  */
2092 #define MAX_WRITEPAGES_EXTENT_LEN 2048
2093 
2094 /*
2095  * mpage_add_bh_to_extent - try to add bh to extent of blocks to map
2096  *
2097  * @mpd - extent of blocks
2098  * @lblk - logical number of the block in the file
2099  * @bh - buffer head we want to add to the extent
2100  *
2101  * The function is used to collect contig. blocks in the same state. If the
2102  * buffer doesn't require mapping for writeback and we haven't started the
2103  * extent of buffers to map yet, the function returns 'true' immediately - the
2104  * caller can write the buffer right away. Otherwise the function returns true
2105  * if the block has been added to the extent, false if the block couldn't be
2106  * added.
2107  */
mpage_add_bh_to_extent(struct mpage_da_data * mpd,ext4_lblk_t lblk,struct buffer_head * bh)2108 static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
2109 				   struct buffer_head *bh)
2110 {
2111 	struct ext4_map_blocks *map = &mpd->map;
2112 
2113 	/* Buffer that doesn't need mapping for writeback? */
2114 	if (!buffer_dirty(bh) || !buffer_mapped(bh) ||
2115 	    (!buffer_delay(bh) && !buffer_unwritten(bh))) {
2116 		/* So far no extent to map => we write the buffer right away */
2117 		if (map->m_len == 0)
2118 			return true;
2119 		return false;
2120 	}
2121 
2122 	/* First block in the extent? */
2123 	if (map->m_len == 0) {
2124 		/* We cannot map unless handle is started... */
2125 		if (!mpd->do_map)
2126 			return false;
2127 		map->m_lblk = lblk;
2128 		map->m_len = 1;
2129 		map->m_flags = bh->b_state & BH_FLAGS;
2130 		return true;
2131 	}
2132 
2133 	/* Don't go larger than mballoc is willing to allocate */
2134 	if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)
2135 		return false;
2136 
2137 	/* Can we merge the block to our big extent? */
2138 	if (lblk == map->m_lblk + map->m_len &&
2139 	    (bh->b_state & BH_FLAGS) == map->m_flags) {
2140 		map->m_len++;
2141 		return true;
2142 	}
2143 	return false;
2144 }
2145 
2146 /*
2147  * mpage_process_page_bufs - submit page buffers for IO or add them to extent
2148  *
2149  * @mpd - extent of blocks for mapping
2150  * @head - the first buffer in the page
2151  * @bh - buffer we should start processing from
2152  * @lblk - logical number of the block in the file corresponding to @bh
2153  *
2154  * Walk through page buffers from @bh upto @head (exclusive) and either submit
2155  * the page for IO if all buffers in this page were mapped and there's no
2156  * accumulated extent of buffers to map or add buffers in the page to the
2157  * extent of buffers to map. The function returns 1 if the caller can continue
2158  * by processing the next page, 0 if it should stop adding buffers to the
2159  * extent to map because we cannot extend it anymore. It can also return value
2160  * < 0 in case of error during IO submission.
2161  */
mpage_process_page_bufs(struct mpage_da_data * mpd,struct buffer_head * head,struct buffer_head * bh,ext4_lblk_t lblk)2162 static int mpage_process_page_bufs(struct mpage_da_data *mpd,
2163 				   struct buffer_head *head,
2164 				   struct buffer_head *bh,
2165 				   ext4_lblk_t lblk)
2166 {
2167 	struct inode *inode = mpd->inode;
2168 	int err;
2169 	ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1)
2170 							>> inode->i_blkbits;
2171 
2172 	if (ext4_verity_in_progress(inode))
2173 		blocks = EXT_MAX_BLOCKS;
2174 
2175 	do {
2176 		BUG_ON(buffer_locked(bh));
2177 
2178 		if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
2179 			/* Found extent to map? */
2180 			if (mpd->map.m_len)
2181 				return 0;
2182 			/* Buffer needs mapping and handle is not started? */
2183 			if (!mpd->do_map)
2184 				return 0;
2185 			/* Everything mapped so far and we hit EOF */
2186 			break;
2187 		}
2188 	} while (lblk++, (bh = bh->b_this_page) != head);
2189 	/* So far everything mapped? Submit the page for IO. */
2190 	if (mpd->map.m_len == 0) {
2191 		err = mpage_submit_folio(mpd, head->b_folio);
2192 		if (err < 0)
2193 			return err;
2194 		mpage_folio_done(mpd, head->b_folio);
2195 	}
2196 	if (lblk >= blocks) {
2197 		mpd->scanned_until_end = 1;
2198 		return 0;
2199 	}
2200 	return 1;
2201 }
2202 
2203 /*
2204  * mpage_process_folio - update folio buffers corresponding to changed extent
2205  *			 and may submit fully mapped page for IO
2206  * @mpd: description of extent to map, on return next extent to map
2207  * @folio: Contains these buffers.
2208  * @m_lblk: logical block mapping.
2209  * @m_pblk: corresponding physical mapping.
2210  * @map_bh: determines on return whether this page requires any further
2211  *		  mapping or not.
2212  *
2213  * Scan given folio buffers corresponding to changed extent and update buffer
2214  * state according to new extent state.
2215  * We map delalloc buffers to their physical location, clear unwritten bits.
2216  * If the given folio is not fully mapped, we update @mpd to the next extent in
2217  * the given folio that needs mapping & return @map_bh as true.
2218  */
mpage_process_folio(struct mpage_da_data * mpd,struct folio * folio,ext4_lblk_t * m_lblk,ext4_fsblk_t * m_pblk,bool * map_bh)2219 static int mpage_process_folio(struct mpage_da_data *mpd, struct folio *folio,
2220 			      ext4_lblk_t *m_lblk, ext4_fsblk_t *m_pblk,
2221 			      bool *map_bh)
2222 {
2223 	struct buffer_head *head, *bh;
2224 	ext4_io_end_t *io_end = mpd->io_submit.io_end;
2225 	ext4_lblk_t lblk = *m_lblk;
2226 	ext4_fsblk_t pblock = *m_pblk;
2227 	int err = 0;
2228 	int blkbits = mpd->inode->i_blkbits;
2229 	ssize_t io_end_size = 0;
2230 	struct ext4_io_end_vec *io_end_vec = ext4_last_io_end_vec(io_end);
2231 
2232 	bh = head = folio_buffers(folio);
2233 	do {
2234 		if (lblk < mpd->map.m_lblk)
2235 			continue;
2236 		if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
2237 			/*
2238 			 * Buffer after end of mapped extent.
2239 			 * Find next buffer in the folio to map.
2240 			 */
2241 			mpd->map.m_len = 0;
2242 			mpd->map.m_flags = 0;
2243 			io_end_vec->size += io_end_size;
2244 
2245 			err = mpage_process_page_bufs(mpd, head, bh, lblk);
2246 			if (err > 0)
2247 				err = 0;
2248 			if (!err && mpd->map.m_len && mpd->map.m_lblk > lblk) {
2249 				io_end_vec = ext4_alloc_io_end_vec(io_end);
2250 				if (IS_ERR(io_end_vec)) {
2251 					err = PTR_ERR(io_end_vec);
2252 					goto out;
2253 				}
2254 				io_end_vec->offset = (loff_t)mpd->map.m_lblk << blkbits;
2255 			}
2256 			*map_bh = true;
2257 			goto out;
2258 		}
2259 		if (buffer_delay(bh)) {
2260 			clear_buffer_delay(bh);
2261 			bh->b_blocknr = pblock++;
2262 		}
2263 		clear_buffer_unwritten(bh);
2264 		io_end_size += (1 << blkbits);
2265 	} while (lblk++, (bh = bh->b_this_page) != head);
2266 
2267 	io_end_vec->size += io_end_size;
2268 	*map_bh = false;
2269 out:
2270 	*m_lblk = lblk;
2271 	*m_pblk = pblock;
2272 	return err;
2273 }
2274 
2275 /*
2276  * mpage_map_buffers - update buffers corresponding to changed extent and
2277  *		       submit fully mapped pages for IO
2278  *
2279  * @mpd - description of extent to map, on return next extent to map
2280  *
2281  * Scan buffers corresponding to changed extent (we expect corresponding pages
2282  * to be already locked) and update buffer state according to new extent state.
2283  * We map delalloc buffers to their physical location, clear unwritten bits,
2284  * and mark buffers as uninit when we perform writes to unwritten extents
2285  * and do extent conversion after IO is finished. If the last page is not fully
2286  * mapped, we update @map to the next extent in the last page that needs
2287  * mapping. Otherwise we submit the page for IO.
2288  */
mpage_map_and_submit_buffers(struct mpage_da_data * mpd)2289 static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
2290 {
2291 	struct folio_batch fbatch;
2292 	unsigned nr, i;
2293 	struct inode *inode = mpd->inode;
2294 	int bpp_bits = PAGE_SHIFT - inode->i_blkbits;
2295 	pgoff_t start, end;
2296 	ext4_lblk_t lblk;
2297 	ext4_fsblk_t pblock;
2298 	int err;
2299 	bool map_bh = false;
2300 
2301 	start = mpd->map.m_lblk >> bpp_bits;
2302 	end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits;
2303 	pblock = mpd->map.m_pblk;
2304 
2305 	folio_batch_init(&fbatch);
2306 	while (start <= end) {
2307 		nr = filemap_get_folios(inode->i_mapping, &start, end, &fbatch);
2308 		if (nr == 0)
2309 			break;
2310 		for (i = 0; i < nr; i++) {
2311 			struct folio *folio = fbatch.folios[i];
2312 
2313 			lblk = folio->index << bpp_bits;
2314 			err = mpage_process_folio(mpd, folio, &lblk, &pblock,
2315 						 &map_bh);
2316 			/*
2317 			 * If map_bh is true, means page may require further bh
2318 			 * mapping, or maybe the page was submitted for IO.
2319 			 * So we return to call further extent mapping.
2320 			 */
2321 			if (err < 0 || map_bh)
2322 				goto out;
2323 			/* Page fully mapped - let IO run! */
2324 			err = mpage_submit_folio(mpd, folio);
2325 			if (err < 0)
2326 				goto out;
2327 			mpage_folio_done(mpd, folio);
2328 		}
2329 		folio_batch_release(&fbatch);
2330 	}
2331 	/* Extent fully mapped and matches with page boundary. We are done. */
2332 	mpd->map.m_len = 0;
2333 	mpd->map.m_flags = 0;
2334 	return 0;
2335 out:
2336 	folio_batch_release(&fbatch);
2337 	return err;
2338 }
2339 
mpage_map_one_extent(handle_t * handle,struct mpage_da_data * mpd)2340 static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
2341 {
2342 	struct inode *inode = mpd->inode;
2343 	struct ext4_map_blocks *map = &mpd->map;
2344 	int get_blocks_flags;
2345 	int err, dioread_nolock;
2346 
2347 	/* Make sure transaction has enough credits for this extent */
2348 	err = ext4_journal_ensure_extent_credits(handle, inode);
2349 	if (err < 0)
2350 		return err;
2351 
2352 	trace_ext4_da_write_pages_extent(inode, map);
2353 	/*
2354 	 * Call ext4_map_blocks() to allocate any delayed allocation blocks, or
2355 	 * to convert an unwritten extent to be initialized (in the case
2356 	 * where we have written into one or more preallocated blocks).  It is
2357 	 * possible that we're going to need more metadata blocks than
2358 	 * previously reserved. However we must not fail because we're in
2359 	 * writeback and there is nothing we can do about it so it might result
2360 	 * in data loss.  So use reserved blocks to allocate metadata if
2361 	 * possible. In addition, do not cache any unrelated extents, as it
2362 	 * only holds the folio lock but does not hold the i_rwsem or
2363 	 * invalidate_lock, which could corrupt the extent status tree.
2364 	 */
2365 	get_blocks_flags = EXT4_GET_BLOCKS_CREATE |
2366 			   EXT4_GET_BLOCKS_METADATA_NOFAIL |
2367 			   EXT4_GET_BLOCKS_IO_SUBMIT |
2368 			   EXT4_EX_NOCACHE;
2369 
2370 	dioread_nolock = ext4_should_dioread_nolock(inode);
2371 	if (dioread_nolock)
2372 		get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;
2373 
2374 	err = ext4_map_blocks(handle, inode, map, get_blocks_flags);
2375 	if (err < 0)
2376 		return err;
2377 	if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {
2378 		if (!mpd->io_submit.io_end->handle &&
2379 		    ext4_handle_valid(handle)) {
2380 			mpd->io_submit.io_end->handle = handle->h_rsv_handle;
2381 			handle->h_rsv_handle = NULL;
2382 		}
2383 		ext4_set_io_unwritten_flag(mpd->io_submit.io_end);
2384 	}
2385 
2386 	BUG_ON(map->m_len == 0);
2387 	return 0;
2388 }
2389 
2390 /*
2391  * This is used to submit mapped buffers in a single folio that is not fully
2392  * mapped for various reasons, such as insufficient space or journal credits.
2393  */
mpage_submit_partial_folio(struct mpage_da_data * mpd)2394 static int mpage_submit_partial_folio(struct mpage_da_data *mpd)
2395 {
2396 	struct inode *inode = mpd->inode;
2397 	struct folio *folio;
2398 	loff_t pos;
2399 	int ret;
2400 
2401 	folio = filemap_get_folio(inode->i_mapping,
2402 				  mpd->start_pos >> PAGE_SHIFT);
2403 	if (IS_ERR(folio))
2404 		return PTR_ERR(folio);
2405 	/*
2406 	 * The mapped position should be within the current processing folio
2407 	 * but must not be the folio start position.
2408 	 */
2409 	pos = ((loff_t)mpd->map.m_lblk) << inode->i_blkbits;
2410 	if (WARN_ON_ONCE((folio_pos(folio) == pos) ||
2411 			 !folio_contains(folio, pos >> PAGE_SHIFT)))
2412 		return -EINVAL;
2413 
2414 	ret = mpage_submit_folio(mpd, folio);
2415 	if (ret)
2416 		goto out;
2417 	/*
2418 	 * Update start_pos to prevent this folio from being released in
2419 	 * mpage_release_unused_pages(), it will be reset to the aligned folio
2420 	 * pos when this folio is written again in the next round. Additionally,
2421 	 * do not update wbc->nr_to_write here, as it will be updated once the
2422 	 * entire folio has finished processing.
2423 	 */
2424 	mpd->start_pos = pos;
2425 out:
2426 	folio_unlock(folio);
2427 	folio_put(folio);
2428 	return ret;
2429 }
2430 
2431 /*
2432  * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length
2433  *				 mpd->len and submit pages underlying it for IO
2434  *
2435  * @handle - handle for journal operations
2436  * @mpd - extent to map
2437  * @give_up_on_write - we set this to true iff there is a fatal error and there
2438  *                     is no hope of writing the data. The caller should discard
2439  *                     dirty pages to avoid infinite loops.
2440  *
2441  * The function maps extent starting at mpd->lblk of length mpd->len. If it is
2442  * delayed, blocks are allocated, if it is unwritten, we may need to convert
2443  * them to initialized or split the described range from larger unwritten
2444  * extent. Note that we need not map all the described range since allocation
2445  * can return less blocks or the range is covered by more unwritten extents. We
2446  * cannot map more because we are limited by reserved transaction credits. On
2447  * the other hand we always make sure that the last touched page is fully
2448  * mapped so that it can be written out (and thus forward progress is
2449  * guaranteed). After mapping we submit all mapped pages for IO.
2450  */
mpage_map_and_submit_extent(handle_t * handle,struct mpage_da_data * mpd,bool * give_up_on_write)2451 static int mpage_map_and_submit_extent(handle_t *handle,
2452 				       struct mpage_da_data *mpd,
2453 				       bool *give_up_on_write)
2454 {
2455 	struct inode *inode = mpd->inode;
2456 	struct ext4_map_blocks *map = &mpd->map;
2457 	int err;
2458 	loff_t disksize;
2459 	int progress = 0;
2460 	ext4_io_end_t *io_end = mpd->io_submit.io_end;
2461 	struct ext4_io_end_vec *io_end_vec;
2462 
2463 	io_end_vec = ext4_alloc_io_end_vec(io_end);
2464 	if (IS_ERR(io_end_vec))
2465 		return PTR_ERR(io_end_vec);
2466 	io_end_vec->offset = ((loff_t)map->m_lblk) << inode->i_blkbits;
2467 	do {
2468 		err = mpage_map_one_extent(handle, mpd);
2469 		if (err < 0) {
2470 			struct super_block *sb = inode->i_sb;
2471 
2472 			if (ext4_emergency_state(sb))
2473 				goto invalidate_dirty_pages;
2474 			/*
2475 			 * Let the uper layers retry transient errors.
2476 			 * In the case of ENOSPC, if ext4_count_free_blocks()
2477 			 * is non-zero, a commit should free up blocks.
2478 			 */
2479 			if ((err == -ENOMEM) || (err == -EAGAIN) ||
2480 			    (err == -ENOSPC && ext4_count_free_clusters(sb))) {
2481 				/*
2482 				 * We may have already allocated extents for
2483 				 * some bhs inside the folio, issue the
2484 				 * corresponding data to prevent stale data.
2485 				 */
2486 				if (progress) {
2487 					if (mpage_submit_partial_folio(mpd))
2488 						goto invalidate_dirty_pages;
2489 					goto update_disksize;
2490 				}
2491 				return err;
2492 			}
2493 			ext4_msg(sb, KERN_CRIT,
2494 				 "Delayed block allocation failed for "
2495 				 "inode %lu at logical offset %llu with"
2496 				 " max blocks %u with error %d",
2497 				 inode->i_ino,
2498 				 (unsigned long long)map->m_lblk,
2499 				 (unsigned)map->m_len, -err);
2500 			ext4_msg(sb, KERN_CRIT,
2501 				 "This should not happen!! Data will "
2502 				 "be lost\n");
2503 			if (err == -ENOSPC)
2504 				ext4_print_free_blocks(inode);
2505 		invalidate_dirty_pages:
2506 			*give_up_on_write = true;
2507 			return err;
2508 		}
2509 		progress = 1;
2510 		/*
2511 		 * Update buffer state, submit mapped pages, and get us new
2512 		 * extent to map
2513 		 */
2514 		err = mpage_map_and_submit_buffers(mpd);
2515 		if (err < 0)
2516 			goto update_disksize;
2517 	} while (map->m_len);
2518 
2519 update_disksize:
2520 	/*
2521 	 * Update on-disk size after IO is submitted.  Races with
2522 	 * truncate are avoided by checking i_size under i_data_sem.
2523 	 */
2524 	disksize = mpd->start_pos;
2525 	if (disksize > READ_ONCE(EXT4_I(inode)->i_disksize)) {
2526 		int err2;
2527 		loff_t i_size;
2528 
2529 		down_write(&EXT4_I(inode)->i_data_sem);
2530 		i_size = i_size_read(inode);
2531 		if (disksize > i_size)
2532 			disksize = i_size;
2533 		if (disksize > EXT4_I(inode)->i_disksize)
2534 			EXT4_I(inode)->i_disksize = disksize;
2535 		up_write(&EXT4_I(inode)->i_data_sem);
2536 		err2 = ext4_mark_inode_dirty(handle, inode);
2537 		if (err2) {
2538 			ext4_error_err(inode->i_sb, -err2,
2539 				       "Failed to mark inode %lu dirty",
2540 				       inode->i_ino);
2541 		}
2542 		if (!err)
2543 			err = err2;
2544 	}
2545 	return err;
2546 }
2547 
ext4_journal_folio_buffers(handle_t * handle,struct folio * folio,size_t len)2548 static int ext4_journal_folio_buffers(handle_t *handle, struct folio *folio,
2549 				     size_t len)
2550 {
2551 	struct buffer_head *page_bufs = folio_buffers(folio);
2552 	struct inode *inode = folio->mapping->host;
2553 	int ret, err;
2554 
2555 	ret = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
2556 				     NULL, do_journal_get_write_access);
2557 	err = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
2558 				     NULL, write_end_fn);
2559 	if (ret == 0)
2560 		ret = err;
2561 	err = ext4_jbd2_inode_add_write(handle, inode, folio_pos(folio), len);
2562 	if (ret == 0)
2563 		ret = err;
2564 	EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
2565 
2566 	return ret;
2567 }
2568 
mpage_journal_page_buffers(handle_t * handle,struct mpage_da_data * mpd,struct folio * folio)2569 static int mpage_journal_page_buffers(handle_t *handle,
2570 				      struct mpage_da_data *mpd,
2571 				      struct folio *folio)
2572 {
2573 	struct inode *inode = mpd->inode;
2574 	loff_t size = i_size_read(inode);
2575 	size_t len = folio_size(folio);
2576 
2577 	folio_clear_checked(folio);
2578 	mpd->wbc->nr_to_write -= folio_nr_pages(folio);
2579 
2580 	if (folio_pos(folio) + len > size &&
2581 	    !ext4_verity_in_progress(inode))
2582 		len = size & (len - 1);
2583 
2584 	return ext4_journal_folio_buffers(handle, folio, len);
2585 }
2586 
2587 /*
2588  * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages
2589  * 				 needing mapping, submit mapped pages
2590  *
2591  * @mpd - where to look for pages
2592  *
2593  * Walk dirty pages in the mapping. If they are fully mapped, submit them for
2594  * IO immediately. If we cannot map blocks, we submit just already mapped
2595  * buffers in the page for IO and keep page dirty. When we can map blocks and
2596  * we find a page which isn't mapped we start accumulating extent of buffers
2597  * underlying these pages that needs mapping (formed by either delayed or
2598  * unwritten buffers). We also lock the pages containing these buffers. The
2599  * extent found is returned in @mpd structure (starting at mpd->lblk with
2600  * length mpd->len blocks).
2601  *
2602  * Note that this function can attach bios to one io_end structure which are
2603  * neither logically nor physically contiguous. Although it may seem as an
2604  * unnecessary complication, it is actually inevitable in blocksize < pagesize
2605  * case as we need to track IO to all buffers underlying a page in one io_end.
2606  */
mpage_prepare_extent_to_map(struct mpage_da_data * mpd)2607 static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
2608 {
2609 	struct address_space *mapping = mpd->inode->i_mapping;
2610 	struct folio_batch fbatch;
2611 	unsigned int nr_folios;
2612 	pgoff_t index = mpd->start_pos >> PAGE_SHIFT;
2613 	pgoff_t end = mpd->end_pos >> PAGE_SHIFT;
2614 	xa_mark_t tag;
2615 	int i, err = 0;
2616 	int blkbits = mpd->inode->i_blkbits;
2617 	ext4_lblk_t lblk;
2618 	struct buffer_head *head;
2619 	handle_t *handle = NULL;
2620 	int bpp = ext4_journal_blocks_per_folio(mpd->inode);
2621 
2622 	if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages)
2623 		tag = PAGECACHE_TAG_TOWRITE;
2624 	else
2625 		tag = PAGECACHE_TAG_DIRTY;
2626 
2627 	mpd->map.m_len = 0;
2628 	mpd->next_pos = mpd->start_pos;
2629 	if (ext4_should_journal_data(mpd->inode)) {
2630 		handle = ext4_journal_start(mpd->inode, EXT4_HT_WRITE_PAGE,
2631 					    bpp);
2632 		if (IS_ERR(handle))
2633 			return PTR_ERR(handle);
2634 	}
2635 	folio_batch_init(&fbatch);
2636 	while (index <= end) {
2637 		nr_folios = filemap_get_folios_tag(mapping, &index, end,
2638 				tag, &fbatch);
2639 		if (nr_folios == 0)
2640 			break;
2641 
2642 		for (i = 0; i < nr_folios; i++) {
2643 			struct folio *folio = fbatch.folios[i];
2644 
2645 			/*
2646 			 * Accumulated enough dirty pages? This doesn't apply
2647 			 * to WB_SYNC_ALL mode. For integrity sync we have to
2648 			 * keep going because someone may be concurrently
2649 			 * dirtying pages, and we might have synced a lot of
2650 			 * newly appeared dirty pages, but have not synced all
2651 			 * of the old dirty pages.
2652 			 */
2653 			if (mpd->wbc->sync_mode == WB_SYNC_NONE &&
2654 			    mpd->wbc->nr_to_write <=
2655 			    mpd->map.m_len >> (PAGE_SHIFT - blkbits))
2656 				goto out;
2657 
2658 			/* If we can't merge this page, we are done. */
2659 			if (mpd->map.m_len > 0 &&
2660 			    mpd->next_pos != folio_pos(folio))
2661 				goto out;
2662 
2663 			if (handle) {
2664 				err = ext4_journal_ensure_credits(handle, bpp,
2665 								  0);
2666 				if (err < 0)
2667 					goto out;
2668 			}
2669 
2670 			folio_lock(folio);
2671 			/*
2672 			 * If the page is no longer dirty, or its mapping no
2673 			 * longer corresponds to inode we are writing (which
2674 			 * means it has been truncated or invalidated), or the
2675 			 * page is already under writeback and we are not doing
2676 			 * a data integrity writeback, skip the page
2677 			 */
2678 			if (!folio_test_dirty(folio) ||
2679 			    (folio_test_writeback(folio) &&
2680 			     (mpd->wbc->sync_mode == WB_SYNC_NONE)) ||
2681 			    unlikely(folio->mapping != mapping)) {
2682 				folio_unlock(folio);
2683 				continue;
2684 			}
2685 
2686 			folio_wait_writeback(folio);
2687 			BUG_ON(folio_test_writeback(folio));
2688 
2689 			/*
2690 			 * Should never happen but for buggy code in
2691 			 * other subsystems that call
2692 			 * set_page_dirty() without properly warning
2693 			 * the file system first.  See [1] for more
2694 			 * information.
2695 			 *
2696 			 * [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz
2697 			 */
2698 			if (!folio_buffers(folio)) {
2699 				ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", folio->index);
2700 				folio_clear_dirty(folio);
2701 				folio_unlock(folio);
2702 				continue;
2703 			}
2704 
2705 			if (mpd->map.m_len == 0)
2706 				mpd->start_pos = folio_pos(folio);
2707 			mpd->next_pos = folio_pos(folio) + folio_size(folio);
2708 			/*
2709 			 * Writeout when we cannot modify metadata is simple.
2710 			 * Just submit the page. For data=journal mode we
2711 			 * first handle writeout of the page for checkpoint and
2712 			 * only after that handle delayed page dirtying. This
2713 			 * makes sure current data is checkpointed to the final
2714 			 * location before possibly journalling it again which
2715 			 * is desirable when the page is frequently dirtied
2716 			 * through a pin.
2717 			 */
2718 			if (!mpd->can_map) {
2719 				err = mpage_submit_folio(mpd, folio);
2720 				if (err < 0)
2721 					goto out;
2722 				/* Pending dirtying of journalled data? */
2723 				if (folio_test_checked(folio)) {
2724 					err = mpage_journal_page_buffers(handle,
2725 						mpd, folio);
2726 					if (err < 0)
2727 						goto out;
2728 					mpd->journalled_more_data = 1;
2729 				}
2730 				mpage_folio_done(mpd, folio);
2731 			} else {
2732 				/* Add all dirty buffers to mpd */
2733 				lblk = ((ext4_lblk_t)folio->index) <<
2734 					(PAGE_SHIFT - blkbits);
2735 				head = folio_buffers(folio);
2736 				err = mpage_process_page_bufs(mpd, head, head,
2737 						lblk);
2738 				if (err <= 0)
2739 					goto out;
2740 				err = 0;
2741 			}
2742 		}
2743 		folio_batch_release(&fbatch);
2744 		cond_resched();
2745 	}
2746 	mpd->scanned_until_end = 1;
2747 	if (handle)
2748 		ext4_journal_stop(handle);
2749 	return 0;
2750 out:
2751 	folio_batch_release(&fbatch);
2752 	if (handle)
2753 		ext4_journal_stop(handle);
2754 	return err;
2755 }
2756 
ext4_do_writepages(struct mpage_da_data * mpd)2757 static int ext4_do_writepages(struct mpage_da_data *mpd)
2758 {
2759 	struct writeback_control *wbc = mpd->wbc;
2760 	pgoff_t	writeback_index = 0;
2761 	long nr_to_write = wbc->nr_to_write;
2762 	int range_whole = 0;
2763 	int cycled = 1;
2764 	handle_t *handle = NULL;
2765 	struct inode *inode = mpd->inode;
2766 	struct address_space *mapping = inode->i_mapping;
2767 	int needed_blocks, rsv_blocks = 0, ret = 0;
2768 	struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2769 	struct blk_plug plug;
2770 	bool give_up_on_write = false;
2771 
2772 	trace_ext4_writepages(inode, wbc);
2773 
2774 	/*
2775 	 * No pages to write? This is mainly a kludge to avoid starting
2776 	 * a transaction for special inodes like journal inode on last iput()
2777 	 * because that could violate lock ordering on umount
2778 	 */
2779 	if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2780 		goto out_writepages;
2781 
2782 	/*
2783 	 * If the filesystem has aborted, it is read-only, so return
2784 	 * right away instead of dumping stack traces later on that
2785 	 * will obscure the real source of the problem.  We test
2786 	 * fs shutdown state instead of sb->s_flag's SB_RDONLY because
2787 	 * the latter could be true if the filesystem is mounted
2788 	 * read-only, and in that case, ext4_writepages should
2789 	 * *never* be called, so if that ever happens, we would want
2790 	 * the stack trace.
2791 	 */
2792 	ret = ext4_emergency_state(mapping->host->i_sb);
2793 	if (unlikely(ret))
2794 		goto out_writepages;
2795 
2796 	/*
2797 	 * If we have inline data and arrive here, it means that
2798 	 * we will soon create the block for the 1st page, so
2799 	 * we'd better clear the inline data here.
2800 	 */
2801 	if (ext4_has_inline_data(inode)) {
2802 		/* Just inode will be modified... */
2803 		handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
2804 		if (IS_ERR(handle)) {
2805 			ret = PTR_ERR(handle);
2806 			goto out_writepages;
2807 		}
2808 		BUG_ON(ext4_test_inode_state(inode,
2809 				EXT4_STATE_MAY_INLINE_DATA));
2810 		ext4_destroy_inline_data(handle, inode);
2811 		ext4_journal_stop(handle);
2812 	}
2813 
2814 	/*
2815 	 * data=journal mode does not do delalloc so we just need to writeout /
2816 	 * journal already mapped buffers. On the other hand we need to commit
2817 	 * transaction to make data stable. We expect all the data to be
2818 	 * already in the journal (the only exception are DMA pinned pages
2819 	 * dirtied behind our back) so we commit transaction here and run the
2820 	 * writeback loop to checkpoint them. The checkpointing is not actually
2821 	 * necessary to make data persistent *but* quite a few places (extent
2822 	 * shifting operations, fsverity, ...) depend on being able to drop
2823 	 * pagecache pages after calling filemap_write_and_wait() and for that
2824 	 * checkpointing needs to happen.
2825 	 */
2826 	if (ext4_should_journal_data(inode)) {
2827 		mpd->can_map = 0;
2828 		if (wbc->sync_mode == WB_SYNC_ALL)
2829 			ext4_fc_commit(sbi->s_journal,
2830 				       EXT4_I(inode)->i_datasync_tid);
2831 	}
2832 	mpd->journalled_more_data = 0;
2833 
2834 	if (ext4_should_dioread_nolock(inode)) {
2835 		int bpf = ext4_journal_blocks_per_folio(inode);
2836 		/*
2837 		 * We may need to convert up to one extent per block in
2838 		 * the folio and we may dirty the inode.
2839 		 */
2840 		rsv_blocks = 1 + ext4_ext_index_trans_blocks(inode, bpf);
2841 	}
2842 
2843 	if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2844 		range_whole = 1;
2845 
2846 	if (wbc->range_cyclic) {
2847 		writeback_index = mapping->writeback_index;
2848 		if (writeback_index)
2849 			cycled = 0;
2850 		mpd->start_pos = writeback_index << PAGE_SHIFT;
2851 		mpd->end_pos = LLONG_MAX;
2852 	} else {
2853 		mpd->start_pos = wbc->range_start;
2854 		mpd->end_pos = wbc->range_end;
2855 	}
2856 
2857 	ext4_io_submit_init(&mpd->io_submit, wbc);
2858 retry:
2859 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2860 		tag_pages_for_writeback(mapping, mpd->start_pos >> PAGE_SHIFT,
2861 					mpd->end_pos >> PAGE_SHIFT);
2862 	blk_start_plug(&plug);
2863 
2864 	/*
2865 	 * First writeback pages that don't need mapping - we can avoid
2866 	 * starting a transaction unnecessarily and also avoid being blocked
2867 	 * in the block layer on device congestion while having transaction
2868 	 * started.
2869 	 */
2870 	mpd->do_map = 0;
2871 	mpd->scanned_until_end = 0;
2872 	mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2873 	if (!mpd->io_submit.io_end) {
2874 		ret = -ENOMEM;
2875 		goto unplug;
2876 	}
2877 	ret = mpage_prepare_extent_to_map(mpd);
2878 	/* Unlock pages we didn't use */
2879 	mpage_release_unused_pages(mpd, false);
2880 	/* Submit prepared bio */
2881 	ext4_io_submit(&mpd->io_submit);
2882 	ext4_put_io_end_defer(mpd->io_submit.io_end);
2883 	mpd->io_submit.io_end = NULL;
2884 	if (ret < 0)
2885 		goto unplug;
2886 
2887 	while (!mpd->scanned_until_end && wbc->nr_to_write > 0) {
2888 		/* For each extent of pages we use new io_end */
2889 		mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2890 		if (!mpd->io_submit.io_end) {
2891 			ret = -ENOMEM;
2892 			break;
2893 		}
2894 
2895 		WARN_ON_ONCE(!mpd->can_map);
2896 		/*
2897 		 * We have two constraints: We find one extent to map and we
2898 		 * must always write out whole page (makes a difference when
2899 		 * blocksize < pagesize) so that we don't block on IO when we
2900 		 * try to write out the rest of the page. Journalled mode is
2901 		 * not supported by delalloc.
2902 		 */
2903 		BUG_ON(ext4_should_journal_data(inode));
2904 		/*
2905 		 * Calculate the number of credits needed to reserve for one
2906 		 * extent of up to MAX_WRITEPAGES_EXTENT_LEN blocks. It will
2907 		 * attempt to extend the transaction or start a new iteration
2908 		 * if the reserved credits are insufficient.
2909 		 */
2910 		needed_blocks = ext4_chunk_trans_blocks(inode,
2911 						MAX_WRITEPAGES_EXTENT_LEN);
2912 		/* start a new transaction */
2913 		handle = ext4_journal_start_with_reserve(inode,
2914 				EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);
2915 		if (IS_ERR(handle)) {
2916 			ret = PTR_ERR(handle);
2917 			ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
2918 			       "%ld pages, ino %lu; err %d", __func__,
2919 				wbc->nr_to_write, inode->i_ino, ret);
2920 			/* Release allocated io_end */
2921 			ext4_put_io_end(mpd->io_submit.io_end);
2922 			mpd->io_submit.io_end = NULL;
2923 			break;
2924 		}
2925 		mpd->do_map = 1;
2926 
2927 		trace_ext4_da_write_folios_start(inode, mpd->start_pos,
2928 				mpd->next_pos, wbc);
2929 		ret = mpage_prepare_extent_to_map(mpd);
2930 		if (!ret && mpd->map.m_len)
2931 			ret = mpage_map_and_submit_extent(handle, mpd,
2932 					&give_up_on_write);
2933 		/*
2934 		 * Caution: If the handle is synchronous,
2935 		 * ext4_journal_stop() can wait for transaction commit
2936 		 * to finish which may depend on writeback of pages to
2937 		 * complete or on page lock to be released.  In that
2938 		 * case, we have to wait until after we have
2939 		 * submitted all the IO, released page locks we hold,
2940 		 * and dropped io_end reference (for extent conversion
2941 		 * to be able to complete) before stopping the handle.
2942 		 */
2943 		if (!ext4_handle_valid(handle) || handle->h_sync == 0) {
2944 			ext4_journal_stop(handle);
2945 			handle = NULL;
2946 			mpd->do_map = 0;
2947 		}
2948 		/* Unlock pages we didn't use */
2949 		mpage_release_unused_pages(mpd, give_up_on_write);
2950 		/* Submit prepared bio */
2951 		ext4_io_submit(&mpd->io_submit);
2952 
2953 		/*
2954 		 * Drop our io_end reference we got from init. We have
2955 		 * to be careful and use deferred io_end finishing if
2956 		 * we are still holding the transaction as we can
2957 		 * release the last reference to io_end which may end
2958 		 * up doing unwritten extent conversion.
2959 		 */
2960 		if (handle) {
2961 			ext4_put_io_end_defer(mpd->io_submit.io_end);
2962 			ext4_journal_stop(handle);
2963 		} else
2964 			ext4_put_io_end(mpd->io_submit.io_end);
2965 		mpd->io_submit.io_end = NULL;
2966 		trace_ext4_da_write_folios_end(inode, mpd->start_pos,
2967 				mpd->next_pos, wbc, ret);
2968 
2969 		if (ret == -ENOSPC && sbi->s_journal) {
2970 			/*
2971 			 * Commit the transaction which would
2972 			 * free blocks released in the transaction
2973 			 * and try again
2974 			 */
2975 			jbd2_journal_force_commit_nested(sbi->s_journal);
2976 			ret = 0;
2977 			continue;
2978 		}
2979 		if (ret == -EAGAIN)
2980 			ret = 0;
2981 		/* Fatal error - ENOMEM, EIO... */
2982 		if (ret)
2983 			break;
2984 	}
2985 unplug:
2986 	blk_finish_plug(&plug);
2987 	if (!ret && !cycled && wbc->nr_to_write > 0) {
2988 		cycled = 1;
2989 		mpd->end_pos = (writeback_index << PAGE_SHIFT) - 1;
2990 		mpd->start_pos = 0;
2991 		goto retry;
2992 	}
2993 
2994 	/* Update index */
2995 	if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2996 		/*
2997 		 * Set the writeback_index so that range_cyclic
2998 		 * mode will write it back later
2999 		 */
3000 		mapping->writeback_index = mpd->start_pos >> PAGE_SHIFT;
3001 
3002 out_writepages:
3003 	trace_ext4_writepages_result(inode, wbc, ret,
3004 				     nr_to_write - wbc->nr_to_write);
3005 	return ret;
3006 }
3007 
ext4_writepages(struct address_space * mapping,struct writeback_control * wbc)3008 static int ext4_writepages(struct address_space *mapping,
3009 			   struct writeback_control *wbc)
3010 {
3011 	struct super_block *sb = mapping->host->i_sb;
3012 	struct mpage_da_data mpd = {
3013 		.inode = mapping->host,
3014 		.wbc = wbc,
3015 		.can_map = 1,
3016 	};
3017 	int ret;
3018 	int alloc_ctx;
3019 
3020 	ret = ext4_emergency_state(sb);
3021 	if (unlikely(ret))
3022 		return ret;
3023 
3024 	alloc_ctx = ext4_writepages_down_read(sb);
3025 	ret = ext4_do_writepages(&mpd);
3026 	/*
3027 	 * For data=journal writeback we could have come across pages marked
3028 	 * for delayed dirtying (PageChecked) which were just added to the
3029 	 * running transaction. Try once more to get them to stable storage.
3030 	 */
3031 	if (!ret && mpd.journalled_more_data)
3032 		ret = ext4_do_writepages(&mpd);
3033 	ext4_writepages_up_read(sb, alloc_ctx);
3034 
3035 	return ret;
3036 }
3037 
ext4_normal_submit_inode_data_buffers(struct jbd2_inode * jinode)3038 int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode)
3039 {
3040 	struct writeback_control wbc = {
3041 		.sync_mode = WB_SYNC_ALL,
3042 		.nr_to_write = LONG_MAX,
3043 		.range_start = jinode->i_dirty_start,
3044 		.range_end = jinode->i_dirty_end,
3045 	};
3046 	struct mpage_da_data mpd = {
3047 		.inode = jinode->i_vfs_inode,
3048 		.wbc = &wbc,
3049 		.can_map = 0,
3050 	};
3051 	return ext4_do_writepages(&mpd);
3052 }
3053 
ext4_dax_writepages(struct address_space * mapping,struct writeback_control * wbc)3054 static int ext4_dax_writepages(struct address_space *mapping,
3055 			       struct writeback_control *wbc)
3056 {
3057 	int ret;
3058 	long nr_to_write = wbc->nr_to_write;
3059 	struct inode *inode = mapping->host;
3060 	int alloc_ctx;
3061 
3062 	ret = ext4_emergency_state(inode->i_sb);
3063 	if (unlikely(ret))
3064 		return ret;
3065 
3066 	alloc_ctx = ext4_writepages_down_read(inode->i_sb);
3067 	trace_ext4_writepages(inode, wbc);
3068 
3069 	ret = dax_writeback_mapping_range(mapping,
3070 					  EXT4_SB(inode->i_sb)->s_daxdev, wbc);
3071 	trace_ext4_writepages_result(inode, wbc, ret,
3072 				     nr_to_write - wbc->nr_to_write);
3073 	ext4_writepages_up_read(inode->i_sb, alloc_ctx);
3074 	return ret;
3075 }
3076 
ext4_nonda_switch(struct super_block * sb)3077 static int ext4_nonda_switch(struct super_block *sb)
3078 {
3079 	s64 free_clusters, dirty_clusters;
3080 	struct ext4_sb_info *sbi = EXT4_SB(sb);
3081 
3082 	/*
3083 	 * switch to non delalloc mode if we are running low
3084 	 * on free block. The free block accounting via percpu
3085 	 * counters can get slightly wrong with percpu_counter_batch getting
3086 	 * accumulated on each CPU without updating global counters
3087 	 * Delalloc need an accurate free block accounting. So switch
3088 	 * to non delalloc when we are near to error range.
3089 	 */
3090 	free_clusters =
3091 		percpu_counter_read_positive(&sbi->s_freeclusters_counter);
3092 	dirty_clusters =
3093 		percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);
3094 	/*
3095 	 * Start pushing delalloc when 1/2 of free blocks are dirty.
3096 	 */
3097 	if (dirty_clusters && (free_clusters < 2 * dirty_clusters))
3098 		try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);
3099 
3100 	if (2 * free_clusters < 3 * dirty_clusters ||
3101 	    free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {
3102 		/*
3103 		 * free block count is less than 150% of dirty blocks
3104 		 * or free blocks is less than watermark
3105 		 */
3106 		return 1;
3107 	}
3108 	return 0;
3109 }
3110 
ext4_da_write_begin(const struct kiocb * iocb,struct address_space * mapping,loff_t pos,unsigned len,struct folio ** foliop,void ** fsdata)3111 static int ext4_da_write_begin(const struct kiocb *iocb,
3112 			       struct address_space *mapping,
3113 			       loff_t pos, unsigned len,
3114 			       struct folio **foliop, void **fsdata)
3115 {
3116 	int ret, retries = 0;
3117 	struct folio *folio;
3118 	pgoff_t index;
3119 	struct inode *inode = mapping->host;
3120 
3121 	ret = ext4_emergency_state(inode->i_sb);
3122 	if (unlikely(ret))
3123 		return ret;
3124 
3125 	index = pos >> PAGE_SHIFT;
3126 
3127 	if (ext4_nonda_switch(inode->i_sb) || ext4_verity_in_progress(inode)) {
3128 		*fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
3129 		return ext4_write_begin(iocb, mapping, pos,
3130 					len, foliop, fsdata);
3131 	}
3132 	*fsdata = (void *)0;
3133 	trace_ext4_da_write_begin(inode, pos, len);
3134 
3135 	if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
3136 		ret = ext4_generic_write_inline_data(mapping, inode, pos, len,
3137 						     foliop, fsdata, true);
3138 		if (ret < 0)
3139 			return ret;
3140 		if (ret == 1)
3141 			return 0;
3142 	}
3143 
3144 retry:
3145 	folio = write_begin_get_folio(iocb, mapping, index, len);
3146 	if (IS_ERR(folio))
3147 		return PTR_ERR(folio);
3148 
3149 	if (pos + len > folio_pos(folio) + folio_size(folio))
3150 		len = folio_pos(folio) + folio_size(folio) - pos;
3151 
3152 	ret = ext4_block_write_begin(NULL, folio, pos, len,
3153 				     ext4_da_get_block_prep);
3154 	if (ret < 0) {
3155 		folio_unlock(folio);
3156 		folio_put(folio);
3157 		/*
3158 		 * block_write_begin may have instantiated a few blocks
3159 		 * outside i_size.  Trim these off again. Don't need
3160 		 * i_size_read because we hold inode lock.
3161 		 */
3162 		if (pos + len > inode->i_size)
3163 			ext4_truncate_failed_write(inode);
3164 
3165 		if (ret == -ENOSPC &&
3166 		    ext4_should_retry_alloc(inode->i_sb, &retries))
3167 			goto retry;
3168 		return ret;
3169 	}
3170 
3171 	*foliop = folio;
3172 	return ret;
3173 }
3174 
3175 /*
3176  * Check if we should update i_disksize
3177  * when write to the end of file but not require block allocation
3178  */
ext4_da_should_update_i_disksize(struct folio * folio,unsigned long offset)3179 static int ext4_da_should_update_i_disksize(struct folio *folio,
3180 					    unsigned long offset)
3181 {
3182 	struct buffer_head *bh;
3183 	struct inode *inode = folio->mapping->host;
3184 	unsigned int idx;
3185 	int i;
3186 
3187 	bh = folio_buffers(folio);
3188 	idx = offset >> inode->i_blkbits;
3189 
3190 	for (i = 0; i < idx; i++)
3191 		bh = bh->b_this_page;
3192 
3193 	if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
3194 		return 0;
3195 	return 1;
3196 }
3197 
ext4_da_do_write_end(struct address_space * mapping,loff_t pos,unsigned len,unsigned copied,struct folio * folio)3198 static int ext4_da_do_write_end(struct address_space *mapping,
3199 			loff_t pos, unsigned len, unsigned copied,
3200 			struct folio *folio)
3201 {
3202 	struct inode *inode = mapping->host;
3203 	loff_t old_size = inode->i_size;
3204 	bool disksize_changed = false;
3205 	loff_t new_i_size, zero_len = 0;
3206 	handle_t *handle;
3207 
3208 	if (unlikely(!folio_buffers(folio))) {
3209 		folio_unlock(folio);
3210 		folio_put(folio);
3211 		return -EIO;
3212 	}
3213 	/*
3214 	 * block_write_end() will mark the inode as dirty with I_DIRTY_PAGES
3215 	 * flag, which all that's needed to trigger page writeback.
3216 	 */
3217 	copied = block_write_end(pos, len, copied, folio);
3218 	new_i_size = pos + copied;
3219 
3220 	/*
3221 	 * It's important to update i_size while still holding folio lock,
3222 	 * because folio writeout could otherwise come in and zero beyond
3223 	 * i_size.
3224 	 *
3225 	 * Since we are holding inode lock, we are sure i_disksize <=
3226 	 * i_size. We also know that if i_disksize < i_size, there are
3227 	 * delalloc writes pending in the range up to i_size. If the end of
3228 	 * the current write is <= i_size, there's no need to touch
3229 	 * i_disksize since writeback will push i_disksize up to i_size
3230 	 * eventually. If the end of the current write is > i_size and
3231 	 * inside an allocated block which ext4_da_should_update_i_disksize()
3232 	 * checked, we need to update i_disksize here as certain
3233 	 * ext4_writepages() paths not allocating blocks and update i_disksize.
3234 	 */
3235 	if (new_i_size > inode->i_size) {
3236 		unsigned long end;
3237 
3238 		i_size_write(inode, new_i_size);
3239 		end = offset_in_folio(folio, new_i_size - 1);
3240 		if (copied && ext4_da_should_update_i_disksize(folio, end)) {
3241 			ext4_update_i_disksize(inode, new_i_size);
3242 			disksize_changed = true;
3243 		}
3244 	}
3245 
3246 	folio_unlock(folio);
3247 	folio_put(folio);
3248 
3249 	if (pos > old_size) {
3250 		pagecache_isize_extended(inode, old_size, pos);
3251 		zero_len = pos - old_size;
3252 	}
3253 
3254 	if (!disksize_changed && !zero_len)
3255 		return copied;
3256 
3257 	handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
3258 	if (IS_ERR(handle))
3259 		return PTR_ERR(handle);
3260 	if (zero_len)
3261 		ext4_zero_partial_blocks(handle, inode, old_size, zero_len);
3262 	ext4_mark_inode_dirty(handle, inode);
3263 	ext4_journal_stop(handle);
3264 
3265 	return copied;
3266 }
3267 
ext4_da_write_end(const struct kiocb * iocb,struct address_space * mapping,loff_t pos,unsigned len,unsigned copied,struct folio * folio,void * fsdata)3268 static int ext4_da_write_end(const struct kiocb *iocb,
3269 			     struct address_space *mapping,
3270 			     loff_t pos, unsigned len, unsigned copied,
3271 			     struct folio *folio, void *fsdata)
3272 {
3273 	struct inode *inode = mapping->host;
3274 	int write_mode = (int)(unsigned long)fsdata;
3275 
3276 	if (write_mode == FALL_BACK_TO_NONDELALLOC)
3277 		return ext4_write_end(iocb, mapping, pos,
3278 				      len, copied, folio, fsdata);
3279 
3280 	trace_ext4_da_write_end(inode, pos, len, copied);
3281 
3282 	if (write_mode != CONVERT_INLINE_DATA &&
3283 	    ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) &&
3284 	    ext4_has_inline_data(inode))
3285 		return ext4_write_inline_data_end(inode, pos, len, copied,
3286 						  folio);
3287 
3288 	if (unlikely(copied < len) && !folio_test_uptodate(folio))
3289 		copied = 0;
3290 
3291 	return ext4_da_do_write_end(mapping, pos, len, copied, folio);
3292 }
3293 
3294 /*
3295  * Force all delayed allocation blocks to be allocated for a given inode.
3296  */
ext4_alloc_da_blocks(struct inode * inode)3297 int ext4_alloc_da_blocks(struct inode *inode)
3298 {
3299 	trace_ext4_alloc_da_blocks(inode);
3300 
3301 	if (!EXT4_I(inode)->i_reserved_data_blocks)
3302 		return 0;
3303 
3304 	/*
3305 	 * We do something simple for now.  The filemap_flush() will
3306 	 * also start triggering a write of the data blocks, which is
3307 	 * not strictly speaking necessary (and for users of
3308 	 * laptop_mode, not even desirable).  However, to do otherwise
3309 	 * would require replicating code paths in:
3310 	 *
3311 	 * ext4_writepages() ->
3312 	 *    write_cache_pages() ---> (via passed in callback function)
3313 	 *        __mpage_da_writepage() -->
3314 	 *           mpage_add_bh_to_extent()
3315 	 *           mpage_da_map_blocks()
3316 	 *
3317 	 * The problem is that write_cache_pages(), located in
3318 	 * mm/page-writeback.c, marks pages clean in preparation for
3319 	 * doing I/O, which is not desirable if we're not planning on
3320 	 * doing I/O at all.
3321 	 *
3322 	 * We could call write_cache_pages(), and then redirty all of
3323 	 * the pages by calling redirty_page_for_writepage() but that
3324 	 * would be ugly in the extreme.  So instead we would need to
3325 	 * replicate parts of the code in the above functions,
3326 	 * simplifying them because we wouldn't actually intend to
3327 	 * write out the pages, but rather only collect contiguous
3328 	 * logical block extents, call the multi-block allocator, and
3329 	 * then update the buffer heads with the block allocations.
3330 	 *
3331 	 * For now, though, we'll cheat by calling filemap_flush(),
3332 	 * which will map the blocks, and start the I/O, but not
3333 	 * actually wait for the I/O to complete.
3334 	 */
3335 	return filemap_flush(inode->i_mapping);
3336 }
3337 
3338 /*
3339  * bmap() is special.  It gets used by applications such as lilo and by
3340  * the swapper to find the on-disk block of a specific piece of data.
3341  *
3342  * Naturally, this is dangerous if the block concerned is still in the
3343  * journal.  If somebody makes a swapfile on an ext4 data-journaling
3344  * filesystem and enables swap, then they may get a nasty shock when the
3345  * data getting swapped to that swapfile suddenly gets overwritten by
3346  * the original zero's written out previously to the journal and
3347  * awaiting writeback in the kernel's buffer cache.
3348  *
3349  * So, if we see any bmap calls here on a modified, data-journaled file,
3350  * take extra steps to flush any blocks which might be in the cache.
3351  */
ext4_bmap(struct address_space * mapping,sector_t block)3352 static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
3353 {
3354 	struct inode *inode = mapping->host;
3355 	sector_t ret = 0;
3356 
3357 	inode_lock_shared(inode);
3358 	/*
3359 	 * We can get here for an inline file via the FIBMAP ioctl
3360 	 */
3361 	if (ext4_has_inline_data(inode))
3362 		goto out;
3363 
3364 	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
3365 	    (test_opt(inode->i_sb, DELALLOC) ||
3366 	     ext4_should_journal_data(inode))) {
3367 		/*
3368 		 * With delalloc or journalled data we want to sync the file so
3369 		 * that we can make sure we allocate blocks for file and data
3370 		 * is in place for the user to see it
3371 		 */
3372 		filemap_write_and_wait(mapping);
3373 	}
3374 
3375 	ret = iomap_bmap(mapping, block, &ext4_iomap_ops);
3376 
3377 out:
3378 	inode_unlock_shared(inode);
3379 	return ret;
3380 }
3381 
ext4_read_folio(struct file * file,struct folio * folio)3382 static int ext4_read_folio(struct file *file, struct folio *folio)
3383 {
3384 	int ret = -EAGAIN;
3385 	struct inode *inode = folio->mapping->host;
3386 
3387 	trace_ext4_read_folio(inode, folio);
3388 
3389 	if (ext4_has_inline_data(inode))
3390 		ret = ext4_readpage_inline(inode, folio);
3391 
3392 	if (ret == -EAGAIN)
3393 		return ext4_mpage_readpages(inode, NULL, folio);
3394 
3395 	return ret;
3396 }
3397 
ext4_readahead(struct readahead_control * rac)3398 static void ext4_readahead(struct readahead_control *rac)
3399 {
3400 	struct inode *inode = rac->mapping->host;
3401 
3402 	/* If the file has inline data, no need to do readahead. */
3403 	if (ext4_has_inline_data(inode))
3404 		return;
3405 
3406 	ext4_mpage_readpages(inode, rac, NULL);
3407 }
3408 
ext4_invalidate_folio(struct folio * folio,size_t offset,size_t length)3409 static void ext4_invalidate_folio(struct folio *folio, size_t offset,
3410 				size_t length)
3411 {
3412 	trace_ext4_invalidate_folio(folio, offset, length);
3413 
3414 	/* No journalling happens on data buffers when this function is used */
3415 	WARN_ON(folio_buffers(folio) && buffer_jbd(folio_buffers(folio)));
3416 
3417 	block_invalidate_folio(folio, offset, length);
3418 }
3419 
__ext4_journalled_invalidate_folio(struct folio * folio,size_t offset,size_t length)3420 static int __ext4_journalled_invalidate_folio(struct folio *folio,
3421 					    size_t offset, size_t length)
3422 {
3423 	journal_t *journal = EXT4_JOURNAL(folio->mapping->host);
3424 
3425 	trace_ext4_journalled_invalidate_folio(folio, offset, length);
3426 
3427 	/*
3428 	 * If it's a full truncate we just forget about the pending dirtying
3429 	 */
3430 	if (offset == 0 && length == folio_size(folio))
3431 		folio_clear_checked(folio);
3432 
3433 	return jbd2_journal_invalidate_folio(journal, folio, offset, length);
3434 }
3435 
3436 /* Wrapper for aops... */
ext4_journalled_invalidate_folio(struct folio * folio,size_t offset,size_t length)3437 static void ext4_journalled_invalidate_folio(struct folio *folio,
3438 					   size_t offset,
3439 					   size_t length)
3440 {
3441 	WARN_ON(__ext4_journalled_invalidate_folio(folio, offset, length) < 0);
3442 }
3443 
ext4_release_folio(struct folio * folio,gfp_t wait)3444 static bool ext4_release_folio(struct folio *folio, gfp_t wait)
3445 {
3446 	struct inode *inode = folio->mapping->host;
3447 	journal_t *journal = EXT4_JOURNAL(inode);
3448 
3449 	trace_ext4_release_folio(inode, folio);
3450 
3451 	/* Page has dirty journalled data -> cannot release */
3452 	if (folio_test_checked(folio))
3453 		return false;
3454 	if (journal)
3455 		return jbd2_journal_try_to_free_buffers(journal, folio);
3456 	else
3457 		return try_to_free_buffers(folio);
3458 }
3459 
ext4_inode_datasync_dirty(struct inode * inode)3460 static bool ext4_inode_datasync_dirty(struct inode *inode)
3461 {
3462 	journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
3463 
3464 	if (journal) {
3465 		if (jbd2_transaction_committed(journal,
3466 			EXT4_I(inode)->i_datasync_tid))
3467 			return false;
3468 		if (test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT))
3469 			return !list_empty(&EXT4_I(inode)->i_fc_list);
3470 		return true;
3471 	}
3472 
3473 	/* Any metadata buffers to write? */
3474 	if (!list_empty(&inode->i_mapping->i_private_list))
3475 		return true;
3476 	return inode->i_state & I_DIRTY_DATASYNC;
3477 }
3478 
ext4_set_iomap(struct inode * inode,struct iomap * iomap,struct ext4_map_blocks * map,loff_t offset,loff_t length,unsigned int flags)3479 static void ext4_set_iomap(struct inode *inode, struct iomap *iomap,
3480 			   struct ext4_map_blocks *map, loff_t offset,
3481 			   loff_t length, unsigned int flags)
3482 {
3483 	u8 blkbits = inode->i_blkbits;
3484 
3485 	/*
3486 	 * Writes that span EOF might trigger an I/O size update on completion,
3487 	 * so consider them to be dirty for the purpose of O_DSYNC, even if
3488 	 * there is no other metadata changes being made or are pending.
3489 	 */
3490 	iomap->flags = 0;
3491 	if (ext4_inode_datasync_dirty(inode) ||
3492 	    offset + length > i_size_read(inode))
3493 		iomap->flags |= IOMAP_F_DIRTY;
3494 
3495 	if (map->m_flags & EXT4_MAP_NEW)
3496 		iomap->flags |= IOMAP_F_NEW;
3497 
3498 	/* HW-offload atomics are always used */
3499 	if (flags & IOMAP_ATOMIC)
3500 		iomap->flags |= IOMAP_F_ATOMIC_BIO;
3501 
3502 	if (flags & IOMAP_DAX)
3503 		iomap->dax_dev = EXT4_SB(inode->i_sb)->s_daxdev;
3504 	else
3505 		iomap->bdev = inode->i_sb->s_bdev;
3506 	iomap->offset = (u64) map->m_lblk << blkbits;
3507 	iomap->length = (u64) map->m_len << blkbits;
3508 
3509 	if ((map->m_flags & EXT4_MAP_MAPPED) &&
3510 	    !ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3511 		iomap->flags |= IOMAP_F_MERGED;
3512 
3513 	/*
3514 	 * Flags passed to ext4_map_blocks() for direct I/O writes can result
3515 	 * in m_flags having both EXT4_MAP_MAPPED and EXT4_MAP_UNWRITTEN bits
3516 	 * set. In order for any allocated unwritten extents to be converted
3517 	 * into written extents correctly within the ->end_io() handler, we
3518 	 * need to ensure that the iomap->type is set appropriately. Hence, the
3519 	 * reason why we need to check whether the EXT4_MAP_UNWRITTEN bit has
3520 	 * been set first.
3521 	 */
3522 	if (map->m_flags & EXT4_MAP_UNWRITTEN) {
3523 		iomap->type = IOMAP_UNWRITTEN;
3524 		iomap->addr = (u64) map->m_pblk << blkbits;
3525 		if (flags & IOMAP_DAX)
3526 			iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off;
3527 	} else if (map->m_flags & EXT4_MAP_MAPPED) {
3528 		iomap->type = IOMAP_MAPPED;
3529 		iomap->addr = (u64) map->m_pblk << blkbits;
3530 		if (flags & IOMAP_DAX)
3531 			iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off;
3532 	} else if (map->m_flags & EXT4_MAP_DELAYED) {
3533 		iomap->type = IOMAP_DELALLOC;
3534 		iomap->addr = IOMAP_NULL_ADDR;
3535 	} else {
3536 		iomap->type = IOMAP_HOLE;
3537 		iomap->addr = IOMAP_NULL_ADDR;
3538 	}
3539 }
3540 
ext4_map_blocks_atomic_write_slow(handle_t * handle,struct inode * inode,struct ext4_map_blocks * map)3541 static int ext4_map_blocks_atomic_write_slow(handle_t *handle,
3542 			struct inode *inode, struct ext4_map_blocks *map)
3543 {
3544 	ext4_lblk_t m_lblk = map->m_lblk;
3545 	unsigned int m_len = map->m_len;
3546 	unsigned int mapped_len = 0, m_flags = 0;
3547 	ext4_fsblk_t next_pblk;
3548 	bool check_next_pblk = false;
3549 	int ret = 0;
3550 
3551 	WARN_ON_ONCE(!ext4_has_feature_bigalloc(inode->i_sb));
3552 
3553 	/*
3554 	 * This is a slow path in case of mixed mapping. We use
3555 	 * EXT4_GET_BLOCKS_CREATE_ZERO flag here to make sure we get a single
3556 	 * contiguous mapped mapping. This will ensure any unwritten or hole
3557 	 * regions within the requested range is zeroed out and we return
3558 	 * a single contiguous mapped extent.
3559 	 */
3560 	m_flags = EXT4_GET_BLOCKS_CREATE_ZERO;
3561 
3562 	do {
3563 		ret = ext4_map_blocks(handle, inode, map, m_flags);
3564 		if (ret < 0 && ret != -ENOSPC)
3565 			goto out_err;
3566 		/*
3567 		 * This should never happen, but let's return an error code to
3568 		 * avoid an infinite loop in here.
3569 		 */
3570 		if (ret == 0) {
3571 			ret = -EFSCORRUPTED;
3572 			ext4_warning_inode(inode,
3573 				"ext4_map_blocks() couldn't allocate blocks m_flags: 0x%x, ret:%d",
3574 				m_flags, ret);
3575 			goto out_err;
3576 		}
3577 		/*
3578 		 * With bigalloc we should never get ENOSPC nor discontiguous
3579 		 * physical extents.
3580 		 */
3581 		if ((check_next_pblk && next_pblk != map->m_pblk) ||
3582 				ret == -ENOSPC) {
3583 			ext4_warning_inode(inode,
3584 				"Non-contiguous allocation detected: expected %llu, got %llu, "
3585 				"or ext4_map_blocks() returned out of space ret: %d",
3586 				next_pblk, map->m_pblk, ret);
3587 			ret = -EFSCORRUPTED;
3588 			goto out_err;
3589 		}
3590 		next_pblk = map->m_pblk + map->m_len;
3591 		check_next_pblk = true;
3592 
3593 		mapped_len += map->m_len;
3594 		map->m_lblk += map->m_len;
3595 		map->m_len = m_len - mapped_len;
3596 	} while (mapped_len < m_len);
3597 
3598 	/*
3599 	 * We might have done some work in above loop, so we need to query the
3600 	 * start of the physical extent, based on the origin m_lblk and m_len.
3601 	 * Let's also ensure we were able to allocate the required range for
3602 	 * mixed mapping case.
3603 	 */
3604 	map->m_lblk = m_lblk;
3605 	map->m_len = m_len;
3606 	map->m_flags = 0;
3607 
3608 	ret = ext4_map_blocks(handle, inode, map,
3609 			      EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF);
3610 	if (ret != m_len) {
3611 		ext4_warning_inode(inode,
3612 			"allocation failed for atomic write request m_lblk:%u, m_len:%u, ret:%d\n",
3613 			m_lblk, m_len, ret);
3614 		ret = -EINVAL;
3615 	}
3616 	return ret;
3617 
3618 out_err:
3619 	/* reset map before returning an error */
3620 	map->m_lblk = m_lblk;
3621 	map->m_len = m_len;
3622 	map->m_flags = 0;
3623 	return ret;
3624 }
3625 
3626 /*
3627  * ext4_map_blocks_atomic: Helper routine to ensure the entire requested
3628  * range in @map [lblk, lblk + len) is one single contiguous extent with no
3629  * mixed mappings.
3630  *
3631  * We first use m_flags passed to us by our caller (ext4_iomap_alloc()).
3632  * We only call EXT4_GET_BLOCKS_ZERO in the slow path, when the underlying
3633  * physical extent for the requested range does not have a single contiguous
3634  * mapping type i.e. (Hole, Mapped, or Unwritten) throughout.
3635  * In that case we will loop over the requested range to allocate and zero out
3636  * the unwritten / holes in between, to get a single mapped extent from
3637  * [m_lblk, m_lblk +  m_len). Note that this is only possible because we know
3638  * this can be called only with bigalloc enabled filesystem where the underlying
3639  * cluster is already allocated. This avoids allocating discontiguous extents
3640  * in the slow path due to multiple calls to ext4_map_blocks().
3641  * The slow path is mostly non-performance critical path, so it should be ok to
3642  * loop using ext4_map_blocks() with appropriate flags to allocate & zero the
3643  * underlying short holes/unwritten extents within the requested range.
3644  */
ext4_map_blocks_atomic_write(handle_t * handle,struct inode * inode,struct ext4_map_blocks * map,int m_flags,bool * force_commit)3645 static int ext4_map_blocks_atomic_write(handle_t *handle, struct inode *inode,
3646 				struct ext4_map_blocks *map, int m_flags,
3647 				bool *force_commit)
3648 {
3649 	ext4_lblk_t m_lblk = map->m_lblk;
3650 	unsigned int m_len = map->m_len;
3651 	int ret = 0;
3652 
3653 	WARN_ON_ONCE(m_len > 1 && !ext4_has_feature_bigalloc(inode->i_sb));
3654 
3655 	ret = ext4_map_blocks(handle, inode, map, m_flags);
3656 	if (ret < 0 || ret == m_len)
3657 		goto out;
3658 	/*
3659 	 * This is a mixed mapping case where we were not able to allocate
3660 	 * a single contiguous extent. In that case let's reset requested
3661 	 * mapping and call the slow path.
3662 	 */
3663 	map->m_lblk = m_lblk;
3664 	map->m_len = m_len;
3665 	map->m_flags = 0;
3666 
3667 	/*
3668 	 * slow path means we have mixed mapping, that means we will need
3669 	 * to force txn commit.
3670 	 */
3671 	*force_commit = true;
3672 	return ext4_map_blocks_atomic_write_slow(handle, inode, map);
3673 out:
3674 	return ret;
3675 }
3676 
ext4_iomap_alloc(struct inode * inode,struct ext4_map_blocks * map,unsigned int flags)3677 static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map,
3678 			    unsigned int flags)
3679 {
3680 	handle_t *handle;
3681 	u8 blkbits = inode->i_blkbits;
3682 	int ret, dio_credits, m_flags = 0, retries = 0;
3683 	bool force_commit = false;
3684 
3685 	/*
3686 	 * Trim the mapping request to the maximum value that we can map at
3687 	 * once for direct I/O.
3688 	 */
3689 	if (map->m_len > DIO_MAX_BLOCKS)
3690 		map->m_len = DIO_MAX_BLOCKS;
3691 
3692 	/*
3693 	 * journal credits estimation for atomic writes. We call
3694 	 * ext4_map_blocks(), to find if there could be a mixed mapping. If yes,
3695 	 * then let's assume the no. of pextents required can be m_len i.e.
3696 	 * every alternate block can be unwritten and hole.
3697 	 */
3698 	if (flags & IOMAP_ATOMIC) {
3699 		unsigned int orig_mlen = map->m_len;
3700 
3701 		ret = ext4_map_blocks(NULL, inode, map, 0);
3702 		if (ret < 0)
3703 			return ret;
3704 		if (map->m_len < orig_mlen) {
3705 			map->m_len = orig_mlen;
3706 			dio_credits = ext4_meta_trans_blocks(inode, orig_mlen,
3707 							     map->m_len);
3708 		} else {
3709 			dio_credits = ext4_chunk_trans_blocks(inode,
3710 							      map->m_len);
3711 		}
3712 	} else {
3713 		dio_credits = ext4_chunk_trans_blocks(inode, map->m_len);
3714 	}
3715 
3716 retry:
3717 	/*
3718 	 * Either we allocate blocks and then don't get an unwritten extent, so
3719 	 * in that case we have reserved enough credits. Or, the blocks are
3720 	 * already allocated and unwritten. In that case, the extent conversion
3721 	 * fits into the credits as well.
3722 	 */
3723 	handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);
3724 	if (IS_ERR(handle))
3725 		return PTR_ERR(handle);
3726 
3727 	/*
3728 	 * DAX and direct I/O are the only two operations that are currently
3729 	 * supported with IOMAP_WRITE.
3730 	 */
3731 	WARN_ON(!(flags & (IOMAP_DAX | IOMAP_DIRECT)));
3732 	if (flags & IOMAP_DAX)
3733 		m_flags = EXT4_GET_BLOCKS_CREATE_ZERO;
3734 	/*
3735 	 * We use i_size instead of i_disksize here because delalloc writeback
3736 	 * can complete at any point during the I/O and subsequently push the
3737 	 * i_disksize out to i_size. This could be beyond where direct I/O is
3738 	 * happening and thus expose allocated blocks to direct I/O reads.
3739 	 */
3740 	else if (((loff_t)map->m_lblk << blkbits) >= i_size_read(inode))
3741 		m_flags = EXT4_GET_BLOCKS_CREATE;
3742 	else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3743 		m_flags = EXT4_GET_BLOCKS_IO_CREATE_EXT;
3744 
3745 	if (flags & IOMAP_ATOMIC)
3746 		ret = ext4_map_blocks_atomic_write(handle, inode, map, m_flags,
3747 						   &force_commit);
3748 	else
3749 		ret = ext4_map_blocks(handle, inode, map, m_flags);
3750 
3751 	/*
3752 	 * We cannot fill holes in indirect tree based inodes as that could
3753 	 * expose stale data in the case of a crash. Use the magic error code
3754 	 * to fallback to buffered I/O.
3755 	 */
3756 	if (!m_flags && !ret)
3757 		ret = -ENOTBLK;
3758 
3759 	ext4_journal_stop(handle);
3760 	if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
3761 		goto retry;
3762 
3763 	/*
3764 	 * Force commit the current transaction if the allocation spans a mixed
3765 	 * mapping range. This ensures any pending metadata updates (like
3766 	 * unwritten to written extents conversion) in this range are in
3767 	 * consistent state with the file data blocks, before performing the
3768 	 * actual write I/O. If the commit fails, the whole I/O must be aborted
3769 	 * to prevent any possible torn writes.
3770 	 */
3771 	if (ret > 0 && force_commit) {
3772 		int ret2;
3773 
3774 		ret2 = ext4_force_commit(inode->i_sb);
3775 		if (ret2)
3776 			return ret2;
3777 	}
3778 
3779 	return ret;
3780 }
3781 
3782 
ext4_iomap_begin(struct inode * inode,loff_t offset,loff_t length,unsigned flags,struct iomap * iomap,struct iomap * srcmap)3783 static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
3784 		unsigned flags, struct iomap *iomap, struct iomap *srcmap)
3785 {
3786 	int ret;
3787 	struct ext4_map_blocks map;
3788 	u8 blkbits = inode->i_blkbits;
3789 	unsigned int orig_mlen;
3790 
3791 	if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3792 		return -EINVAL;
3793 
3794 	if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
3795 		return -ERANGE;
3796 
3797 	/*
3798 	 * Calculate the first and last logical blocks respectively.
3799 	 */
3800 	map.m_lblk = offset >> blkbits;
3801 	map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3802 			  EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
3803 	orig_mlen = map.m_len;
3804 
3805 	if (flags & IOMAP_WRITE) {
3806 		/*
3807 		 * We check here if the blocks are already allocated, then we
3808 		 * don't need to start a journal txn and we can directly return
3809 		 * the mapping information. This could boost performance
3810 		 * especially in multi-threaded overwrite requests.
3811 		 */
3812 		if (offset + length <= i_size_read(inode)) {
3813 			ret = ext4_map_blocks(NULL, inode, &map, 0);
3814 			/*
3815 			 * For atomic writes the entire requested length should
3816 			 * be mapped.
3817 			 */
3818 			if (map.m_flags & EXT4_MAP_MAPPED) {
3819 				if ((!(flags & IOMAP_ATOMIC) && ret > 0) ||
3820 				   (flags & IOMAP_ATOMIC && ret >= orig_mlen))
3821 					goto out;
3822 			}
3823 			map.m_len = orig_mlen;
3824 		}
3825 		ret = ext4_iomap_alloc(inode, &map, flags);
3826 	} else {
3827 		/*
3828 		 * This can be called for overwrites path from
3829 		 * ext4_iomap_overwrite_begin().
3830 		 */
3831 		ret = ext4_map_blocks(NULL, inode, &map, 0);
3832 	}
3833 
3834 	if (ret < 0)
3835 		return ret;
3836 out:
3837 	/*
3838 	 * When inline encryption is enabled, sometimes I/O to an encrypted file
3839 	 * has to be broken up to guarantee DUN contiguity.  Handle this by
3840 	 * limiting the length of the mapping returned.
3841 	 */
3842 	map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len);
3843 
3844 	/*
3845 	 * Before returning to iomap, let's ensure the allocated mapping
3846 	 * covers the entire requested length for atomic writes.
3847 	 */
3848 	if (flags & IOMAP_ATOMIC) {
3849 		if (map.m_len < (length >> blkbits)) {
3850 			WARN_ON_ONCE(1);
3851 			return -EINVAL;
3852 		}
3853 	}
3854 	ext4_set_iomap(inode, iomap, &map, offset, length, flags);
3855 
3856 	return 0;
3857 }
3858 
ext4_iomap_overwrite_begin(struct inode * inode,loff_t offset,loff_t length,unsigned flags,struct iomap * iomap,struct iomap * srcmap)3859 static int ext4_iomap_overwrite_begin(struct inode *inode, loff_t offset,
3860 		loff_t length, unsigned flags, struct iomap *iomap,
3861 		struct iomap *srcmap)
3862 {
3863 	int ret;
3864 
3865 	/*
3866 	 * Even for writes we don't need to allocate blocks, so just pretend
3867 	 * we are reading to save overhead of starting a transaction.
3868 	 */
3869 	flags &= ~IOMAP_WRITE;
3870 	ret = ext4_iomap_begin(inode, offset, length, flags, iomap, srcmap);
3871 	WARN_ON_ONCE(!ret && iomap->type != IOMAP_MAPPED);
3872 	return ret;
3873 }
3874 
ext4_want_directio_fallback(unsigned flags,ssize_t written)3875 static inline bool ext4_want_directio_fallback(unsigned flags, ssize_t written)
3876 {
3877 	/* must be a directio to fall back to buffered */
3878 	if ((flags & (IOMAP_WRITE | IOMAP_DIRECT)) !=
3879 		    (IOMAP_WRITE | IOMAP_DIRECT))
3880 		return false;
3881 
3882 	/* atomic writes are all-or-nothing */
3883 	if (flags & IOMAP_ATOMIC)
3884 		return false;
3885 
3886 	/* can only try again if we wrote nothing */
3887 	return written == 0;
3888 }
3889 
ext4_iomap_end(struct inode * inode,loff_t offset,loff_t length,ssize_t written,unsigned flags,struct iomap * iomap)3890 static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length,
3891 			  ssize_t written, unsigned flags, struct iomap *iomap)
3892 {
3893 	/*
3894 	 * Check to see whether an error occurred while writing out the data to
3895 	 * the allocated blocks. If so, return the magic error code for
3896 	 * non-atomic write so that we fallback to buffered I/O and attempt to
3897 	 * complete the remainder of the I/O.
3898 	 * For non-atomic writes, any blocks that may have been
3899 	 * allocated in preparation for the direct I/O will be reused during
3900 	 * buffered I/O. For atomic write, we never fallback to buffered-io.
3901 	 */
3902 	if (ext4_want_directio_fallback(flags, written))
3903 		return -ENOTBLK;
3904 
3905 	return 0;
3906 }
3907 
3908 const struct iomap_ops ext4_iomap_ops = {
3909 	.iomap_begin		= ext4_iomap_begin,
3910 	.iomap_end		= ext4_iomap_end,
3911 };
3912 
3913 const struct iomap_ops ext4_iomap_overwrite_ops = {
3914 	.iomap_begin		= ext4_iomap_overwrite_begin,
3915 	.iomap_end		= ext4_iomap_end,
3916 };
3917 
ext4_iomap_begin_report(struct inode * inode,loff_t offset,loff_t length,unsigned int flags,struct iomap * iomap,struct iomap * srcmap)3918 static int ext4_iomap_begin_report(struct inode *inode, loff_t offset,
3919 				   loff_t length, unsigned int flags,
3920 				   struct iomap *iomap, struct iomap *srcmap)
3921 {
3922 	int ret;
3923 	struct ext4_map_blocks map;
3924 	u8 blkbits = inode->i_blkbits;
3925 
3926 	if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3927 		return -EINVAL;
3928 
3929 	if (ext4_has_inline_data(inode)) {
3930 		ret = ext4_inline_data_iomap(inode, iomap);
3931 		if (ret != -EAGAIN) {
3932 			if (ret == 0 && offset >= iomap->length)
3933 				ret = -ENOENT;
3934 			return ret;
3935 		}
3936 	}
3937 
3938 	/*
3939 	 * Calculate the first and last logical block respectively.
3940 	 */
3941 	map.m_lblk = offset >> blkbits;
3942 	map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3943 			  EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
3944 
3945 	/*
3946 	 * Fiemap callers may call for offset beyond s_bitmap_maxbytes.
3947 	 * So handle it here itself instead of querying ext4_map_blocks().
3948 	 * Since ext4_map_blocks() will warn about it and will return
3949 	 * -EIO error.
3950 	 */
3951 	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
3952 		struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
3953 
3954 		if (offset >= sbi->s_bitmap_maxbytes) {
3955 			map.m_flags = 0;
3956 			goto set_iomap;
3957 		}
3958 	}
3959 
3960 	ret = ext4_map_blocks(NULL, inode, &map, 0);
3961 	if (ret < 0)
3962 		return ret;
3963 set_iomap:
3964 	ext4_set_iomap(inode, iomap, &map, offset, length, flags);
3965 
3966 	return 0;
3967 }
3968 
3969 const struct iomap_ops ext4_iomap_report_ops = {
3970 	.iomap_begin = ext4_iomap_begin_report,
3971 };
3972 
3973 /*
3974  * For data=journal mode, folio should be marked dirty only when it was
3975  * writeably mapped. When that happens, it was already attached to the
3976  * transaction and marked as jbddirty (we take care of this in
3977  * ext4_page_mkwrite()). On transaction commit, we writeprotect page mappings
3978  * so we should have nothing to do here, except for the case when someone
3979  * had the page pinned and dirtied the page through this pin (e.g. by doing
3980  * direct IO to it). In that case we'd need to attach buffers here to the
3981  * transaction but we cannot due to lock ordering.  We cannot just dirty the
3982  * folio and leave attached buffers clean, because the buffers' dirty state is
3983  * "definitive".  We cannot just set the buffers dirty or jbddirty because all
3984  * the journalling code will explode.  So what we do is to mark the folio
3985  * "pending dirty" and next time ext4_writepages() is called, attach buffers
3986  * to the transaction appropriately.
3987  */
ext4_journalled_dirty_folio(struct address_space * mapping,struct folio * folio)3988 static bool ext4_journalled_dirty_folio(struct address_space *mapping,
3989 		struct folio *folio)
3990 {
3991 	WARN_ON_ONCE(!folio_buffers(folio));
3992 	if (folio_maybe_dma_pinned(folio))
3993 		folio_set_checked(folio);
3994 	return filemap_dirty_folio(mapping, folio);
3995 }
3996 
ext4_dirty_folio(struct address_space * mapping,struct folio * folio)3997 static bool ext4_dirty_folio(struct address_space *mapping, struct folio *folio)
3998 {
3999 	WARN_ON_ONCE(!folio_test_locked(folio) && !folio_test_dirty(folio));
4000 	WARN_ON_ONCE(!folio_buffers(folio));
4001 	return block_dirty_folio(mapping, folio);
4002 }
4003 
ext4_iomap_swap_activate(struct swap_info_struct * sis,struct file * file,sector_t * span)4004 static int ext4_iomap_swap_activate(struct swap_info_struct *sis,
4005 				    struct file *file, sector_t *span)
4006 {
4007 	return iomap_swapfile_activate(sis, file, span,
4008 				       &ext4_iomap_report_ops);
4009 }
4010 
4011 static const struct address_space_operations ext4_aops = {
4012 	.read_folio		= ext4_read_folio,
4013 	.readahead		= ext4_readahead,
4014 	.writepages		= ext4_writepages,
4015 	.write_begin		= ext4_write_begin,
4016 	.write_end		= ext4_write_end,
4017 	.dirty_folio		= ext4_dirty_folio,
4018 	.bmap			= ext4_bmap,
4019 	.invalidate_folio	= ext4_invalidate_folio,
4020 	.release_folio		= ext4_release_folio,
4021 	.migrate_folio		= buffer_migrate_folio,
4022 	.is_partially_uptodate  = block_is_partially_uptodate,
4023 	.error_remove_folio	= generic_error_remove_folio,
4024 	.swap_activate		= ext4_iomap_swap_activate,
4025 };
4026 
4027 static const struct address_space_operations ext4_journalled_aops = {
4028 	.read_folio		= ext4_read_folio,
4029 	.readahead		= ext4_readahead,
4030 	.writepages		= ext4_writepages,
4031 	.write_begin		= ext4_write_begin,
4032 	.write_end		= ext4_journalled_write_end,
4033 	.dirty_folio		= ext4_journalled_dirty_folio,
4034 	.bmap			= ext4_bmap,
4035 	.invalidate_folio	= ext4_journalled_invalidate_folio,
4036 	.release_folio		= ext4_release_folio,
4037 	.migrate_folio		= buffer_migrate_folio_norefs,
4038 	.is_partially_uptodate  = block_is_partially_uptodate,
4039 	.error_remove_folio	= generic_error_remove_folio,
4040 	.swap_activate		= ext4_iomap_swap_activate,
4041 };
4042 
4043 static const struct address_space_operations ext4_da_aops = {
4044 	.read_folio		= ext4_read_folio,
4045 	.readahead		= ext4_readahead,
4046 	.writepages		= ext4_writepages,
4047 	.write_begin		= ext4_da_write_begin,
4048 	.write_end		= ext4_da_write_end,
4049 	.dirty_folio		= ext4_dirty_folio,
4050 	.bmap			= ext4_bmap,
4051 	.invalidate_folio	= ext4_invalidate_folio,
4052 	.release_folio		= ext4_release_folio,
4053 	.migrate_folio		= buffer_migrate_folio,
4054 	.is_partially_uptodate  = block_is_partially_uptodate,
4055 	.error_remove_folio	= generic_error_remove_folio,
4056 	.swap_activate		= ext4_iomap_swap_activate,
4057 };
4058 
4059 static const struct address_space_operations ext4_dax_aops = {
4060 	.writepages		= ext4_dax_writepages,
4061 	.dirty_folio		= noop_dirty_folio,
4062 	.bmap			= ext4_bmap,
4063 	.swap_activate		= ext4_iomap_swap_activate,
4064 };
4065 
ext4_set_aops(struct inode * inode)4066 void ext4_set_aops(struct inode *inode)
4067 {
4068 	switch (ext4_inode_journal_mode(inode)) {
4069 	case EXT4_INODE_ORDERED_DATA_MODE:
4070 	case EXT4_INODE_WRITEBACK_DATA_MODE:
4071 		break;
4072 	case EXT4_INODE_JOURNAL_DATA_MODE:
4073 		inode->i_mapping->a_ops = &ext4_journalled_aops;
4074 		return;
4075 	default:
4076 		BUG();
4077 	}
4078 	if (IS_DAX(inode))
4079 		inode->i_mapping->a_ops = &ext4_dax_aops;
4080 	else if (test_opt(inode->i_sb, DELALLOC))
4081 		inode->i_mapping->a_ops = &ext4_da_aops;
4082 	else
4083 		inode->i_mapping->a_ops = &ext4_aops;
4084 }
4085 
4086 /*
4087  * Here we can't skip an unwritten buffer even though it usually reads zero
4088  * because it might have data in pagecache (eg, if called from ext4_zero_range,
4089  * ext4_punch_hole, etc) which needs to be properly zeroed out. Otherwise a
4090  * racing writeback can come later and flush the stale pagecache to disk.
4091  */
__ext4_block_zero_page_range(handle_t * handle,struct address_space * mapping,loff_t from,loff_t length)4092 static int __ext4_block_zero_page_range(handle_t *handle,
4093 		struct address_space *mapping, loff_t from, loff_t length)
4094 {
4095 	unsigned int offset, blocksize, pos;
4096 	ext4_lblk_t iblock;
4097 	struct inode *inode = mapping->host;
4098 	struct buffer_head *bh;
4099 	struct folio *folio;
4100 	int err = 0;
4101 
4102 	folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT,
4103 				    FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
4104 				    mapping_gfp_constraint(mapping, ~__GFP_FS));
4105 	if (IS_ERR(folio))
4106 		return PTR_ERR(folio);
4107 
4108 	blocksize = inode->i_sb->s_blocksize;
4109 
4110 	iblock = folio->index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits);
4111 
4112 	bh = folio_buffers(folio);
4113 	if (!bh)
4114 		bh = create_empty_buffers(folio, blocksize, 0);
4115 
4116 	/* Find the buffer that contains "offset" */
4117 	offset = offset_in_folio(folio, from);
4118 	pos = blocksize;
4119 	while (offset >= pos) {
4120 		bh = bh->b_this_page;
4121 		iblock++;
4122 		pos += blocksize;
4123 	}
4124 	if (buffer_freed(bh)) {
4125 		BUFFER_TRACE(bh, "freed: skip");
4126 		goto unlock;
4127 	}
4128 	if (!buffer_mapped(bh)) {
4129 		BUFFER_TRACE(bh, "unmapped");
4130 		ext4_get_block(inode, iblock, bh, 0);
4131 		/* unmapped? It's a hole - nothing to do */
4132 		if (!buffer_mapped(bh)) {
4133 			BUFFER_TRACE(bh, "still unmapped");
4134 			goto unlock;
4135 		}
4136 	}
4137 
4138 	/* Ok, it's mapped. Make sure it's up-to-date */
4139 	if (folio_test_uptodate(folio))
4140 		set_buffer_uptodate(bh);
4141 
4142 	if (!buffer_uptodate(bh)) {
4143 		err = ext4_read_bh_lock(bh, 0, true);
4144 		if (err)
4145 			goto unlock;
4146 		if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
4147 			/* We expect the key to be set. */
4148 			BUG_ON(!fscrypt_has_encryption_key(inode));
4149 			err = fscrypt_decrypt_pagecache_blocks(folio,
4150 							       blocksize,
4151 							       bh_offset(bh));
4152 			if (err) {
4153 				clear_buffer_uptodate(bh);
4154 				goto unlock;
4155 			}
4156 		}
4157 	}
4158 	if (ext4_should_journal_data(inode)) {
4159 		BUFFER_TRACE(bh, "get write access");
4160 		err = ext4_journal_get_write_access(handle, inode->i_sb, bh,
4161 						    EXT4_JTR_NONE);
4162 		if (err)
4163 			goto unlock;
4164 	}
4165 	folio_zero_range(folio, offset, length);
4166 	BUFFER_TRACE(bh, "zeroed end of block");
4167 
4168 	if (ext4_should_journal_data(inode)) {
4169 		err = ext4_dirty_journalled_data(handle, bh);
4170 	} else {
4171 		err = 0;
4172 		mark_buffer_dirty(bh);
4173 		if (ext4_should_order_data(inode))
4174 			err = ext4_jbd2_inode_add_write(handle, inode, from,
4175 					length);
4176 	}
4177 
4178 unlock:
4179 	folio_unlock(folio);
4180 	folio_put(folio);
4181 	return err;
4182 }
4183 
4184 /*
4185  * ext4_block_zero_page_range() zeros out a mapping of length 'length'
4186  * starting from file offset 'from'.  The range to be zero'd must
4187  * be contained with in one block.  If the specified range exceeds
4188  * the end of the block it will be shortened to end of the block
4189  * that corresponds to 'from'
4190  */
ext4_block_zero_page_range(handle_t * handle,struct address_space * mapping,loff_t from,loff_t length)4191 static int ext4_block_zero_page_range(handle_t *handle,
4192 		struct address_space *mapping, loff_t from, loff_t length)
4193 {
4194 	struct inode *inode = mapping->host;
4195 	unsigned offset = from & (PAGE_SIZE-1);
4196 	unsigned blocksize = inode->i_sb->s_blocksize;
4197 	unsigned max = blocksize - (offset & (blocksize - 1));
4198 
4199 	/*
4200 	 * correct length if it does not fall between
4201 	 * 'from' and the end of the block
4202 	 */
4203 	if (length > max || length < 0)
4204 		length = max;
4205 
4206 	if (IS_DAX(inode)) {
4207 		return dax_zero_range(inode, from, length, NULL,
4208 				      &ext4_iomap_ops);
4209 	}
4210 	return __ext4_block_zero_page_range(handle, mapping, from, length);
4211 }
4212 
4213 /*
4214  * ext4_block_truncate_page() zeroes out a mapping from file offset `from'
4215  * up to the end of the block which corresponds to `from'.
4216  * This required during truncate. We need to physically zero the tail end
4217  * of that block so it doesn't yield old data if the file is later grown.
4218  */
ext4_block_truncate_page(handle_t * handle,struct address_space * mapping,loff_t from)4219 static int ext4_block_truncate_page(handle_t *handle,
4220 		struct address_space *mapping, loff_t from)
4221 {
4222 	unsigned offset = from & (PAGE_SIZE-1);
4223 	unsigned length;
4224 	unsigned blocksize;
4225 	struct inode *inode = mapping->host;
4226 
4227 	/* If we are processing an encrypted inode during orphan list handling */
4228 	if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode))
4229 		return 0;
4230 
4231 	blocksize = inode->i_sb->s_blocksize;
4232 	length = blocksize - (offset & (blocksize - 1));
4233 
4234 	return ext4_block_zero_page_range(handle, mapping, from, length);
4235 }
4236 
ext4_zero_partial_blocks(handle_t * handle,struct inode * inode,loff_t lstart,loff_t length)4237 int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode,
4238 			     loff_t lstart, loff_t length)
4239 {
4240 	struct super_block *sb = inode->i_sb;
4241 	struct address_space *mapping = inode->i_mapping;
4242 	unsigned partial_start, partial_end;
4243 	ext4_fsblk_t start, end;
4244 	loff_t byte_end = (lstart + length - 1);
4245 	int err = 0;
4246 
4247 	partial_start = lstart & (sb->s_blocksize - 1);
4248 	partial_end = byte_end & (sb->s_blocksize - 1);
4249 
4250 	start = lstart >> sb->s_blocksize_bits;
4251 	end = byte_end >> sb->s_blocksize_bits;
4252 
4253 	/* Handle partial zero within the single block */
4254 	if (start == end &&
4255 	    (partial_start || (partial_end != sb->s_blocksize - 1))) {
4256 		err = ext4_block_zero_page_range(handle, mapping,
4257 						 lstart, length);
4258 		return err;
4259 	}
4260 	/* Handle partial zero out on the start of the range */
4261 	if (partial_start) {
4262 		err = ext4_block_zero_page_range(handle, mapping,
4263 						 lstart, sb->s_blocksize);
4264 		if (err)
4265 			return err;
4266 	}
4267 	/* Handle partial zero out on the end of the range */
4268 	if (partial_end != sb->s_blocksize - 1)
4269 		err = ext4_block_zero_page_range(handle, mapping,
4270 						 byte_end - partial_end,
4271 						 partial_end + 1);
4272 	return err;
4273 }
4274 
ext4_can_truncate(struct inode * inode)4275 int ext4_can_truncate(struct inode *inode)
4276 {
4277 	if (S_ISREG(inode->i_mode))
4278 		return 1;
4279 	if (S_ISDIR(inode->i_mode))
4280 		return 1;
4281 	if (S_ISLNK(inode->i_mode))
4282 		return !ext4_inode_is_fast_symlink(inode);
4283 	return 0;
4284 }
4285 
4286 /*
4287  * We have to make sure i_disksize gets properly updated before we truncate
4288  * page cache due to hole punching or zero range. Otherwise i_disksize update
4289  * can get lost as it may have been postponed to submission of writeback but
4290  * that will never happen after we truncate page cache.
4291  */
ext4_update_disksize_before_punch(struct inode * inode,loff_t offset,loff_t len)4292 int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset,
4293 				      loff_t len)
4294 {
4295 	handle_t *handle;
4296 	int ret;
4297 
4298 	loff_t size = i_size_read(inode);
4299 
4300 	WARN_ON(!inode_is_locked(inode));
4301 	if (offset > size || offset + len < size)
4302 		return 0;
4303 
4304 	if (EXT4_I(inode)->i_disksize >= size)
4305 		return 0;
4306 
4307 	handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
4308 	if (IS_ERR(handle))
4309 		return PTR_ERR(handle);
4310 	ext4_update_i_disksize(inode, size);
4311 	ret = ext4_mark_inode_dirty(handle, inode);
4312 	ext4_journal_stop(handle);
4313 
4314 	return ret;
4315 }
4316 
ext4_truncate_folio(struct inode * inode,loff_t start,loff_t end)4317 static inline void ext4_truncate_folio(struct inode *inode,
4318 				       loff_t start, loff_t end)
4319 {
4320 	unsigned long blocksize = i_blocksize(inode);
4321 	struct folio *folio;
4322 
4323 	/* Nothing to be done if no complete block needs to be truncated. */
4324 	if (round_up(start, blocksize) >= round_down(end, blocksize))
4325 		return;
4326 
4327 	folio = filemap_lock_folio(inode->i_mapping, start >> PAGE_SHIFT);
4328 	if (IS_ERR(folio))
4329 		return;
4330 
4331 	if (folio_mkclean(folio))
4332 		folio_mark_dirty(folio);
4333 	folio_unlock(folio);
4334 	folio_put(folio);
4335 }
4336 
ext4_truncate_page_cache_block_range(struct inode * inode,loff_t start,loff_t end)4337 int ext4_truncate_page_cache_block_range(struct inode *inode,
4338 					 loff_t start, loff_t end)
4339 {
4340 	unsigned long blocksize = i_blocksize(inode);
4341 	int ret;
4342 
4343 	/*
4344 	 * For journalled data we need to write (and checkpoint) pages
4345 	 * before discarding page cache to avoid inconsitent data on disk
4346 	 * in case of crash before freeing or unwritten converting trans
4347 	 * is committed.
4348 	 */
4349 	if (ext4_should_journal_data(inode)) {
4350 		ret = filemap_write_and_wait_range(inode->i_mapping, start,
4351 						   end - 1);
4352 		if (ret)
4353 			return ret;
4354 		goto truncate_pagecache;
4355 	}
4356 
4357 	/*
4358 	 * If the block size is less than the page size, the file's mapped
4359 	 * blocks within one page could be freed or converted to unwritten.
4360 	 * So it's necessary to remove writable userspace mappings, and then
4361 	 * ext4_page_mkwrite() can be called during subsequent write access
4362 	 * to these partial folios.
4363 	 */
4364 	if (!IS_ALIGNED(start | end, PAGE_SIZE) &&
4365 	    blocksize < PAGE_SIZE && start < inode->i_size) {
4366 		loff_t page_boundary = round_up(start, PAGE_SIZE);
4367 
4368 		ext4_truncate_folio(inode, start, min(page_boundary, end));
4369 		if (end > page_boundary)
4370 			ext4_truncate_folio(inode,
4371 					    round_down(end, PAGE_SIZE), end);
4372 	}
4373 
4374 truncate_pagecache:
4375 	truncate_pagecache_range(inode, start, end - 1);
4376 	return 0;
4377 }
4378 
ext4_wait_dax_page(struct inode * inode)4379 static void ext4_wait_dax_page(struct inode *inode)
4380 {
4381 	filemap_invalidate_unlock(inode->i_mapping);
4382 	schedule();
4383 	filemap_invalidate_lock(inode->i_mapping);
4384 }
4385 
ext4_break_layouts(struct inode * inode)4386 int ext4_break_layouts(struct inode *inode)
4387 {
4388 	if (WARN_ON_ONCE(!rwsem_is_locked(&inode->i_mapping->invalidate_lock)))
4389 		return -EINVAL;
4390 
4391 	return dax_break_layout_inode(inode, ext4_wait_dax_page);
4392 }
4393 
4394 /*
4395  * ext4_punch_hole: punches a hole in a file by releasing the blocks
4396  * associated with the given offset and length
4397  *
4398  * @inode:  File inode
4399  * @offset: The offset where the hole will begin
4400  * @len:    The length of the hole
4401  *
4402  * Returns: 0 on success or negative on failure
4403  */
4404 
ext4_punch_hole(struct file * file,loff_t offset,loff_t length)4405 int ext4_punch_hole(struct file *file, loff_t offset, loff_t length)
4406 {
4407 	struct inode *inode = file_inode(file);
4408 	struct super_block *sb = inode->i_sb;
4409 	ext4_lblk_t start_lblk, end_lblk;
4410 	loff_t max_end = sb->s_maxbytes;
4411 	loff_t end = offset + length;
4412 	handle_t *handle;
4413 	unsigned int credits;
4414 	int ret;
4415 
4416 	trace_ext4_punch_hole(inode, offset, length, 0);
4417 	WARN_ON_ONCE(!inode_is_locked(inode));
4418 
4419 	/*
4420 	 * For indirect-block based inodes, make sure that the hole within
4421 	 * one block before last range.
4422 	 */
4423 	if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4424 		max_end = EXT4_SB(sb)->s_bitmap_maxbytes - sb->s_blocksize;
4425 
4426 	/* No need to punch hole beyond i_size */
4427 	if (offset >= inode->i_size || offset >= max_end)
4428 		return 0;
4429 
4430 	/*
4431 	 * If the hole extends beyond i_size, set the hole to end after
4432 	 * the page that contains i_size.
4433 	 */
4434 	if (end > inode->i_size)
4435 		end = round_up(inode->i_size, PAGE_SIZE);
4436 	if (end > max_end)
4437 		end = max_end;
4438 	length = end - offset;
4439 
4440 	/*
4441 	 * Attach jinode to inode for jbd2 if we do any zeroing of partial
4442 	 * block.
4443 	 */
4444 	if (!IS_ALIGNED(offset | end, sb->s_blocksize)) {
4445 		ret = ext4_inode_attach_jinode(inode);
4446 		if (ret < 0)
4447 			return ret;
4448 	}
4449 
4450 
4451 	ret = ext4_update_disksize_before_punch(inode, offset, length);
4452 	if (ret)
4453 		return ret;
4454 
4455 	/* Now release the pages and zero block aligned part of pages*/
4456 	ret = ext4_truncate_page_cache_block_range(inode, offset, end);
4457 	if (ret)
4458 		return ret;
4459 
4460 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4461 		credits = ext4_chunk_trans_extent(inode, 2);
4462 	else
4463 		credits = ext4_blocks_for_truncate(inode);
4464 	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4465 	if (IS_ERR(handle)) {
4466 		ret = PTR_ERR(handle);
4467 		ext4_std_error(sb, ret);
4468 		return ret;
4469 	}
4470 
4471 	ret = ext4_zero_partial_blocks(handle, inode, offset, length);
4472 	if (ret)
4473 		goto out_handle;
4474 
4475 	/* If there are blocks to remove, do it */
4476 	start_lblk = EXT4_B_TO_LBLK(inode, offset);
4477 	end_lblk = end >> inode->i_blkbits;
4478 
4479 	if (end_lblk > start_lblk) {
4480 		ext4_lblk_t hole_len = end_lblk - start_lblk;
4481 
4482 		ext4_fc_track_inode(handle, inode);
4483 		ext4_check_map_extents_env(inode);
4484 		down_write(&EXT4_I(inode)->i_data_sem);
4485 		ext4_discard_preallocations(inode);
4486 
4487 		ext4_es_remove_extent(inode, start_lblk, hole_len);
4488 
4489 		if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4490 			ret = ext4_ext_remove_space(inode, start_lblk,
4491 						    end_lblk - 1);
4492 		else
4493 			ret = ext4_ind_remove_space(handle, inode, start_lblk,
4494 						    end_lblk);
4495 		if (ret) {
4496 			up_write(&EXT4_I(inode)->i_data_sem);
4497 			goto out_handle;
4498 		}
4499 
4500 		ext4_es_insert_extent(inode, start_lblk, hole_len, ~0,
4501 				      EXTENT_STATUS_HOLE, 0);
4502 		up_write(&EXT4_I(inode)->i_data_sem);
4503 	}
4504 	ext4_fc_track_range(handle, inode, start_lblk, end_lblk);
4505 
4506 	ret = ext4_mark_inode_dirty(handle, inode);
4507 	if (unlikely(ret))
4508 		goto out_handle;
4509 
4510 	ext4_update_inode_fsync_trans(handle, inode, 1);
4511 	if (IS_SYNC(inode))
4512 		ext4_handle_sync(handle);
4513 out_handle:
4514 	ext4_journal_stop(handle);
4515 	return ret;
4516 }
4517 
ext4_inode_attach_jinode(struct inode * inode)4518 int ext4_inode_attach_jinode(struct inode *inode)
4519 {
4520 	struct ext4_inode_info *ei = EXT4_I(inode);
4521 	struct jbd2_inode *jinode;
4522 
4523 	if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal)
4524 		return 0;
4525 
4526 	jinode = jbd2_alloc_inode(GFP_KERNEL);
4527 	spin_lock(&inode->i_lock);
4528 	if (!ei->jinode) {
4529 		if (!jinode) {
4530 			spin_unlock(&inode->i_lock);
4531 			return -ENOMEM;
4532 		}
4533 		ei->jinode = jinode;
4534 		jbd2_journal_init_jbd_inode(ei->jinode, inode);
4535 		jinode = NULL;
4536 	}
4537 	spin_unlock(&inode->i_lock);
4538 	if (unlikely(jinode != NULL))
4539 		jbd2_free_inode(jinode);
4540 	return 0;
4541 }
4542 
4543 /*
4544  * ext4_truncate()
4545  *
4546  * We block out ext4_get_block() block instantiations across the entire
4547  * transaction, and VFS/VM ensures that ext4_truncate() cannot run
4548  * simultaneously on behalf of the same inode.
4549  *
4550  * As we work through the truncate and commit bits of it to the journal there
4551  * is one core, guiding principle: the file's tree must always be consistent on
4552  * disk.  We must be able to restart the truncate after a crash.
4553  *
4554  * The file's tree may be transiently inconsistent in memory (although it
4555  * probably isn't), but whenever we close off and commit a journal transaction,
4556  * the contents of (the filesystem + the journal) must be consistent and
4557  * restartable.  It's pretty simple, really: bottom up, right to left (although
4558  * left-to-right works OK too).
4559  *
4560  * Note that at recovery time, journal replay occurs *before* the restart of
4561  * truncate against the orphan inode list.
4562  *
4563  * The committed inode has the new, desired i_size (which is the same as
4564  * i_disksize in this case).  After a crash, ext4_orphan_cleanup() will see
4565  * that this inode's truncate did not complete and it will again call
4566  * ext4_truncate() to have another go.  So there will be instantiated blocks
4567  * to the right of the truncation point in a crashed ext4 filesystem.  But
4568  * that's fine - as long as they are linked from the inode, the post-crash
4569  * ext4_truncate() run will find them and release them.
4570  */
ext4_truncate(struct inode * inode)4571 int ext4_truncate(struct inode *inode)
4572 {
4573 	struct ext4_inode_info *ei = EXT4_I(inode);
4574 	unsigned int credits;
4575 	int err = 0, err2;
4576 	handle_t *handle;
4577 	struct address_space *mapping = inode->i_mapping;
4578 
4579 	/*
4580 	 * There is a possibility that we're either freeing the inode
4581 	 * or it's a completely new inode. In those cases we might not
4582 	 * have i_rwsem locked because it's not necessary.
4583 	 */
4584 	if (!(inode->i_state & (I_NEW|I_FREEING)))
4585 		WARN_ON(!inode_is_locked(inode));
4586 	trace_ext4_truncate_enter(inode);
4587 
4588 	if (!ext4_can_truncate(inode))
4589 		goto out_trace;
4590 
4591 	if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
4592 		ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
4593 
4594 	if (ext4_has_inline_data(inode)) {
4595 		int has_inline = 1;
4596 
4597 		err = ext4_inline_data_truncate(inode, &has_inline);
4598 		if (err || has_inline)
4599 			goto out_trace;
4600 	}
4601 
4602 	/* If we zero-out tail of the page, we have to create jinode for jbd2 */
4603 	if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {
4604 		err = ext4_inode_attach_jinode(inode);
4605 		if (err)
4606 			goto out_trace;
4607 	}
4608 
4609 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4610 		credits = ext4_chunk_trans_extent(inode, 1);
4611 	else
4612 		credits = ext4_blocks_for_truncate(inode);
4613 
4614 	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4615 	if (IS_ERR(handle)) {
4616 		err = PTR_ERR(handle);
4617 		goto out_trace;
4618 	}
4619 
4620 	if (inode->i_size & (inode->i_sb->s_blocksize - 1))
4621 		ext4_block_truncate_page(handle, mapping, inode->i_size);
4622 
4623 	/*
4624 	 * We add the inode to the orphan list, so that if this
4625 	 * truncate spans multiple transactions, and we crash, we will
4626 	 * resume the truncate when the filesystem recovers.  It also
4627 	 * marks the inode dirty, to catch the new size.
4628 	 *
4629 	 * Implication: the file must always be in a sane, consistent
4630 	 * truncatable state while each transaction commits.
4631 	 */
4632 	err = ext4_orphan_add(handle, inode);
4633 	if (err)
4634 		goto out_stop;
4635 
4636 	ext4_fc_track_inode(handle, inode);
4637 	ext4_check_map_extents_env(inode);
4638 
4639 	down_write(&EXT4_I(inode)->i_data_sem);
4640 	ext4_discard_preallocations(inode);
4641 
4642 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4643 		err = ext4_ext_truncate(handle, inode);
4644 	else
4645 		ext4_ind_truncate(handle, inode);
4646 
4647 	up_write(&ei->i_data_sem);
4648 	if (err)
4649 		goto out_stop;
4650 
4651 	if (IS_SYNC(inode))
4652 		ext4_handle_sync(handle);
4653 
4654 out_stop:
4655 	/*
4656 	 * If this was a simple ftruncate() and the file will remain alive,
4657 	 * then we need to clear up the orphan record which we created above.
4658 	 * However, if this was a real unlink then we were called by
4659 	 * ext4_evict_inode(), and we allow that function to clean up the
4660 	 * orphan info for us.
4661 	 */
4662 	if (inode->i_nlink)
4663 		ext4_orphan_del(handle, inode);
4664 
4665 	inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
4666 	err2 = ext4_mark_inode_dirty(handle, inode);
4667 	if (unlikely(err2 && !err))
4668 		err = err2;
4669 	ext4_journal_stop(handle);
4670 
4671 out_trace:
4672 	trace_ext4_truncate_exit(inode);
4673 	return err;
4674 }
4675 
ext4_inode_peek_iversion(const struct inode * inode)4676 static inline u64 ext4_inode_peek_iversion(const struct inode *inode)
4677 {
4678 	if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4679 		return inode_peek_iversion_raw(inode);
4680 	else
4681 		return inode_peek_iversion(inode);
4682 }
4683 
ext4_inode_blocks_set(struct ext4_inode * raw_inode,struct ext4_inode_info * ei)4684 static int ext4_inode_blocks_set(struct ext4_inode *raw_inode,
4685 				 struct ext4_inode_info *ei)
4686 {
4687 	struct inode *inode = &(ei->vfs_inode);
4688 	u64 i_blocks = READ_ONCE(inode->i_blocks);
4689 	struct super_block *sb = inode->i_sb;
4690 
4691 	if (i_blocks <= ~0U) {
4692 		/*
4693 		 * i_blocks can be represented in a 32 bit variable
4694 		 * as multiple of 512 bytes
4695 		 */
4696 		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
4697 		raw_inode->i_blocks_high = 0;
4698 		ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
4699 		return 0;
4700 	}
4701 
4702 	/*
4703 	 * This should never happen since sb->s_maxbytes should not have
4704 	 * allowed this, sb->s_maxbytes was set according to the huge_file
4705 	 * feature in ext4_fill_super().
4706 	 */
4707 	if (!ext4_has_feature_huge_file(sb))
4708 		return -EFSCORRUPTED;
4709 
4710 	if (i_blocks <= 0xffffffffffffULL) {
4711 		/*
4712 		 * i_blocks can be represented in a 48 bit variable
4713 		 * as multiple of 512 bytes
4714 		 */
4715 		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
4716 		raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
4717 		ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
4718 	} else {
4719 		ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE);
4720 		/* i_block is stored in file system block size */
4721 		i_blocks = i_blocks >> (inode->i_blkbits - 9);
4722 		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
4723 		raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
4724 	}
4725 	return 0;
4726 }
4727 
ext4_fill_raw_inode(struct inode * inode,struct ext4_inode * raw_inode)4728 static int ext4_fill_raw_inode(struct inode *inode, struct ext4_inode *raw_inode)
4729 {
4730 	struct ext4_inode_info *ei = EXT4_I(inode);
4731 	uid_t i_uid;
4732 	gid_t i_gid;
4733 	projid_t i_projid;
4734 	int block;
4735 	int err;
4736 
4737 	err = ext4_inode_blocks_set(raw_inode, ei);
4738 
4739 	raw_inode->i_mode = cpu_to_le16(inode->i_mode);
4740 	i_uid = i_uid_read(inode);
4741 	i_gid = i_gid_read(inode);
4742 	i_projid = from_kprojid(&init_user_ns, ei->i_projid);
4743 	if (!(test_opt(inode->i_sb, NO_UID32))) {
4744 		raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid));
4745 		raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid));
4746 		/*
4747 		 * Fix up interoperability with old kernels. Otherwise,
4748 		 * old inodes get re-used with the upper 16 bits of the
4749 		 * uid/gid intact.
4750 		 */
4751 		if (ei->i_dtime && list_empty(&ei->i_orphan)) {
4752 			raw_inode->i_uid_high = 0;
4753 			raw_inode->i_gid_high = 0;
4754 		} else {
4755 			raw_inode->i_uid_high =
4756 				cpu_to_le16(high_16_bits(i_uid));
4757 			raw_inode->i_gid_high =
4758 				cpu_to_le16(high_16_bits(i_gid));
4759 		}
4760 	} else {
4761 		raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid));
4762 		raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid));
4763 		raw_inode->i_uid_high = 0;
4764 		raw_inode->i_gid_high = 0;
4765 	}
4766 	raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);
4767 
4768 	EXT4_INODE_SET_CTIME(inode, raw_inode);
4769 	EXT4_INODE_SET_MTIME(inode, raw_inode);
4770 	EXT4_INODE_SET_ATIME(inode, raw_inode);
4771 	EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);
4772 
4773 	raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
4774 	raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);
4775 	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT)))
4776 		raw_inode->i_file_acl_high =
4777 			cpu_to_le16(ei->i_file_acl >> 32);
4778 	raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);
4779 	ext4_isize_set(raw_inode, ei->i_disksize);
4780 
4781 	raw_inode->i_generation = cpu_to_le32(inode->i_generation);
4782 	if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
4783 		if (old_valid_dev(inode->i_rdev)) {
4784 			raw_inode->i_block[0] =
4785 				cpu_to_le32(old_encode_dev(inode->i_rdev));
4786 			raw_inode->i_block[1] = 0;
4787 		} else {
4788 			raw_inode->i_block[0] = 0;
4789 			raw_inode->i_block[1] =
4790 				cpu_to_le32(new_encode_dev(inode->i_rdev));
4791 			raw_inode->i_block[2] = 0;
4792 		}
4793 	} else if (!ext4_has_inline_data(inode)) {
4794 		for (block = 0; block < EXT4_N_BLOCKS; block++)
4795 			raw_inode->i_block[block] = ei->i_data[block];
4796 	}
4797 
4798 	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
4799 		u64 ivers = ext4_inode_peek_iversion(inode);
4800 
4801 		raw_inode->i_disk_version = cpu_to_le32(ivers);
4802 		if (ei->i_extra_isize) {
4803 			if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
4804 				raw_inode->i_version_hi =
4805 					cpu_to_le32(ivers >> 32);
4806 			raw_inode->i_extra_isize =
4807 				cpu_to_le16(ei->i_extra_isize);
4808 		}
4809 	}
4810 
4811 	if (i_projid != EXT4_DEF_PROJID &&
4812 	    !ext4_has_feature_project(inode->i_sb))
4813 		err = err ?: -EFSCORRUPTED;
4814 
4815 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
4816 	    EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
4817 		raw_inode->i_projid = cpu_to_le32(i_projid);
4818 
4819 	ext4_inode_csum_set(inode, raw_inode, ei);
4820 	return err;
4821 }
4822 
4823 /*
4824  * ext4_get_inode_loc returns with an extra refcount against the inode's
4825  * underlying buffer_head on success. If we pass 'inode' and it does not
4826  * have in-inode xattr, we have all inode data in memory that is needed
4827  * to recreate the on-disk version of this inode.
4828  */
__ext4_get_inode_loc(struct super_block * sb,unsigned long ino,struct inode * inode,struct ext4_iloc * iloc,ext4_fsblk_t * ret_block)4829 static int __ext4_get_inode_loc(struct super_block *sb, unsigned long ino,
4830 				struct inode *inode, struct ext4_iloc *iloc,
4831 				ext4_fsblk_t *ret_block)
4832 {
4833 	struct ext4_group_desc	*gdp;
4834 	struct buffer_head	*bh;
4835 	ext4_fsblk_t		block;
4836 	struct blk_plug		plug;
4837 	int			inodes_per_block, inode_offset;
4838 
4839 	iloc->bh = NULL;
4840 	if (ino < EXT4_ROOT_INO ||
4841 	    ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))
4842 		return -EFSCORRUPTED;
4843 
4844 	iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb);
4845 	gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
4846 	if (!gdp)
4847 		return -EIO;
4848 
4849 	/*
4850 	 * Figure out the offset within the block group inode table
4851 	 */
4852 	inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
4853 	inode_offset = ((ino - 1) %
4854 			EXT4_INODES_PER_GROUP(sb));
4855 	iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
4856 
4857 	block = ext4_inode_table(sb, gdp);
4858 	if ((block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) ||
4859 	    (block >= ext4_blocks_count(EXT4_SB(sb)->s_es))) {
4860 		ext4_error(sb, "Invalid inode table block %llu in "
4861 			   "block_group %u", block, iloc->block_group);
4862 		return -EFSCORRUPTED;
4863 	}
4864 	block += (inode_offset / inodes_per_block);
4865 
4866 	bh = sb_getblk(sb, block);
4867 	if (unlikely(!bh))
4868 		return -ENOMEM;
4869 	if (ext4_buffer_uptodate(bh))
4870 		goto has_buffer;
4871 
4872 	lock_buffer(bh);
4873 	if (ext4_buffer_uptodate(bh)) {
4874 		/* Someone brought it uptodate while we waited */
4875 		unlock_buffer(bh);
4876 		goto has_buffer;
4877 	}
4878 
4879 	/*
4880 	 * If we have all information of the inode in memory and this
4881 	 * is the only valid inode in the block, we need not read the
4882 	 * block.
4883 	 */
4884 	if (inode && !ext4_test_inode_state(inode, EXT4_STATE_XATTR)) {
4885 		struct buffer_head *bitmap_bh;
4886 		int i, start;
4887 
4888 		start = inode_offset & ~(inodes_per_block - 1);
4889 
4890 		/* Is the inode bitmap in cache? */
4891 		bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp));
4892 		if (unlikely(!bitmap_bh))
4893 			goto make_io;
4894 
4895 		/*
4896 		 * If the inode bitmap isn't in cache then the
4897 		 * optimisation may end up performing two reads instead
4898 		 * of one, so skip it.
4899 		 */
4900 		if (!buffer_uptodate(bitmap_bh)) {
4901 			brelse(bitmap_bh);
4902 			goto make_io;
4903 		}
4904 		for (i = start; i < start + inodes_per_block; i++) {
4905 			if (i == inode_offset)
4906 				continue;
4907 			if (ext4_test_bit(i, bitmap_bh->b_data))
4908 				break;
4909 		}
4910 		brelse(bitmap_bh);
4911 		if (i == start + inodes_per_block) {
4912 			struct ext4_inode *raw_inode =
4913 				(struct ext4_inode *) (bh->b_data + iloc->offset);
4914 
4915 			/* all other inodes are free, so skip I/O */
4916 			memset(bh->b_data, 0, bh->b_size);
4917 			if (!ext4_test_inode_state(inode, EXT4_STATE_NEW))
4918 				ext4_fill_raw_inode(inode, raw_inode);
4919 			set_buffer_uptodate(bh);
4920 			unlock_buffer(bh);
4921 			goto has_buffer;
4922 		}
4923 	}
4924 
4925 make_io:
4926 	/*
4927 	 * If we need to do any I/O, try to pre-readahead extra
4928 	 * blocks from the inode table.
4929 	 */
4930 	blk_start_plug(&plug);
4931 	if (EXT4_SB(sb)->s_inode_readahead_blks) {
4932 		ext4_fsblk_t b, end, table;
4933 		unsigned num;
4934 		__u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks;
4935 
4936 		table = ext4_inode_table(sb, gdp);
4937 		/* s_inode_readahead_blks is always a power of 2 */
4938 		b = block & ~((ext4_fsblk_t) ra_blks - 1);
4939 		if (table > b)
4940 			b = table;
4941 		end = b + ra_blks;
4942 		num = EXT4_INODES_PER_GROUP(sb);
4943 		if (ext4_has_group_desc_csum(sb))
4944 			num -= ext4_itable_unused_count(sb, gdp);
4945 		table += num / inodes_per_block;
4946 		if (end > table)
4947 			end = table;
4948 		while (b <= end)
4949 			ext4_sb_breadahead_unmovable(sb, b++);
4950 	}
4951 
4952 	/*
4953 	 * There are other valid inodes in the buffer, this inode
4954 	 * has in-inode xattrs, or we don't have this inode in memory.
4955 	 * Read the block from disk.
4956 	 */
4957 	trace_ext4_load_inode(sb, ino);
4958 	ext4_read_bh_nowait(bh, REQ_META | REQ_PRIO, NULL,
4959 			    ext4_simulate_fail(sb, EXT4_SIM_INODE_EIO));
4960 	blk_finish_plug(&plug);
4961 	wait_on_buffer(bh);
4962 	if (!buffer_uptodate(bh)) {
4963 		if (ret_block)
4964 			*ret_block = block;
4965 		brelse(bh);
4966 		return -EIO;
4967 	}
4968 has_buffer:
4969 	iloc->bh = bh;
4970 	return 0;
4971 }
4972 
__ext4_get_inode_loc_noinmem(struct inode * inode,struct ext4_iloc * iloc)4973 static int __ext4_get_inode_loc_noinmem(struct inode *inode,
4974 					struct ext4_iloc *iloc)
4975 {
4976 	ext4_fsblk_t err_blk = 0;
4977 	int ret;
4978 
4979 	ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, NULL, iloc,
4980 					&err_blk);
4981 
4982 	if (ret == -EIO)
4983 		ext4_error_inode_block(inode, err_blk, EIO,
4984 					"unable to read itable block");
4985 
4986 	return ret;
4987 }
4988 
ext4_get_inode_loc(struct inode * inode,struct ext4_iloc * iloc)4989 int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
4990 {
4991 	ext4_fsblk_t err_blk = 0;
4992 	int ret;
4993 
4994 	ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, inode, iloc,
4995 					&err_blk);
4996 
4997 	if (ret == -EIO)
4998 		ext4_error_inode_block(inode, err_blk, EIO,
4999 					"unable to read itable block");
5000 
5001 	return ret;
5002 }
5003 
5004 
ext4_get_fc_inode_loc(struct super_block * sb,unsigned long ino,struct ext4_iloc * iloc)5005 int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino,
5006 			  struct ext4_iloc *iloc)
5007 {
5008 	return __ext4_get_inode_loc(sb, ino, NULL, iloc, NULL);
5009 }
5010 
ext4_should_enable_dax(struct inode * inode)5011 static bool ext4_should_enable_dax(struct inode *inode)
5012 {
5013 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5014 
5015 	if (test_opt2(inode->i_sb, DAX_NEVER))
5016 		return false;
5017 	if (!S_ISREG(inode->i_mode))
5018 		return false;
5019 	if (ext4_should_journal_data(inode))
5020 		return false;
5021 	if (ext4_has_inline_data(inode))
5022 		return false;
5023 	if (ext4_test_inode_flag(inode, EXT4_INODE_ENCRYPT))
5024 		return false;
5025 	if (ext4_test_inode_flag(inode, EXT4_INODE_VERITY))
5026 		return false;
5027 	if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags))
5028 		return false;
5029 	if (test_opt(inode->i_sb, DAX_ALWAYS))
5030 		return true;
5031 
5032 	return ext4_test_inode_flag(inode, EXT4_INODE_DAX);
5033 }
5034 
ext4_set_inode_flags(struct inode * inode,bool init)5035 void ext4_set_inode_flags(struct inode *inode, bool init)
5036 {
5037 	unsigned int flags = EXT4_I(inode)->i_flags;
5038 	unsigned int new_fl = 0;
5039 
5040 	WARN_ON_ONCE(IS_DAX(inode) && init);
5041 
5042 	if (flags & EXT4_SYNC_FL)
5043 		new_fl |= S_SYNC;
5044 	if (flags & EXT4_APPEND_FL)
5045 		new_fl |= S_APPEND;
5046 	if (flags & EXT4_IMMUTABLE_FL)
5047 		new_fl |= S_IMMUTABLE;
5048 	if (flags & EXT4_NOATIME_FL)
5049 		new_fl |= S_NOATIME;
5050 	if (flags & EXT4_DIRSYNC_FL)
5051 		new_fl |= S_DIRSYNC;
5052 
5053 	/* Because of the way inode_set_flags() works we must preserve S_DAX
5054 	 * here if already set. */
5055 	new_fl |= (inode->i_flags & S_DAX);
5056 	if (init && ext4_should_enable_dax(inode))
5057 		new_fl |= S_DAX;
5058 
5059 	if (flags & EXT4_ENCRYPT_FL)
5060 		new_fl |= S_ENCRYPTED;
5061 	if (flags & EXT4_CASEFOLD_FL)
5062 		new_fl |= S_CASEFOLD;
5063 	if (flags & EXT4_VERITY_FL)
5064 		new_fl |= S_VERITY;
5065 	inode_set_flags(inode, new_fl,
5066 			S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX|
5067 			S_ENCRYPTED|S_CASEFOLD|S_VERITY);
5068 }
5069 
ext4_inode_blocks(struct ext4_inode * raw_inode,struct ext4_inode_info * ei)5070 static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,
5071 				  struct ext4_inode_info *ei)
5072 {
5073 	blkcnt_t i_blocks ;
5074 	struct inode *inode = &(ei->vfs_inode);
5075 	struct super_block *sb = inode->i_sb;
5076 
5077 	if (ext4_has_feature_huge_file(sb)) {
5078 		/* we are using combined 48 bit field */
5079 		i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |
5080 					le32_to_cpu(raw_inode->i_blocks_lo);
5081 		if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) {
5082 			/* i_blocks represent file system block size */
5083 			return i_blocks  << (inode->i_blkbits - 9);
5084 		} else {
5085 			return i_blocks;
5086 		}
5087 	} else {
5088 		return le32_to_cpu(raw_inode->i_blocks_lo);
5089 	}
5090 }
5091 
ext4_iget_extra_inode(struct inode * inode,struct ext4_inode * raw_inode,struct ext4_inode_info * ei)5092 static inline int ext4_iget_extra_inode(struct inode *inode,
5093 					 struct ext4_inode *raw_inode,
5094 					 struct ext4_inode_info *ei)
5095 {
5096 	__le32 *magic = (void *)raw_inode +
5097 			EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;
5098 
5099 	if (EXT4_INODE_HAS_XATTR_SPACE(inode)  &&
5100 	    *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
5101 		int err;
5102 
5103 		err = xattr_check_inode(inode, IHDR(inode, raw_inode),
5104 					ITAIL(inode, raw_inode));
5105 		if (err)
5106 			return err;
5107 
5108 		ext4_set_inode_state(inode, EXT4_STATE_XATTR);
5109 		err = ext4_find_inline_data_nolock(inode);
5110 		if (!err && ext4_has_inline_data(inode))
5111 			ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
5112 		return err;
5113 	} else
5114 		EXT4_I(inode)->i_inline_off = 0;
5115 	return 0;
5116 }
5117 
ext4_get_projid(struct inode * inode,kprojid_t * projid)5118 int ext4_get_projid(struct inode *inode, kprojid_t *projid)
5119 {
5120 	if (!ext4_has_feature_project(inode->i_sb))
5121 		return -EOPNOTSUPP;
5122 	*projid = EXT4_I(inode)->i_projid;
5123 	return 0;
5124 }
5125 
5126 /*
5127  * ext4 has self-managed i_version for ea inodes, it stores the lower 32bit of
5128  * refcount in i_version, so use raw values if inode has EXT4_EA_INODE_FL flag
5129  * set.
5130  */
ext4_inode_set_iversion_queried(struct inode * inode,u64 val)5131 static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val)
5132 {
5133 	if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
5134 		inode_set_iversion_raw(inode, val);
5135 	else
5136 		inode_set_iversion_queried(inode, val);
5137 }
5138 
check_igot_inode(struct inode * inode,ext4_iget_flags flags,const char * function,unsigned int line)5139 static int check_igot_inode(struct inode *inode, ext4_iget_flags flags,
5140 			    const char *function, unsigned int line)
5141 {
5142 	const char *err_str;
5143 
5144 	if (flags & EXT4_IGET_EA_INODE) {
5145 		if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
5146 			err_str = "missing EA_INODE flag";
5147 			goto error;
5148 		}
5149 		if (ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
5150 		    EXT4_I(inode)->i_file_acl) {
5151 			err_str = "ea_inode with extended attributes";
5152 			goto error;
5153 		}
5154 	} else {
5155 		if ((EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
5156 			/*
5157 			 * open_by_handle_at() could provide an old inode number
5158 			 * that has since been reused for an ea_inode; this does
5159 			 * not indicate filesystem corruption
5160 			 */
5161 			if (flags & EXT4_IGET_HANDLE)
5162 				return -ESTALE;
5163 			err_str = "unexpected EA_INODE flag";
5164 			goto error;
5165 		}
5166 	}
5167 	if (is_bad_inode(inode) && !(flags & EXT4_IGET_BAD)) {
5168 		err_str = "unexpected bad inode w/o EXT4_IGET_BAD";
5169 		goto error;
5170 	}
5171 	return 0;
5172 
5173 error:
5174 	ext4_error_inode(inode, function, line, 0, "%s", err_str);
5175 	return -EFSCORRUPTED;
5176 }
5177 
ext4_should_enable_large_folio(struct inode * inode)5178 static bool ext4_should_enable_large_folio(struct inode *inode)
5179 {
5180 	struct super_block *sb = inode->i_sb;
5181 
5182 	if (!S_ISREG(inode->i_mode))
5183 		return false;
5184 	if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA ||
5185 	    ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA))
5186 		return false;
5187 	if (ext4_has_feature_verity(sb))
5188 		return false;
5189 	if (ext4_has_feature_encrypt(sb))
5190 		return false;
5191 
5192 	return true;
5193 }
5194 
5195 /*
5196  * Limit the maximum folio order to 2048 blocks to prevent overestimation
5197  * of reserve handle credits during the folio writeback in environments
5198  * where the PAGE_SIZE exceeds 4KB.
5199  */
5200 #define EXT4_MAX_PAGECACHE_ORDER(i)		\
5201 		umin(MAX_PAGECACHE_ORDER, (11 + (i)->i_blkbits - PAGE_SHIFT))
ext4_set_inode_mapping_order(struct inode * inode)5202 void ext4_set_inode_mapping_order(struct inode *inode)
5203 {
5204 	if (!ext4_should_enable_large_folio(inode))
5205 		return;
5206 
5207 	mapping_set_folio_order_range(inode->i_mapping, 0,
5208 				      EXT4_MAX_PAGECACHE_ORDER(inode));
5209 }
5210 
__ext4_iget(struct super_block * sb,unsigned long ino,ext4_iget_flags flags,const char * function,unsigned int line)5211 struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
5212 			  ext4_iget_flags flags, const char *function,
5213 			  unsigned int line)
5214 {
5215 	struct ext4_iloc iloc;
5216 	struct ext4_inode *raw_inode;
5217 	struct ext4_inode_info *ei;
5218 	struct ext4_super_block *es = EXT4_SB(sb)->s_es;
5219 	struct inode *inode;
5220 	journal_t *journal = EXT4_SB(sb)->s_journal;
5221 	long ret;
5222 	loff_t size;
5223 	int block;
5224 	uid_t i_uid;
5225 	gid_t i_gid;
5226 	projid_t i_projid;
5227 
5228 	if ((!(flags & EXT4_IGET_SPECIAL) && is_special_ino(sb, ino)) ||
5229 	    (ino < EXT4_ROOT_INO) ||
5230 	    (ino > le32_to_cpu(es->s_inodes_count))) {
5231 		if (flags & EXT4_IGET_HANDLE)
5232 			return ERR_PTR(-ESTALE);
5233 		__ext4_error(sb, function, line, false, EFSCORRUPTED, 0,
5234 			     "inode #%lu: comm %s: iget: illegal inode #",
5235 			     ino, current->comm);
5236 		return ERR_PTR(-EFSCORRUPTED);
5237 	}
5238 
5239 	inode = iget_locked(sb, ino);
5240 	if (!inode)
5241 		return ERR_PTR(-ENOMEM);
5242 	if (!(inode->i_state & I_NEW)) {
5243 		ret = check_igot_inode(inode, flags, function, line);
5244 		if (ret) {
5245 			iput(inode);
5246 			return ERR_PTR(ret);
5247 		}
5248 		return inode;
5249 	}
5250 
5251 	ei = EXT4_I(inode);
5252 	iloc.bh = NULL;
5253 
5254 	ret = __ext4_get_inode_loc_noinmem(inode, &iloc);
5255 	if (ret < 0)
5256 		goto bad_inode;
5257 	raw_inode = ext4_raw_inode(&iloc);
5258 
5259 	if ((flags & EXT4_IGET_HANDLE) &&
5260 	    (raw_inode->i_links_count == 0) && (raw_inode->i_mode == 0)) {
5261 		ret = -ESTALE;
5262 		goto bad_inode;
5263 	}
5264 
5265 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
5266 		ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
5267 		if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
5268 			EXT4_INODE_SIZE(inode->i_sb) ||
5269 		    (ei->i_extra_isize & 3)) {
5270 			ext4_error_inode(inode, function, line, 0,
5271 					 "iget: bad extra_isize %u "
5272 					 "(inode size %u)",
5273 					 ei->i_extra_isize,
5274 					 EXT4_INODE_SIZE(inode->i_sb));
5275 			ret = -EFSCORRUPTED;
5276 			goto bad_inode;
5277 		}
5278 	} else
5279 		ei->i_extra_isize = 0;
5280 
5281 	/* Precompute checksum seed for inode metadata */
5282 	if (ext4_has_feature_metadata_csum(sb)) {
5283 		struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5284 		__u32 csum;
5285 		__le32 inum = cpu_to_le32(inode->i_ino);
5286 		__le32 gen = raw_inode->i_generation;
5287 		csum = ext4_chksum(sbi->s_csum_seed, (__u8 *)&inum,
5288 				   sizeof(inum));
5289 		ei->i_csum_seed = ext4_chksum(csum, (__u8 *)&gen, sizeof(gen));
5290 	}
5291 
5292 	if ((!ext4_inode_csum_verify(inode, raw_inode, ei) ||
5293 	    ext4_simulate_fail(sb, EXT4_SIM_INODE_CRC)) &&
5294 	     (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))) {
5295 		ext4_error_inode_err(inode, function, line, 0,
5296 				EFSBADCRC, "iget: checksum invalid");
5297 		ret = -EFSBADCRC;
5298 		goto bad_inode;
5299 	}
5300 
5301 	inode->i_mode = le16_to_cpu(raw_inode->i_mode);
5302 	i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);
5303 	i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);
5304 	if (ext4_has_feature_project(sb) &&
5305 	    EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE &&
5306 	    EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
5307 		i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid);
5308 	else
5309 		i_projid = EXT4_DEF_PROJID;
5310 
5311 	if (!(test_opt(inode->i_sb, NO_UID32))) {
5312 		i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;
5313 		i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;
5314 	}
5315 	i_uid_write(inode, i_uid);
5316 	i_gid_write(inode, i_gid);
5317 	ei->i_projid = make_kprojid(&init_user_ns, i_projid);
5318 	set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));
5319 
5320 	ext4_clear_state_flags(ei);	/* Only relevant on 32-bit archs */
5321 	ei->i_inline_off = 0;
5322 	ei->i_dir_start_lookup = 0;
5323 	ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
5324 	/* We now have enough fields to check if the inode was active or not.
5325 	 * This is needed because nfsd might try to access dead inodes
5326 	 * the test is that same one that e2fsck uses
5327 	 * NeilBrown 1999oct15
5328 	 */
5329 	if (inode->i_nlink == 0) {
5330 		if ((inode->i_mode == 0 || flags & EXT4_IGET_SPECIAL ||
5331 		     !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) &&
5332 		    ino != EXT4_BOOT_LOADER_INO) {
5333 			/* this inode is deleted or unallocated */
5334 			if (flags & EXT4_IGET_SPECIAL) {
5335 				ext4_error_inode(inode, function, line, 0,
5336 						 "iget: special inode unallocated");
5337 				ret = -EFSCORRUPTED;
5338 			} else
5339 				ret = -ESTALE;
5340 			goto bad_inode;
5341 		}
5342 		/* The only unlinked inodes we let through here have
5343 		 * valid i_mode and are being read by the orphan
5344 		 * recovery code: that's fine, we're about to complete
5345 		 * the process of deleting those.
5346 		 * OR it is the EXT4_BOOT_LOADER_INO which is
5347 		 * not initialized on a new filesystem. */
5348 	}
5349 	ei->i_flags = le32_to_cpu(raw_inode->i_flags);
5350 	ext4_set_inode_flags(inode, true);
5351 	inode->i_blocks = ext4_inode_blocks(raw_inode, ei);
5352 	ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);
5353 	if (ext4_has_feature_64bit(sb))
5354 		ei->i_file_acl |=
5355 			((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
5356 	inode->i_size = ext4_isize(sb, raw_inode);
5357 	size = i_size_read(inode);
5358 	if (size < 0 || size > ext4_get_maxbytes(inode)) {
5359 		ext4_error_inode(inode, function, line, 0,
5360 				 "iget: bad i_size value: %lld", size);
5361 		ret = -EFSCORRUPTED;
5362 		goto bad_inode;
5363 	}
5364 	/*
5365 	 * If dir_index is not enabled but there's dir with INDEX flag set,
5366 	 * we'd normally treat htree data as empty space. But with metadata
5367 	 * checksumming that corrupts checksums so forbid that.
5368 	 */
5369 	if (!ext4_has_feature_dir_index(sb) &&
5370 	    ext4_has_feature_metadata_csum(sb) &&
5371 	    ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) {
5372 		ext4_error_inode(inode, function, line, 0,
5373 			 "iget: Dir with htree data on filesystem without dir_index feature.");
5374 		ret = -EFSCORRUPTED;
5375 		goto bad_inode;
5376 	}
5377 	ei->i_disksize = inode->i_size;
5378 #ifdef CONFIG_QUOTA
5379 	ei->i_reserved_quota = 0;
5380 #endif
5381 	inode->i_generation = le32_to_cpu(raw_inode->i_generation);
5382 	ei->i_block_group = iloc.block_group;
5383 	ei->i_last_alloc_group = ~0;
5384 	/*
5385 	 * NOTE! The in-memory inode i_data array is in little-endian order
5386 	 * even on big-endian machines: we do NOT byteswap the block numbers!
5387 	 */
5388 	for (block = 0; block < EXT4_N_BLOCKS; block++)
5389 		ei->i_data[block] = raw_inode->i_block[block];
5390 	INIT_LIST_HEAD(&ei->i_orphan);
5391 	ext4_fc_init_inode(&ei->vfs_inode);
5392 
5393 	/*
5394 	 * Set transaction id's of transactions that have to be committed
5395 	 * to finish f[data]sync. We set them to currently running transaction
5396 	 * as we cannot be sure that the inode or some of its metadata isn't
5397 	 * part of the transaction - the inode could have been reclaimed and
5398 	 * now it is reread from disk.
5399 	 */
5400 	if (journal) {
5401 		transaction_t *transaction;
5402 		tid_t tid;
5403 
5404 		read_lock(&journal->j_state_lock);
5405 		if (journal->j_running_transaction)
5406 			transaction = journal->j_running_transaction;
5407 		else
5408 			transaction = journal->j_committing_transaction;
5409 		if (transaction)
5410 			tid = transaction->t_tid;
5411 		else
5412 			tid = journal->j_commit_sequence;
5413 		read_unlock(&journal->j_state_lock);
5414 		ei->i_sync_tid = tid;
5415 		ei->i_datasync_tid = tid;
5416 	}
5417 
5418 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
5419 		if (ei->i_extra_isize == 0) {
5420 			/* The extra space is currently unused. Use it. */
5421 			BUILD_BUG_ON(sizeof(struct ext4_inode) & 3);
5422 			ei->i_extra_isize = sizeof(struct ext4_inode) -
5423 					    EXT4_GOOD_OLD_INODE_SIZE;
5424 		} else {
5425 			ret = ext4_iget_extra_inode(inode, raw_inode, ei);
5426 			if (ret)
5427 				goto bad_inode;
5428 		}
5429 	}
5430 
5431 	EXT4_INODE_GET_CTIME(inode, raw_inode);
5432 	EXT4_INODE_GET_ATIME(inode, raw_inode);
5433 	EXT4_INODE_GET_MTIME(inode, raw_inode);
5434 	EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);
5435 
5436 	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
5437 		u64 ivers = le32_to_cpu(raw_inode->i_disk_version);
5438 
5439 		if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
5440 			if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
5441 				ivers |=
5442 		    (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;
5443 		}
5444 		ext4_inode_set_iversion_queried(inode, ivers);
5445 	}
5446 
5447 	ret = 0;
5448 	if (ei->i_file_acl &&
5449 	    !ext4_inode_block_valid(inode, ei->i_file_acl, 1)) {
5450 		ext4_error_inode(inode, function, line, 0,
5451 				 "iget: bad extended attribute block %llu",
5452 				 ei->i_file_acl);
5453 		ret = -EFSCORRUPTED;
5454 		goto bad_inode;
5455 	} else if (!ext4_has_inline_data(inode)) {
5456 		/* validate the block references in the inode */
5457 		if (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) &&
5458 			(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
5459 			(S_ISLNK(inode->i_mode) &&
5460 			!ext4_inode_is_fast_symlink(inode)))) {
5461 			if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5462 				ret = ext4_ext_check_inode(inode);
5463 			else
5464 				ret = ext4_ind_check_inode(inode);
5465 		}
5466 	}
5467 	if (ret)
5468 		goto bad_inode;
5469 
5470 	if (S_ISREG(inode->i_mode)) {
5471 		inode->i_op = &ext4_file_inode_operations;
5472 		inode->i_fop = &ext4_file_operations;
5473 		ext4_set_aops(inode);
5474 	} else if (S_ISDIR(inode->i_mode)) {
5475 		inode->i_op = &ext4_dir_inode_operations;
5476 		inode->i_fop = &ext4_dir_operations;
5477 	} else if (S_ISLNK(inode->i_mode)) {
5478 		/* VFS does not allow setting these so must be corruption */
5479 		if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) {
5480 			ext4_error_inode(inode, function, line, 0,
5481 					 "iget: immutable or append flags "
5482 					 "not allowed on symlinks");
5483 			ret = -EFSCORRUPTED;
5484 			goto bad_inode;
5485 		}
5486 		if (IS_ENCRYPTED(inode)) {
5487 			inode->i_op = &ext4_encrypted_symlink_inode_operations;
5488 		} else if (ext4_inode_is_fast_symlink(inode)) {
5489 			inode->i_op = &ext4_fast_symlink_inode_operations;
5490 			if (inode->i_size == 0 ||
5491 			    inode->i_size >= sizeof(ei->i_data) ||
5492 			    strnlen((char *)ei->i_data, inode->i_size + 1) !=
5493 								inode->i_size) {
5494 				ext4_error_inode(inode, function, line, 0,
5495 					"invalid fast symlink length %llu",
5496 					 (unsigned long long)inode->i_size);
5497 				ret = -EFSCORRUPTED;
5498 				goto bad_inode;
5499 			}
5500 			inode_set_cached_link(inode, (char *)ei->i_data,
5501 					      inode->i_size);
5502 		} else {
5503 			inode->i_op = &ext4_symlink_inode_operations;
5504 		}
5505 	} else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
5506 	      S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
5507 		inode->i_op = &ext4_special_inode_operations;
5508 		if (raw_inode->i_block[0])
5509 			init_special_inode(inode, inode->i_mode,
5510 			   old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));
5511 		else
5512 			init_special_inode(inode, inode->i_mode,
5513 			   new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
5514 	} else if (ino == EXT4_BOOT_LOADER_INO) {
5515 		make_bad_inode(inode);
5516 	} else {
5517 		ret = -EFSCORRUPTED;
5518 		ext4_error_inode(inode, function, line, 0,
5519 				 "iget: bogus i_mode (%o)", inode->i_mode);
5520 		goto bad_inode;
5521 	}
5522 	if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(inode->i_sb)) {
5523 		ext4_error_inode(inode, function, line, 0,
5524 				 "casefold flag without casefold feature");
5525 		ret = -EFSCORRUPTED;
5526 		goto bad_inode;
5527 	}
5528 
5529 	ext4_set_inode_mapping_order(inode);
5530 
5531 	ret = check_igot_inode(inode, flags, function, line);
5532 	/*
5533 	 * -ESTALE here means there is nothing inherently wrong with the inode,
5534 	 * it's just not an inode we can return for an fhandle lookup.
5535 	 */
5536 	if (ret == -ESTALE) {
5537 		brelse(iloc.bh);
5538 		unlock_new_inode(inode);
5539 		iput(inode);
5540 		return ERR_PTR(-ESTALE);
5541 	}
5542 	if (ret)
5543 		goto bad_inode;
5544 	brelse(iloc.bh);
5545 
5546 	unlock_new_inode(inode);
5547 	return inode;
5548 
5549 bad_inode:
5550 	brelse(iloc.bh);
5551 	iget_failed(inode);
5552 	return ERR_PTR(ret);
5553 }
5554 
__ext4_update_other_inode_time(struct super_block * sb,unsigned long orig_ino,unsigned long ino,struct ext4_inode * raw_inode)5555 static void __ext4_update_other_inode_time(struct super_block *sb,
5556 					   unsigned long orig_ino,
5557 					   unsigned long ino,
5558 					   struct ext4_inode *raw_inode)
5559 {
5560 	struct inode *inode;
5561 
5562 	inode = find_inode_by_ino_rcu(sb, ino);
5563 	if (!inode)
5564 		return;
5565 
5566 	if (!inode_is_dirtytime_only(inode))
5567 		return;
5568 
5569 	spin_lock(&inode->i_lock);
5570 	if (inode_is_dirtytime_only(inode)) {
5571 		struct ext4_inode_info	*ei = EXT4_I(inode);
5572 
5573 		inode->i_state &= ~I_DIRTY_TIME;
5574 		spin_unlock(&inode->i_lock);
5575 
5576 		spin_lock(&ei->i_raw_lock);
5577 		EXT4_INODE_SET_CTIME(inode, raw_inode);
5578 		EXT4_INODE_SET_MTIME(inode, raw_inode);
5579 		EXT4_INODE_SET_ATIME(inode, raw_inode);
5580 		ext4_inode_csum_set(inode, raw_inode, ei);
5581 		spin_unlock(&ei->i_raw_lock);
5582 		trace_ext4_other_inode_update_time(inode, orig_ino);
5583 		return;
5584 	}
5585 	spin_unlock(&inode->i_lock);
5586 }
5587 
5588 /*
5589  * Opportunistically update the other time fields for other inodes in
5590  * the same inode table block.
5591  */
ext4_update_other_inodes_time(struct super_block * sb,unsigned long orig_ino,char * buf)5592 static void ext4_update_other_inodes_time(struct super_block *sb,
5593 					  unsigned long orig_ino, char *buf)
5594 {
5595 	unsigned long ino;
5596 	int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
5597 	int inode_size = EXT4_INODE_SIZE(sb);
5598 
5599 	/*
5600 	 * Calculate the first inode in the inode table block.  Inode
5601 	 * numbers are one-based.  That is, the first inode in a block
5602 	 * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1).
5603 	 */
5604 	ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1;
5605 	rcu_read_lock();
5606 	for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) {
5607 		if (ino == orig_ino)
5608 			continue;
5609 		__ext4_update_other_inode_time(sb, orig_ino, ino,
5610 					       (struct ext4_inode *)buf);
5611 	}
5612 	rcu_read_unlock();
5613 }
5614 
5615 /*
5616  * Post the struct inode info into an on-disk inode location in the
5617  * buffer-cache.  This gobbles the caller's reference to the
5618  * buffer_head in the inode location struct.
5619  *
5620  * The caller must have write access to iloc->bh.
5621  */
ext4_do_update_inode(handle_t * handle,struct inode * inode,struct ext4_iloc * iloc)5622 static int ext4_do_update_inode(handle_t *handle,
5623 				struct inode *inode,
5624 				struct ext4_iloc *iloc)
5625 {
5626 	struct ext4_inode *raw_inode = ext4_raw_inode(iloc);
5627 	struct ext4_inode_info *ei = EXT4_I(inode);
5628 	struct buffer_head *bh = iloc->bh;
5629 	struct super_block *sb = inode->i_sb;
5630 	int err;
5631 	int need_datasync = 0, set_large_file = 0;
5632 
5633 	spin_lock(&ei->i_raw_lock);
5634 
5635 	/*
5636 	 * For fields not tracked in the in-memory inode, initialise them
5637 	 * to zero for new inodes.
5638 	 */
5639 	if (ext4_test_inode_state(inode, EXT4_STATE_NEW))
5640 		memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
5641 
5642 	if (READ_ONCE(ei->i_disksize) != ext4_isize(inode->i_sb, raw_inode))
5643 		need_datasync = 1;
5644 	if (ei->i_disksize > 0x7fffffffULL) {
5645 		if (!ext4_has_feature_large_file(sb) ||
5646 		    EXT4_SB(sb)->s_es->s_rev_level == cpu_to_le32(EXT4_GOOD_OLD_REV))
5647 			set_large_file = 1;
5648 	}
5649 
5650 	err = ext4_fill_raw_inode(inode, raw_inode);
5651 	spin_unlock(&ei->i_raw_lock);
5652 	if (err) {
5653 		EXT4_ERROR_INODE(inode, "corrupted inode contents");
5654 		goto out_brelse;
5655 	}
5656 
5657 	if (inode->i_sb->s_flags & SB_LAZYTIME)
5658 		ext4_update_other_inodes_time(inode->i_sb, inode->i_ino,
5659 					      bh->b_data);
5660 
5661 	BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
5662 	err = ext4_handle_dirty_metadata(handle, NULL, bh);
5663 	if (err)
5664 		goto out_error;
5665 	ext4_clear_inode_state(inode, EXT4_STATE_NEW);
5666 	if (set_large_file) {
5667 		BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");
5668 		err = ext4_journal_get_write_access(handle, sb,
5669 						    EXT4_SB(sb)->s_sbh,
5670 						    EXT4_JTR_NONE);
5671 		if (err)
5672 			goto out_error;
5673 		lock_buffer(EXT4_SB(sb)->s_sbh);
5674 		ext4_set_feature_large_file(sb);
5675 		ext4_superblock_csum_set(sb);
5676 		unlock_buffer(EXT4_SB(sb)->s_sbh);
5677 		ext4_handle_sync(handle);
5678 		err = ext4_handle_dirty_metadata(handle, NULL,
5679 						 EXT4_SB(sb)->s_sbh);
5680 	}
5681 	ext4_update_inode_fsync_trans(handle, inode, need_datasync);
5682 out_error:
5683 	ext4_std_error(inode->i_sb, err);
5684 out_brelse:
5685 	brelse(bh);
5686 	return err;
5687 }
5688 
5689 /*
5690  * ext4_write_inode()
5691  *
5692  * We are called from a few places:
5693  *
5694  * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files.
5695  *   Here, there will be no transaction running. We wait for any running
5696  *   transaction to commit.
5697  *
5698  * - Within flush work (sys_sync(), kupdate and such).
5699  *   We wait on commit, if told to.
5700  *
5701  * - Within iput_final() -> write_inode_now()
5702  *   We wait on commit, if told to.
5703  *
5704  * In all cases it is actually safe for us to return without doing anything,
5705  * because the inode has been copied into a raw inode buffer in
5706  * ext4_mark_inode_dirty().  This is a correctness thing for WB_SYNC_ALL
5707  * writeback.
5708  *
5709  * Note that we are absolutely dependent upon all inode dirtiers doing the
5710  * right thing: they *must* call mark_inode_dirty() after dirtying info in
5711  * which we are interested.
5712  *
5713  * It would be a bug for them to not do this.  The code:
5714  *
5715  *	mark_inode_dirty(inode)
5716  *	stuff();
5717  *	inode->i_size = expr;
5718  *
5719  * is in error because write_inode() could occur while `stuff()' is running,
5720  * and the new i_size will be lost.  Plus the inode will no longer be on the
5721  * superblock's dirty inode list.
5722  */
ext4_write_inode(struct inode * inode,struct writeback_control * wbc)5723 int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
5724 {
5725 	int err;
5726 
5727 	if (WARN_ON_ONCE(current->flags & PF_MEMALLOC))
5728 		return 0;
5729 
5730 	err = ext4_emergency_state(inode->i_sb);
5731 	if (unlikely(err))
5732 		return err;
5733 
5734 	if (EXT4_SB(inode->i_sb)->s_journal) {
5735 		if (ext4_journal_current_handle()) {
5736 			ext4_debug("called recursively, non-PF_MEMALLOC!\n");
5737 			dump_stack();
5738 			return -EIO;
5739 		}
5740 
5741 		/*
5742 		 * No need to force transaction in WB_SYNC_NONE mode. Also
5743 		 * ext4_sync_fs() will force the commit after everything is
5744 		 * written.
5745 		 */
5746 		if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync)
5747 			return 0;
5748 
5749 		err = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal,
5750 						EXT4_I(inode)->i_sync_tid);
5751 	} else {
5752 		struct ext4_iloc iloc;
5753 
5754 		err = __ext4_get_inode_loc_noinmem(inode, &iloc);
5755 		if (err)
5756 			return err;
5757 		/*
5758 		 * sync(2) will flush the whole buffer cache. No need to do
5759 		 * it here separately for each inode.
5760 		 */
5761 		if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
5762 			sync_dirty_buffer(iloc.bh);
5763 		if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) {
5764 			ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO,
5765 					       "IO error syncing inode");
5766 			err = -EIO;
5767 		}
5768 		brelse(iloc.bh);
5769 	}
5770 	return err;
5771 }
5772 
5773 /*
5774  * In data=journal mode ext4_journalled_invalidate_folio() may fail to invalidate
5775  * buffers that are attached to a folio straddling i_size and are undergoing
5776  * commit. In that case we have to wait for commit to finish and try again.
5777  */
ext4_wait_for_tail_page_commit(struct inode * inode)5778 static void ext4_wait_for_tail_page_commit(struct inode *inode)
5779 {
5780 	unsigned offset;
5781 	journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
5782 	tid_t commit_tid;
5783 	int ret;
5784 	bool has_transaction;
5785 
5786 	offset = inode->i_size & (PAGE_SIZE - 1);
5787 	/*
5788 	 * If the folio is fully truncated, we don't need to wait for any commit
5789 	 * (and we even should not as __ext4_journalled_invalidate_folio() may
5790 	 * strip all buffers from the folio but keep the folio dirty which can then
5791 	 * confuse e.g. concurrent ext4_writepages() seeing dirty folio without
5792 	 * buffers). Also we don't need to wait for any commit if all buffers in
5793 	 * the folio remain valid. This is most beneficial for the common case of
5794 	 * blocksize == PAGESIZE.
5795 	 */
5796 	if (!offset || offset > (PAGE_SIZE - i_blocksize(inode)))
5797 		return;
5798 	while (1) {
5799 		struct folio *folio = filemap_lock_folio(inode->i_mapping,
5800 				      inode->i_size >> PAGE_SHIFT);
5801 		if (IS_ERR(folio))
5802 			return;
5803 		ret = __ext4_journalled_invalidate_folio(folio, offset,
5804 						folio_size(folio) - offset);
5805 		folio_unlock(folio);
5806 		folio_put(folio);
5807 		if (ret != -EBUSY)
5808 			return;
5809 		has_transaction = false;
5810 		read_lock(&journal->j_state_lock);
5811 		if (journal->j_committing_transaction) {
5812 			commit_tid = journal->j_committing_transaction->t_tid;
5813 			has_transaction = true;
5814 		}
5815 		read_unlock(&journal->j_state_lock);
5816 		if (has_transaction)
5817 			jbd2_log_wait_commit(journal, commit_tid);
5818 	}
5819 }
5820 
5821 /*
5822  * ext4_setattr()
5823  *
5824  * Called from notify_change.
5825  *
5826  * We want to trap VFS attempts to truncate the file as soon as
5827  * possible.  In particular, we want to make sure that when the VFS
5828  * shrinks i_size, we put the inode on the orphan list and modify
5829  * i_disksize immediately, so that during the subsequent flushing of
5830  * dirty pages and freeing of disk blocks, we can guarantee that any
5831  * commit will leave the blocks being flushed in an unused state on
5832  * disk.  (On recovery, the inode will get truncated and the blocks will
5833  * be freed, so we have a strong guarantee that no future commit will
5834  * leave these blocks visible to the user.)
5835  *
5836  * Another thing we have to assure is that if we are in ordered mode
5837  * and inode is still attached to the committing transaction, we must
5838  * we start writeout of all the dirty pages which are being truncated.
5839  * This way we are sure that all the data written in the previous
5840  * transaction are already on disk (truncate waits for pages under
5841  * writeback).
5842  *
5843  * Called with inode->i_rwsem down.
5844  */
ext4_setattr(struct mnt_idmap * idmap,struct dentry * dentry,struct iattr * attr)5845 int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
5846 		 struct iattr *attr)
5847 {
5848 	struct inode *inode = d_inode(dentry);
5849 	int error, rc = 0;
5850 	int orphan = 0;
5851 	const unsigned int ia_valid = attr->ia_valid;
5852 	bool inc_ivers = true;
5853 
5854 	error = ext4_emergency_state(inode->i_sb);
5855 	if (unlikely(error))
5856 		return error;
5857 
5858 	if (unlikely(IS_IMMUTABLE(inode)))
5859 		return -EPERM;
5860 
5861 	if (unlikely(IS_APPEND(inode) &&
5862 		     (ia_valid & (ATTR_MODE | ATTR_UID |
5863 				  ATTR_GID | ATTR_TIMES_SET))))
5864 		return -EPERM;
5865 
5866 	error = setattr_prepare(idmap, dentry, attr);
5867 	if (error)
5868 		return error;
5869 
5870 	error = fscrypt_prepare_setattr(dentry, attr);
5871 	if (error)
5872 		return error;
5873 
5874 	error = fsverity_prepare_setattr(dentry, attr);
5875 	if (error)
5876 		return error;
5877 
5878 	if (is_quota_modification(idmap, inode, attr)) {
5879 		error = dquot_initialize(inode);
5880 		if (error)
5881 			return error;
5882 	}
5883 
5884 	if (i_uid_needs_update(idmap, attr, inode) ||
5885 	    i_gid_needs_update(idmap, attr, inode)) {
5886 		handle_t *handle;
5887 
5888 		/* (user+group)*(old+new) structure, inode write (sb,
5889 		 * inode block, ? - but truncate inode update has it) */
5890 		handle = ext4_journal_start(inode, EXT4_HT_QUOTA,
5891 			(EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) +
5892 			 EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3);
5893 		if (IS_ERR(handle)) {
5894 			error = PTR_ERR(handle);
5895 			goto err_out;
5896 		}
5897 
5898 		/* dquot_transfer() calls back ext4_get_inode_usage() which
5899 		 * counts xattr inode references.
5900 		 */
5901 		down_read(&EXT4_I(inode)->xattr_sem);
5902 		error = dquot_transfer(idmap, inode, attr);
5903 		up_read(&EXT4_I(inode)->xattr_sem);
5904 
5905 		if (error) {
5906 			ext4_journal_stop(handle);
5907 			return error;
5908 		}
5909 		/* Update corresponding info in inode so that everything is in
5910 		 * one transaction */
5911 		i_uid_update(idmap, attr, inode);
5912 		i_gid_update(idmap, attr, inode);
5913 		error = ext4_mark_inode_dirty(handle, inode);
5914 		ext4_journal_stop(handle);
5915 		if (unlikely(error)) {
5916 			return error;
5917 		}
5918 	}
5919 
5920 	if (attr->ia_valid & ATTR_SIZE) {
5921 		handle_t *handle;
5922 		loff_t oldsize = inode->i_size;
5923 		loff_t old_disksize;
5924 		int shrink = (attr->ia_size < inode->i_size);
5925 
5926 		if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
5927 			struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5928 
5929 			if (attr->ia_size > sbi->s_bitmap_maxbytes) {
5930 				return -EFBIG;
5931 			}
5932 		}
5933 		if (!S_ISREG(inode->i_mode)) {
5934 			return -EINVAL;
5935 		}
5936 
5937 		if (attr->ia_size == inode->i_size)
5938 			inc_ivers = false;
5939 
5940 		if (shrink) {
5941 			if (ext4_should_order_data(inode)) {
5942 				error = ext4_begin_ordered_truncate(inode,
5943 							    attr->ia_size);
5944 				if (error)
5945 					goto err_out;
5946 			}
5947 			/*
5948 			 * Blocks are going to be removed from the inode. Wait
5949 			 * for dio in flight.
5950 			 */
5951 			inode_dio_wait(inode);
5952 		}
5953 
5954 		filemap_invalidate_lock(inode->i_mapping);
5955 
5956 		rc = ext4_break_layouts(inode);
5957 		if (rc) {
5958 			filemap_invalidate_unlock(inode->i_mapping);
5959 			goto err_out;
5960 		}
5961 
5962 		if (attr->ia_size != inode->i_size) {
5963 			/* attach jbd2 jinode for EOF folio tail zeroing */
5964 			if (attr->ia_size & (inode->i_sb->s_blocksize - 1) ||
5965 			    oldsize & (inode->i_sb->s_blocksize - 1)) {
5966 				error = ext4_inode_attach_jinode(inode);
5967 				if (error)
5968 					goto out_mmap_sem;
5969 			}
5970 
5971 			handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);
5972 			if (IS_ERR(handle)) {
5973 				error = PTR_ERR(handle);
5974 				goto out_mmap_sem;
5975 			}
5976 			if (ext4_handle_valid(handle) && shrink) {
5977 				error = ext4_orphan_add(handle, inode);
5978 				orphan = 1;
5979 			}
5980 			/*
5981 			 * Update c/mtime and tail zero the EOF folio on
5982 			 * truncate up. ext4_truncate() handles the shrink case
5983 			 * below.
5984 			 */
5985 			if (!shrink) {
5986 				inode_set_mtime_to_ts(inode,
5987 						      inode_set_ctime_current(inode));
5988 				if (oldsize & (inode->i_sb->s_blocksize - 1))
5989 					ext4_block_truncate_page(handle,
5990 							inode->i_mapping, oldsize);
5991 			}
5992 
5993 			if (shrink)
5994 				ext4_fc_track_range(handle, inode,
5995 					(attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
5996 					inode->i_sb->s_blocksize_bits,
5997 					EXT_MAX_BLOCKS - 1);
5998 			else
5999 				ext4_fc_track_range(
6000 					handle, inode,
6001 					(oldsize > 0 ? oldsize - 1 : oldsize) >>
6002 					inode->i_sb->s_blocksize_bits,
6003 					(attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
6004 					inode->i_sb->s_blocksize_bits);
6005 
6006 			down_write(&EXT4_I(inode)->i_data_sem);
6007 			old_disksize = EXT4_I(inode)->i_disksize;
6008 			EXT4_I(inode)->i_disksize = attr->ia_size;
6009 
6010 			/*
6011 			 * We have to update i_size under i_data_sem together
6012 			 * with i_disksize to avoid races with writeback code
6013 			 * running ext4_wb_update_i_disksize().
6014 			 */
6015 			if (!error)
6016 				i_size_write(inode, attr->ia_size);
6017 			else
6018 				EXT4_I(inode)->i_disksize = old_disksize;
6019 			up_write(&EXT4_I(inode)->i_data_sem);
6020 			rc = ext4_mark_inode_dirty(handle, inode);
6021 			if (!error)
6022 				error = rc;
6023 			ext4_journal_stop(handle);
6024 			if (error)
6025 				goto out_mmap_sem;
6026 			if (!shrink) {
6027 				pagecache_isize_extended(inode, oldsize,
6028 							 inode->i_size);
6029 			} else if (ext4_should_journal_data(inode)) {
6030 				ext4_wait_for_tail_page_commit(inode);
6031 			}
6032 		}
6033 
6034 		/*
6035 		 * Truncate pagecache after we've waited for commit
6036 		 * in data=journal mode to make pages freeable.
6037 		 */
6038 		truncate_pagecache(inode, inode->i_size);
6039 		/*
6040 		 * Call ext4_truncate() even if i_size didn't change to
6041 		 * truncate possible preallocated blocks.
6042 		 */
6043 		if (attr->ia_size <= oldsize) {
6044 			rc = ext4_truncate(inode);
6045 			if (rc)
6046 				error = rc;
6047 		}
6048 out_mmap_sem:
6049 		filemap_invalidate_unlock(inode->i_mapping);
6050 	}
6051 
6052 	if (!error) {
6053 		if (inc_ivers)
6054 			inode_inc_iversion(inode);
6055 		setattr_copy(idmap, inode, attr);
6056 		mark_inode_dirty(inode);
6057 	}
6058 
6059 	/*
6060 	 * If the call to ext4_truncate failed to get a transaction handle at
6061 	 * all, we need to clean up the in-core orphan list manually.
6062 	 */
6063 	if (orphan && inode->i_nlink)
6064 		ext4_orphan_del(NULL, inode);
6065 
6066 	if (!error && (ia_valid & ATTR_MODE))
6067 		rc = posix_acl_chmod(idmap, dentry, inode->i_mode);
6068 
6069 err_out:
6070 	if  (error)
6071 		ext4_std_error(inode->i_sb, error);
6072 	if (!error)
6073 		error = rc;
6074 	return error;
6075 }
6076 
ext4_dio_alignment(struct inode * inode)6077 u32 ext4_dio_alignment(struct inode *inode)
6078 {
6079 	if (fsverity_active(inode))
6080 		return 0;
6081 	if (ext4_should_journal_data(inode))
6082 		return 0;
6083 	if (ext4_has_inline_data(inode))
6084 		return 0;
6085 	if (IS_ENCRYPTED(inode)) {
6086 		if (!fscrypt_dio_supported(inode))
6087 			return 0;
6088 		return i_blocksize(inode);
6089 	}
6090 	return 1; /* use the iomap defaults */
6091 }
6092 
ext4_getattr(struct mnt_idmap * idmap,const struct path * path,struct kstat * stat,u32 request_mask,unsigned int query_flags)6093 int ext4_getattr(struct mnt_idmap *idmap, const struct path *path,
6094 		 struct kstat *stat, u32 request_mask, unsigned int query_flags)
6095 {
6096 	struct inode *inode = d_inode(path->dentry);
6097 	struct ext4_inode *raw_inode;
6098 	struct ext4_inode_info *ei = EXT4_I(inode);
6099 	unsigned int flags;
6100 
6101 	if ((request_mask & STATX_BTIME) &&
6102 	    EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) {
6103 		stat->result_mask |= STATX_BTIME;
6104 		stat->btime.tv_sec = ei->i_crtime.tv_sec;
6105 		stat->btime.tv_nsec = ei->i_crtime.tv_nsec;
6106 	}
6107 
6108 	/*
6109 	 * Return the DIO alignment restrictions if requested.  We only return
6110 	 * this information when requested, since on encrypted files it might
6111 	 * take a fair bit of work to get if the file wasn't opened recently.
6112 	 */
6113 	if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) {
6114 		u32 dio_align = ext4_dio_alignment(inode);
6115 
6116 		stat->result_mask |= STATX_DIOALIGN;
6117 		if (dio_align == 1) {
6118 			struct block_device *bdev = inode->i_sb->s_bdev;
6119 
6120 			/* iomap defaults */
6121 			stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;
6122 			stat->dio_offset_align = bdev_logical_block_size(bdev);
6123 		} else {
6124 			stat->dio_mem_align = dio_align;
6125 			stat->dio_offset_align = dio_align;
6126 		}
6127 	}
6128 
6129 	if ((request_mask & STATX_WRITE_ATOMIC) && S_ISREG(inode->i_mode)) {
6130 		struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
6131 		unsigned int awu_min = 0, awu_max = 0;
6132 
6133 		if (ext4_inode_can_atomic_write(inode)) {
6134 			awu_min = sbi->s_awu_min;
6135 			awu_max = sbi->s_awu_max;
6136 		}
6137 
6138 		generic_fill_statx_atomic_writes(stat, awu_min, awu_max, 0);
6139 	}
6140 
6141 	flags = ei->i_flags & EXT4_FL_USER_VISIBLE;
6142 	if (flags & EXT4_APPEND_FL)
6143 		stat->attributes |= STATX_ATTR_APPEND;
6144 	if (flags & EXT4_COMPR_FL)
6145 		stat->attributes |= STATX_ATTR_COMPRESSED;
6146 	if (flags & EXT4_ENCRYPT_FL)
6147 		stat->attributes |= STATX_ATTR_ENCRYPTED;
6148 	if (flags & EXT4_IMMUTABLE_FL)
6149 		stat->attributes |= STATX_ATTR_IMMUTABLE;
6150 	if (flags & EXT4_NODUMP_FL)
6151 		stat->attributes |= STATX_ATTR_NODUMP;
6152 	if (flags & EXT4_VERITY_FL)
6153 		stat->attributes |= STATX_ATTR_VERITY;
6154 
6155 	stat->attributes_mask |= (STATX_ATTR_APPEND |
6156 				  STATX_ATTR_COMPRESSED |
6157 				  STATX_ATTR_ENCRYPTED |
6158 				  STATX_ATTR_IMMUTABLE |
6159 				  STATX_ATTR_NODUMP |
6160 				  STATX_ATTR_VERITY);
6161 
6162 	generic_fillattr(idmap, request_mask, inode, stat);
6163 	return 0;
6164 }
6165 
ext4_file_getattr(struct mnt_idmap * idmap,const struct path * path,struct kstat * stat,u32 request_mask,unsigned int query_flags)6166 int ext4_file_getattr(struct mnt_idmap *idmap,
6167 		      const struct path *path, struct kstat *stat,
6168 		      u32 request_mask, unsigned int query_flags)
6169 {
6170 	struct inode *inode = d_inode(path->dentry);
6171 	u64 delalloc_blocks;
6172 
6173 	ext4_getattr(idmap, path, stat, request_mask, query_flags);
6174 
6175 	/*
6176 	 * If there is inline data in the inode, the inode will normally not
6177 	 * have data blocks allocated (it may have an external xattr block).
6178 	 * Report at least one sector for such files, so tools like tar, rsync,
6179 	 * others don't incorrectly think the file is completely sparse.
6180 	 */
6181 	if (unlikely(ext4_has_inline_data(inode)))
6182 		stat->blocks += (stat->size + 511) >> 9;
6183 
6184 	/*
6185 	 * We can't update i_blocks if the block allocation is delayed
6186 	 * otherwise in the case of system crash before the real block
6187 	 * allocation is done, we will have i_blocks inconsistent with
6188 	 * on-disk file blocks.
6189 	 * We always keep i_blocks updated together with real
6190 	 * allocation. But to not confuse with user, stat
6191 	 * will return the blocks that include the delayed allocation
6192 	 * blocks for this file.
6193 	 */
6194 	delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb),
6195 				   EXT4_I(inode)->i_reserved_data_blocks);
6196 	stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9);
6197 	return 0;
6198 }
6199 
ext4_index_trans_blocks(struct inode * inode,int lblocks,int pextents)6200 static int ext4_index_trans_blocks(struct inode *inode, int lblocks,
6201 				   int pextents)
6202 {
6203 	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
6204 		return ext4_ind_trans_blocks(inode, lblocks);
6205 	return ext4_ext_index_trans_blocks(inode, pextents);
6206 }
6207 
6208 /*
6209  * Account for index blocks, block groups bitmaps and block group
6210  * descriptor blocks if modify datablocks and index blocks
6211  * worse case, the indexs blocks spread over different block groups
6212  *
6213  * If datablocks are discontiguous, they are possible to spread over
6214  * different block groups too. If they are contiguous, with flexbg,
6215  * they could still across block group boundary.
6216  *
6217  * Also account for superblock, inode, quota and xattr blocks
6218  */
ext4_meta_trans_blocks(struct inode * inode,int lblocks,int pextents)6219 int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents)
6220 {
6221 	ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);
6222 	int gdpblocks;
6223 	int idxblocks;
6224 	int ret;
6225 
6226 	/*
6227 	 * How many index and leaf blocks need to touch to map @lblocks
6228 	 * logical blocks to @pextents physical extents?
6229 	 */
6230 	idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents);
6231 
6232 	/*
6233 	 * Now let's see how many group bitmaps and group descriptors need
6234 	 * to account
6235 	 */
6236 	groups = idxblocks + pextents;
6237 	gdpblocks = groups;
6238 	if (groups > ngroups)
6239 		groups = ngroups;
6240 	if (groups > EXT4_SB(inode->i_sb)->s_gdb_count)
6241 		gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count;
6242 
6243 	/* bitmaps and block group descriptor blocks */
6244 	ret = idxblocks + groups + gdpblocks;
6245 
6246 	/* Blocks for super block, inode, quota and xattr blocks */
6247 	ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);
6248 
6249 	return ret;
6250 }
6251 
6252 /*
6253  * Calculate the journal credits for modifying the number of blocks
6254  * in a single extent within one transaction. 'nrblocks' is used only
6255  * for non-extent inodes. For extent type inodes, 'nrblocks' can be
6256  * zero if the exact number of blocks is unknown.
6257  */
ext4_chunk_trans_extent(struct inode * inode,int nrblocks)6258 int ext4_chunk_trans_extent(struct inode *inode, int nrblocks)
6259 {
6260 	int ret;
6261 
6262 	ret = ext4_meta_trans_blocks(inode, nrblocks, 1);
6263 	/* Account for data blocks for journalled mode */
6264 	if (ext4_should_journal_data(inode))
6265 		ret += nrblocks;
6266 	return ret;
6267 }
6268 
6269 /*
6270  * Calculate the journal credits for a chunk of data modification.
6271  *
6272  * This is called from DIO, fallocate or whoever calling
6273  * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks.
6274  *
6275  * journal buffers for data blocks are not included here, as DIO
6276  * and fallocate do no need to journal data buffers.
6277  */
ext4_chunk_trans_blocks(struct inode * inode,int nrblocks)6278 int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
6279 {
6280 	return ext4_meta_trans_blocks(inode, nrblocks, 1);
6281 }
6282 
6283 /*
6284  * The caller must have previously called ext4_reserve_inode_write().
6285  * Give this, we know that the caller already has write access to iloc->bh.
6286  */
ext4_mark_iloc_dirty(handle_t * handle,struct inode * inode,struct ext4_iloc * iloc)6287 int ext4_mark_iloc_dirty(handle_t *handle,
6288 			 struct inode *inode, struct ext4_iloc *iloc)
6289 {
6290 	int err = 0;
6291 
6292 	err = ext4_emergency_state(inode->i_sb);
6293 	if (unlikely(err)) {
6294 		put_bh(iloc->bh);
6295 		return err;
6296 	}
6297 	ext4_fc_track_inode(handle, inode);
6298 
6299 	/* the do_update_inode consumes one bh->b_count */
6300 	get_bh(iloc->bh);
6301 
6302 	/* ext4_do_update_inode() does jbd2_journal_dirty_metadata */
6303 	err = ext4_do_update_inode(handle, inode, iloc);
6304 	put_bh(iloc->bh);
6305 	return err;
6306 }
6307 
6308 /*
6309  * On success, We end up with an outstanding reference count against
6310  * iloc->bh.  This _must_ be cleaned up later.
6311  */
6312 
6313 int
ext4_reserve_inode_write(handle_t * handle,struct inode * inode,struct ext4_iloc * iloc)6314 ext4_reserve_inode_write(handle_t *handle, struct inode *inode,
6315 			 struct ext4_iloc *iloc)
6316 {
6317 	int err;
6318 
6319 	err = ext4_emergency_state(inode->i_sb);
6320 	if (unlikely(err))
6321 		return err;
6322 
6323 	err = ext4_get_inode_loc(inode, iloc);
6324 	if (!err) {
6325 		BUFFER_TRACE(iloc->bh, "get_write_access");
6326 		err = ext4_journal_get_write_access(handle, inode->i_sb,
6327 						    iloc->bh, EXT4_JTR_NONE);
6328 		if (err) {
6329 			brelse(iloc->bh);
6330 			iloc->bh = NULL;
6331 		}
6332 		ext4_fc_track_inode(handle, inode);
6333 	}
6334 	ext4_std_error(inode->i_sb, err);
6335 	return err;
6336 }
6337 
__ext4_expand_extra_isize(struct inode * inode,unsigned int new_extra_isize,struct ext4_iloc * iloc,handle_t * handle,int * no_expand)6338 static int __ext4_expand_extra_isize(struct inode *inode,
6339 				     unsigned int new_extra_isize,
6340 				     struct ext4_iloc *iloc,
6341 				     handle_t *handle, int *no_expand)
6342 {
6343 	struct ext4_inode *raw_inode;
6344 	struct ext4_xattr_ibody_header *header;
6345 	unsigned int inode_size = EXT4_INODE_SIZE(inode->i_sb);
6346 	struct ext4_inode_info *ei = EXT4_I(inode);
6347 	int error;
6348 
6349 	/* this was checked at iget time, but double check for good measure */
6350 	if ((EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > inode_size) ||
6351 	    (ei->i_extra_isize & 3)) {
6352 		EXT4_ERROR_INODE(inode, "bad extra_isize %u (inode size %u)",
6353 				 ei->i_extra_isize,
6354 				 EXT4_INODE_SIZE(inode->i_sb));
6355 		return -EFSCORRUPTED;
6356 	}
6357 	if ((new_extra_isize < ei->i_extra_isize) ||
6358 	    (new_extra_isize < 4) ||
6359 	    (new_extra_isize > inode_size - EXT4_GOOD_OLD_INODE_SIZE))
6360 		return -EINVAL;	/* Should never happen */
6361 
6362 	raw_inode = ext4_raw_inode(iloc);
6363 
6364 	header = IHDR(inode, raw_inode);
6365 
6366 	/* No extended attributes present */
6367 	if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
6368 	    header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
6369 		memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE +
6370 		       EXT4_I(inode)->i_extra_isize, 0,
6371 		       new_extra_isize - EXT4_I(inode)->i_extra_isize);
6372 		EXT4_I(inode)->i_extra_isize = new_extra_isize;
6373 		return 0;
6374 	}
6375 
6376 	/*
6377 	 * We may need to allocate external xattr block so we need quotas
6378 	 * initialized. Here we can be called with various locks held so we
6379 	 * cannot affort to initialize quotas ourselves. So just bail.
6380 	 */
6381 	if (dquot_initialize_needed(inode))
6382 		return -EAGAIN;
6383 
6384 	/* try to expand with EAs present */
6385 	error = ext4_expand_extra_isize_ea(inode, new_extra_isize,
6386 					   raw_inode, handle);
6387 	if (error) {
6388 		/*
6389 		 * Inode size expansion failed; don't try again
6390 		 */
6391 		*no_expand = 1;
6392 	}
6393 
6394 	return error;
6395 }
6396 
6397 /*
6398  * Expand an inode by new_extra_isize bytes.
6399  * Returns 0 on success or negative error number on failure.
6400  */
ext4_try_to_expand_extra_isize(struct inode * inode,unsigned int new_extra_isize,struct ext4_iloc iloc,handle_t * handle)6401 static int ext4_try_to_expand_extra_isize(struct inode *inode,
6402 					  unsigned int new_extra_isize,
6403 					  struct ext4_iloc iloc,
6404 					  handle_t *handle)
6405 {
6406 	int no_expand;
6407 	int error;
6408 
6409 	if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND))
6410 		return -EOVERFLOW;
6411 
6412 	/*
6413 	 * In nojournal mode, we can immediately attempt to expand
6414 	 * the inode.  When journaled, we first need to obtain extra
6415 	 * buffer credits since we may write into the EA block
6416 	 * with this same handle. If journal_extend fails, then it will
6417 	 * only result in a minor loss of functionality for that inode.
6418 	 * If this is felt to be critical, then e2fsck should be run to
6419 	 * force a large enough s_min_extra_isize.
6420 	 */
6421 	if (ext4_journal_extend(handle,
6422 				EXT4_DATA_TRANS_BLOCKS(inode->i_sb), 0) != 0)
6423 		return -ENOSPC;
6424 
6425 	if (ext4_write_trylock_xattr(inode, &no_expand) == 0)
6426 		return -EBUSY;
6427 
6428 	error = __ext4_expand_extra_isize(inode, new_extra_isize, &iloc,
6429 					  handle, &no_expand);
6430 	ext4_write_unlock_xattr(inode, &no_expand);
6431 
6432 	return error;
6433 }
6434 
ext4_expand_extra_isize(struct inode * inode,unsigned int new_extra_isize,struct ext4_iloc * iloc)6435 int ext4_expand_extra_isize(struct inode *inode,
6436 			    unsigned int new_extra_isize,
6437 			    struct ext4_iloc *iloc)
6438 {
6439 	handle_t *handle;
6440 	int no_expand;
6441 	int error, rc;
6442 
6443 	if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
6444 		brelse(iloc->bh);
6445 		return -EOVERFLOW;
6446 	}
6447 
6448 	handle = ext4_journal_start(inode, EXT4_HT_INODE,
6449 				    EXT4_DATA_TRANS_BLOCKS(inode->i_sb));
6450 	if (IS_ERR(handle)) {
6451 		error = PTR_ERR(handle);
6452 		brelse(iloc->bh);
6453 		return error;
6454 	}
6455 
6456 	ext4_write_lock_xattr(inode, &no_expand);
6457 
6458 	BUFFER_TRACE(iloc->bh, "get_write_access");
6459 	error = ext4_journal_get_write_access(handle, inode->i_sb, iloc->bh,
6460 					      EXT4_JTR_NONE);
6461 	if (error) {
6462 		brelse(iloc->bh);
6463 		goto out_unlock;
6464 	}
6465 
6466 	error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc,
6467 					  handle, &no_expand);
6468 
6469 	rc = ext4_mark_iloc_dirty(handle, inode, iloc);
6470 	if (!error)
6471 		error = rc;
6472 
6473 out_unlock:
6474 	ext4_write_unlock_xattr(inode, &no_expand);
6475 	ext4_journal_stop(handle);
6476 	return error;
6477 }
6478 
6479 /*
6480  * What we do here is to mark the in-core inode as clean with respect to inode
6481  * dirtiness (it may still be data-dirty).
6482  * This means that the in-core inode may be reaped by prune_icache
6483  * without having to perform any I/O.  This is a very good thing,
6484  * because *any* task may call prune_icache - even ones which
6485  * have a transaction open against a different journal.
6486  *
6487  * Is this cheating?  Not really.  Sure, we haven't written the
6488  * inode out, but prune_icache isn't a user-visible syncing function.
6489  * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync)
6490  * we start and wait on commits.
6491  */
__ext4_mark_inode_dirty(handle_t * handle,struct inode * inode,const char * func,unsigned int line)6492 int __ext4_mark_inode_dirty(handle_t *handle, struct inode *inode,
6493 				const char *func, unsigned int line)
6494 {
6495 	struct ext4_iloc iloc;
6496 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
6497 	int err;
6498 
6499 	might_sleep();
6500 	trace_ext4_mark_inode_dirty(inode, _RET_IP_);
6501 	err = ext4_reserve_inode_write(handle, inode, &iloc);
6502 	if (err)
6503 		goto out;
6504 
6505 	if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize)
6506 		ext4_try_to_expand_extra_isize(inode, sbi->s_want_extra_isize,
6507 					       iloc, handle);
6508 
6509 	err = ext4_mark_iloc_dirty(handle, inode, &iloc);
6510 out:
6511 	if (unlikely(err))
6512 		ext4_error_inode_err(inode, func, line, 0, err,
6513 					"mark_inode_dirty error");
6514 	return err;
6515 }
6516 
6517 /*
6518  * ext4_dirty_inode() is called from __mark_inode_dirty()
6519  *
6520  * We're really interested in the case where a file is being extended.
6521  * i_size has been changed by generic_commit_write() and we thus need
6522  * to include the updated inode in the current transaction.
6523  *
6524  * Also, dquot_alloc_block() will always dirty the inode when blocks
6525  * are allocated to the file.
6526  *
6527  * If the inode is marked synchronous, we don't honour that here - doing
6528  * so would cause a commit on atime updates, which we don't bother doing.
6529  * We handle synchronous inodes at the highest possible level.
6530  */
ext4_dirty_inode(struct inode * inode,int flags)6531 void ext4_dirty_inode(struct inode *inode, int flags)
6532 {
6533 	handle_t *handle;
6534 
6535 	handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
6536 	if (IS_ERR(handle))
6537 		return;
6538 	ext4_mark_inode_dirty(handle, inode);
6539 	ext4_journal_stop(handle);
6540 }
6541 
ext4_change_inode_journal_flag(struct inode * inode,int val)6542 int ext4_change_inode_journal_flag(struct inode *inode, int val)
6543 {
6544 	journal_t *journal;
6545 	handle_t *handle;
6546 	int err;
6547 	int alloc_ctx;
6548 
6549 	/*
6550 	 * We have to be very careful here: changing a data block's
6551 	 * journaling status dynamically is dangerous.  If we write a
6552 	 * data block to the journal, change the status and then delete
6553 	 * that block, we risk forgetting to revoke the old log record
6554 	 * from the journal and so a subsequent replay can corrupt data.
6555 	 * So, first we make sure that the journal is empty and that
6556 	 * nobody is changing anything.
6557 	 */
6558 
6559 	journal = EXT4_JOURNAL(inode);
6560 	if (!journal)
6561 		return 0;
6562 	if (is_journal_aborted(journal))
6563 		return -EROFS;
6564 
6565 	/* Wait for all existing dio workers */
6566 	inode_dio_wait(inode);
6567 
6568 	/*
6569 	 * Before flushing the journal and switching inode's aops, we have
6570 	 * to flush all dirty data the inode has. There can be outstanding
6571 	 * delayed allocations, there can be unwritten extents created by
6572 	 * fallocate or buffered writes in dioread_nolock mode covered by
6573 	 * dirty data which can be converted only after flushing the dirty
6574 	 * data (and journalled aops don't know how to handle these cases).
6575 	 */
6576 	if (val) {
6577 		filemap_invalidate_lock(inode->i_mapping);
6578 		err = filemap_write_and_wait(inode->i_mapping);
6579 		if (err < 0) {
6580 			filemap_invalidate_unlock(inode->i_mapping);
6581 			return err;
6582 		}
6583 	}
6584 
6585 	alloc_ctx = ext4_writepages_down_write(inode->i_sb);
6586 	jbd2_journal_lock_updates(journal);
6587 
6588 	/*
6589 	 * OK, there are no updates running now, and all cached data is
6590 	 * synced to disk.  We are now in a completely consistent state
6591 	 * which doesn't have anything in the journal, and we know that
6592 	 * no filesystem updates are running, so it is safe to modify
6593 	 * the inode's in-core data-journaling state flag now.
6594 	 */
6595 
6596 	if (val)
6597 		ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
6598 	else {
6599 		err = jbd2_journal_flush(journal, 0);
6600 		if (err < 0) {
6601 			jbd2_journal_unlock_updates(journal);
6602 			ext4_writepages_up_write(inode->i_sb, alloc_ctx);
6603 			return err;
6604 		}
6605 		ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
6606 	}
6607 	ext4_set_aops(inode);
6608 
6609 	jbd2_journal_unlock_updates(journal);
6610 	ext4_writepages_up_write(inode->i_sb, alloc_ctx);
6611 
6612 	if (val)
6613 		filemap_invalidate_unlock(inode->i_mapping);
6614 
6615 	/* Finally we can mark the inode as dirty. */
6616 
6617 	handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
6618 	if (IS_ERR(handle))
6619 		return PTR_ERR(handle);
6620 
6621 	ext4_fc_mark_ineligible(inode->i_sb,
6622 		EXT4_FC_REASON_JOURNAL_FLAG_CHANGE, handle);
6623 	err = ext4_mark_inode_dirty(handle, inode);
6624 	ext4_handle_sync(handle);
6625 	ext4_journal_stop(handle);
6626 	ext4_std_error(inode->i_sb, err);
6627 
6628 	return err;
6629 }
6630 
ext4_bh_unmapped(handle_t * handle,struct inode * inode,struct buffer_head * bh)6631 static int ext4_bh_unmapped(handle_t *handle, struct inode *inode,
6632 			    struct buffer_head *bh)
6633 {
6634 	return !buffer_mapped(bh);
6635 }
6636 
ext4_block_page_mkwrite(struct inode * inode,struct folio * folio,get_block_t get_block)6637 static int ext4_block_page_mkwrite(struct inode *inode, struct folio *folio,
6638 				   get_block_t get_block)
6639 {
6640 	handle_t *handle;
6641 	loff_t size;
6642 	unsigned long len;
6643 	int credits;
6644 	int ret;
6645 
6646 	credits = ext4_chunk_trans_extent(inode,
6647 			ext4_journal_blocks_per_folio(inode));
6648 	handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, credits);
6649 	if (IS_ERR(handle))
6650 		return PTR_ERR(handle);
6651 
6652 	folio_lock(folio);
6653 	size = i_size_read(inode);
6654 	/* Page got truncated from under us? */
6655 	if (folio->mapping != inode->i_mapping || folio_pos(folio) > size) {
6656 		ret = -EFAULT;
6657 		goto out_error;
6658 	}
6659 
6660 	len = folio_size(folio);
6661 	if (folio_pos(folio) + len > size)
6662 		len = size - folio_pos(folio);
6663 
6664 	ret = ext4_block_write_begin(handle, folio, 0, len, get_block);
6665 	if (ret)
6666 		goto out_error;
6667 
6668 	if (!ext4_should_journal_data(inode)) {
6669 		block_commit_write(folio, 0, len);
6670 		folio_mark_dirty(folio);
6671 	} else {
6672 		ret = ext4_journal_folio_buffers(handle, folio, len);
6673 		if (ret)
6674 			goto out_error;
6675 	}
6676 	ext4_journal_stop(handle);
6677 	folio_wait_stable(folio);
6678 	return ret;
6679 
6680 out_error:
6681 	folio_unlock(folio);
6682 	ext4_journal_stop(handle);
6683 	return ret;
6684 }
6685 
ext4_page_mkwrite(struct vm_fault * vmf)6686 vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf)
6687 {
6688 	struct vm_area_struct *vma = vmf->vma;
6689 	struct folio *folio = page_folio(vmf->page);
6690 	loff_t size;
6691 	unsigned long len;
6692 	int err;
6693 	vm_fault_t ret;
6694 	struct file *file = vma->vm_file;
6695 	struct inode *inode = file_inode(file);
6696 	struct address_space *mapping = inode->i_mapping;
6697 	get_block_t *get_block = ext4_get_block;
6698 	int retries = 0;
6699 
6700 	if (unlikely(IS_IMMUTABLE(inode)))
6701 		return VM_FAULT_SIGBUS;
6702 
6703 	sb_start_pagefault(inode->i_sb);
6704 	file_update_time(vma->vm_file);
6705 
6706 	filemap_invalidate_lock_shared(mapping);
6707 
6708 	err = ext4_convert_inline_data(inode);
6709 	if (err)
6710 		goto out_ret;
6711 
6712 	/*
6713 	 * On data journalling we skip straight to the transaction handle:
6714 	 * there's no delalloc; page truncated will be checked later; the
6715 	 * early return w/ all buffers mapped (calculates size/len) can't
6716 	 * be used; and there's no dioread_nolock, so only ext4_get_block.
6717 	 */
6718 	if (ext4_should_journal_data(inode))
6719 		goto retry_alloc;
6720 
6721 	/* Delalloc case is easy... */
6722 	if (test_opt(inode->i_sb, DELALLOC) &&
6723 	    !ext4_nonda_switch(inode->i_sb)) {
6724 		do {
6725 			err = block_page_mkwrite(vma, vmf,
6726 						   ext4_da_get_block_prep);
6727 		} while (err == -ENOSPC &&
6728 		       ext4_should_retry_alloc(inode->i_sb, &retries));
6729 		goto out_ret;
6730 	}
6731 
6732 	folio_lock(folio);
6733 	size = i_size_read(inode);
6734 	/* Page got truncated from under us? */
6735 	if (folio->mapping != mapping || folio_pos(folio) > size) {
6736 		folio_unlock(folio);
6737 		ret = VM_FAULT_NOPAGE;
6738 		goto out;
6739 	}
6740 
6741 	len = folio_size(folio);
6742 	if (folio_pos(folio) + len > size)
6743 		len = size - folio_pos(folio);
6744 	/*
6745 	 * Return if we have all the buffers mapped. This avoids the need to do
6746 	 * journal_start/journal_stop which can block and take a long time
6747 	 *
6748 	 * This cannot be done for data journalling, as we have to add the
6749 	 * inode to the transaction's list to writeprotect pages on commit.
6750 	 */
6751 	if (folio_buffers(folio)) {
6752 		if (!ext4_walk_page_buffers(NULL, inode, folio_buffers(folio),
6753 					    0, len, NULL,
6754 					    ext4_bh_unmapped)) {
6755 			/* Wait so that we don't change page under IO */
6756 			folio_wait_stable(folio);
6757 			ret = VM_FAULT_LOCKED;
6758 			goto out;
6759 		}
6760 	}
6761 	folio_unlock(folio);
6762 	/* OK, we need to fill the hole... */
6763 	if (ext4_should_dioread_nolock(inode))
6764 		get_block = ext4_get_block_unwritten;
6765 retry_alloc:
6766 	/* Start journal and allocate blocks */
6767 	err = ext4_block_page_mkwrite(inode, folio, get_block);
6768 	if (err == -EAGAIN ||
6769 	    (err == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)))
6770 		goto retry_alloc;
6771 out_ret:
6772 	ret = vmf_fs_error(err);
6773 out:
6774 	filemap_invalidate_unlock_shared(mapping);
6775 	sb_end_pagefault(inode->i_sb);
6776 	return ret;
6777 }
6778