1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 #ifndef BTRFS_EXTENT_IO_H
4 #define BTRFS_EXTENT_IO_H
5
6 #include <linux/rbtree.h>
7 #include <linux/refcount.h>
8 #include <linux/fiemap.h>
9 #include <linux/btrfs_tree.h>
10 #include <linux/spinlock.h>
11 #include <linux/atomic.h>
12 #include <linux/rwsem.h>
13 #include <linux/list.h>
14 #include <linux/slab.h>
15 #include "messages.h"
16 #include "ulist.h"
17 #include "misc.h"
18
19 struct page;
20 struct file;
21 struct folio;
22 struct inode;
23 struct fiemap_extent_info;
24 struct readahead_control;
25 struct address_space;
26 struct writeback_control;
27 struct extent_io_tree;
28 struct extent_map_tree;
29 struct extent_state;
30 struct btrfs_block_group;
31 struct btrfs_fs_info;
32 struct btrfs_inode;
33 struct btrfs_root;
34 struct btrfs_trans_handle;
35 struct btrfs_tree_parent_check;
36
37 enum {
38 EXTENT_BUFFER_UPTODATE,
39 EXTENT_BUFFER_DIRTY,
40 EXTENT_BUFFER_TREE_REF,
41 EXTENT_BUFFER_STALE,
42 EXTENT_BUFFER_WRITEBACK,
43 EXTENT_BUFFER_UNMAPPED,
44 /* write IO error */
45 EXTENT_BUFFER_WRITE_ERR,
46 /* Indicate the extent buffer is written zeroed out (for zoned) */
47 EXTENT_BUFFER_ZONED_ZEROOUT,
48 /* Indicate that extent buffer pages a being read */
49 EXTENT_BUFFER_READING,
50 };
51
52 /* these are flags for __process_pages_contig */
53 enum {
54 ENUM_BIT(PAGE_UNLOCK),
55 /* Page starts writeback, clear dirty bit and set writeback bit */
56 ENUM_BIT(PAGE_START_WRITEBACK),
57 ENUM_BIT(PAGE_END_WRITEBACK),
58 ENUM_BIT(PAGE_SET_ORDERED),
59 };
60
61 /*
62 * Folio private values. Every page that is controlled by the extent map has
63 * folio private set to this value.
64 */
65 #define EXTENT_FOLIO_PRIVATE 1
66
67 /*
68 * The extent buffer bitmap operations are done with byte granularity instead of
69 * word granularity for two reasons:
70 * 1. The bitmaps must be little-endian on disk.
71 * 2. Bitmap items are not guaranteed to be aligned to a word and therefore a
72 * single word in a bitmap may straddle two pages in the extent buffer.
73 */
74 #define BIT_BYTE(nr) ((nr) / BITS_PER_BYTE)
75 #define BYTE_MASK ((1U << BITS_PER_BYTE) - 1)
76 #define BITMAP_FIRST_BYTE_MASK(start) \
77 ((BYTE_MASK << ((start) & (BITS_PER_BYTE - 1))) & BYTE_MASK)
78 #define BITMAP_LAST_BYTE_MASK(nbits) \
79 (BYTE_MASK >> (-(nbits) & (BITS_PER_BYTE - 1)))
80
81
82 int __init extent_buffer_init_cachep(void);
83 void __cold extent_buffer_free_cachep(void);
84
85 #define INLINE_EXTENT_BUFFER_PAGES (BTRFS_MAX_METADATA_BLOCKSIZE / PAGE_SIZE)
86 struct extent_buffer {
87 u64 start;
88 u32 len;
89 u32 folio_size;
90 unsigned long bflags;
91 struct btrfs_fs_info *fs_info;
92
93 /*
94 * The address where the eb can be accessed without any cross-page handling.
95 * This can be NULL if not possible.
96 */
97 void *addr;
98
99 spinlock_t refs_lock;
100 refcount_t refs;
101 int read_mirror;
102 /* Inhibit WB_SYNC_NONE writeback when > 0. */
103 atomic_t writeback_inhibitors;
104 /* >= 0 if eb belongs to a log tree, -1 otherwise */
105 s8 log_index;
106 u8 folio_shift;
107 struct rcu_head rcu_head;
108
109 struct rw_semaphore lock;
110
111 /*
112 * Pointers to all the folios of the extent buffer.
113 *
114 * For now the folio is always order 0 (aka, a single page).
115 */
116 struct folio *folios[INLINE_EXTENT_BUFFER_PAGES];
117 #ifdef CONFIG_BTRFS_DEBUG
118 struct list_head leak_list;
119 pid_t lock_owner;
120 #endif
121 };
122
123 struct btrfs_eb_write_context {
124 struct writeback_control *wbc;
125 struct extent_buffer *eb;
126 /* Block group @eb resides in. Only used for zoned mode. */
127 struct btrfs_block_group *zoned_bg;
128 };
129
offset_in_eb_folio(const struct extent_buffer * eb,u64 start)130 static inline unsigned long offset_in_eb_folio(const struct extent_buffer *eb,
131 u64 start)
132 {
133 ASSERT(eb->folio_size);
134 return start & (eb->folio_size - 1);
135 }
136
137 /*
138 * Get the correct offset inside the page of extent buffer.
139 *
140 * @eb: target extent buffer
141 * @start: offset inside the extent buffer
142 *
143 * Will handle both sectorsize == PAGE_SIZE and sectorsize < PAGE_SIZE cases.
144 */
get_eb_offset_in_folio(const struct extent_buffer * eb,unsigned long offset)145 static inline size_t get_eb_offset_in_folio(const struct extent_buffer *eb,
146 unsigned long offset)
147 {
148 /*
149 * 1) sectorsize == PAGE_SIZE and nodesize >= PAGE_SIZE case
150 * 1.1) One large folio covering the whole eb
151 * The eb->start is aligned to folio size, thus adding it
152 * won't cause any difference.
153 * 1.2) Several page sized folios
154 * The eb->start is aligned to folio (page) size, thus
155 * adding it won't cause any difference.
156 *
157 * 2) sectorsize < PAGE_SIZE and nodesize < PAGE_SIZE case
158 * In this case there would only be one page sized folio, and there
159 * may be several different extent buffers in the page/folio.
160 * We need to add eb->start to properly access the offset inside
161 * that eb.
162 */
163 return offset_in_folio(eb->folios[0], offset + eb->start);
164 }
165
get_eb_folio_index(const struct extent_buffer * eb,unsigned long offset)166 static inline unsigned long get_eb_folio_index(const struct extent_buffer *eb,
167 unsigned long offset)
168 {
169 /*
170 * 1) sectorsize == PAGE_SIZE and nodesize >= PAGE_SIZE case
171 * 1.1) One large folio covering the whole eb.
172 * the folio_shift would be large enough to always make us
173 * return 0 as index.
174 * 1.2) Several page sized folios
175 * The folio_shift would be PAGE_SHIFT, giving us the correct
176 * index.
177 *
178 * 2) sectorsize < PAGE_SIZE and nodesize < PAGE_SIZE case
179 * The folio would only be page sized, and always give us 0 as index.
180 */
181 return offset >> eb->folio_shift;
182 }
183
184 /*
185 * Structure to record how many bytes and which ranges are set/cleared
186 */
187 struct extent_changeset {
188 /* How many bytes are set/cleared in this operation */
189 u64 bytes_changed;
190
191 /* Changed ranges */
192 struct ulist range_changed;
193 };
194
extent_changeset_init(struct extent_changeset * changeset)195 static inline void extent_changeset_init(struct extent_changeset *changeset)
196 {
197 changeset->bytes_changed = 0;
198 ulist_init(&changeset->range_changed);
199 }
200
201 /*
202 * Sentinel value for range_changed.prealloc indicating that the changeset
203 * only tracks bytes_changed and does not record individual ranges. This
204 * avoids GFP_ATOMIC allocations inside add_extent_changeset() when the
205 * caller doesn't need to iterate the changed ranges afterwards.
206 */
207 #define EXTENT_CHANGESET_BYTES_ONLY ((struct ulist_node *)1)
208
extent_changeset_init_bytes_only(struct extent_changeset * changeset)209 static inline void extent_changeset_init_bytes_only(struct extent_changeset *changeset)
210 {
211 changeset->bytes_changed = 0;
212 changeset->range_changed.prealloc = EXTENT_CHANGESET_BYTES_ONLY;
213 }
214
extent_changeset_tracks_ranges(const struct extent_changeset * changeset)215 static inline bool extent_changeset_tracks_ranges(const struct extent_changeset *changeset)
216 {
217 return changeset->range_changed.prealloc != EXTENT_CHANGESET_BYTES_ONLY;
218 }
219
extent_changeset_alloc(void)220 static inline struct extent_changeset *extent_changeset_alloc(void)
221 {
222 struct extent_changeset *ret;
223
224 ret = kmalloc_obj(*ret);
225 if (!ret)
226 return NULL;
227
228 extent_changeset_init(ret);
229 return ret;
230 }
231
extent_changeset_prealloc(struct extent_changeset * changeset,gfp_t gfp_mask)232 static inline void extent_changeset_prealloc(struct extent_changeset *changeset, gfp_t gfp_mask)
233 {
234 ASSERT(extent_changeset_tracks_ranges(changeset));
235 ulist_prealloc(&changeset->range_changed, gfp_mask);
236 }
237
extent_changeset_release(struct extent_changeset * changeset)238 static inline void extent_changeset_release(struct extent_changeset *changeset)
239 {
240 if (!changeset)
241 return;
242 changeset->bytes_changed = 0;
243 if (extent_changeset_tracks_ranges(changeset))
244 ulist_release(&changeset->range_changed);
245 }
246
extent_changeset_free(struct extent_changeset * changeset)247 static inline void extent_changeset_free(struct extent_changeset *changeset)
248 {
249 if (!changeset)
250 return;
251 extent_changeset_release(changeset);
252 kfree(changeset);
253 }
254
255 bool try_release_extent_mapping(struct folio *folio, gfp_t mask);
256 int try_release_extent_buffer(struct folio *folio);
257
258 int btrfs_read_folio(struct file *file, struct folio *folio);
259 void extent_write_locked_range(struct inode *inode, const struct folio *locked_folio,
260 u64 start, u64 end, struct writeback_control *wbc,
261 bool pages_dirty);
262 int btrfs_writepages(struct address_space *mapping, struct writeback_control *wbc);
263 int btree_writepages(struct address_space *mapping, struct writeback_control *wbc);
264 void btrfs_btree_wait_writeback_range(struct btrfs_fs_info *fs_info, u64 start, u64 end);
265 void btrfs_readahead(struct readahead_control *rac);
266 int set_folio_extent_mapped(struct folio *folio);
267 void clear_folio_extent_mapped(struct folio *folio);
268
269 struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
270 u64 start, u64 owner_root, int level);
271 struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
272 u64 start);
273 struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src);
274 struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
275 u64 start);
276 void free_extent_buffer(struct extent_buffer *eb);
277 void free_extent_buffer_stale(struct extent_buffer *eb);
278 int read_extent_buffer_pages(struct extent_buffer *eb, int mirror_num,
279 const struct btrfs_tree_parent_check *parent_check);
280 int read_extent_buffer_pages_nowait(struct extent_buffer *eb, int mirror_num,
281 const struct btrfs_tree_parent_check *parent_check);
282
wait_on_extent_buffer_writeback(struct extent_buffer * eb)283 static inline void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
284 {
285 wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
286 TASK_UNINTERRUPTIBLE);
287 }
288
289 void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
290 u64 bytenr, u64 owner_root, u64 gen, int level);
291 void btrfs_readahead_node_child(struct extent_buffer *node, int slot);
292
293 /* Note: this can be used in for loops without caching the value in a variable. */
num_extent_pages(const struct extent_buffer * eb)294 static inline int __pure num_extent_pages(const struct extent_buffer *eb)
295 {
296 /*
297 * For sectorsize == PAGE_SIZE case, since nodesize is always aligned to
298 * sectorsize, it's just eb->len >> PAGE_SHIFT.
299 *
300 * For sectorsize < PAGE_SIZE case, we could have nodesize < PAGE_SIZE,
301 * thus have to ensure we get at least one page.
302 */
303 return (eb->len >> PAGE_SHIFT) ?: 1;
304 }
305
306 /*
307 * This can only be determined at runtime by checking eb::folios[0].
308 *
309 * As we can have either one large folio covering the whole eb
310 * (either nodesize <= PAGE_SIZE, or high order folio), or multiple
311 * single-paged folios.
312 *
313 * Note: this can be used in for loops without caching the value in a variable.
314 */
num_extent_folios(const struct extent_buffer * eb)315 static inline int __pure num_extent_folios(const struct extent_buffer *eb)
316 {
317 if (!eb->folios[0])
318 return 0;
319 if (folio_order(eb->folios[0]))
320 return 1;
321 return num_extent_pages(eb);
322 }
323
extent_buffer_uptodate(const struct extent_buffer * eb)324 static inline bool extent_buffer_uptodate(const struct extent_buffer *eb)
325 {
326 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
327 }
328
329 int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
330 unsigned long start, unsigned long len);
331 void read_extent_buffer(const struct extent_buffer *eb, void *dst,
332 unsigned long start,
333 unsigned long len);
334 int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
335 void __user *dst, unsigned long start,
336 unsigned long len);
337 void write_extent_buffer(const struct extent_buffer *eb, const void *src,
338 unsigned long start, unsigned long len);
339
write_extent_buffer_chunk_tree_uuid(const struct extent_buffer * eb,const void * chunk_tree_uuid)340 static inline void write_extent_buffer_chunk_tree_uuid(
341 const struct extent_buffer *eb, const void *chunk_tree_uuid)
342 {
343 write_extent_buffer(eb, chunk_tree_uuid,
344 offsetof(struct btrfs_header, chunk_tree_uuid),
345 BTRFS_FSID_SIZE);
346 }
347
write_extent_buffer_fsid(const struct extent_buffer * eb,const void * fsid)348 static inline void write_extent_buffer_fsid(const struct extent_buffer *eb,
349 const void *fsid)
350 {
351 write_extent_buffer(eb, fsid, offsetof(struct btrfs_header, fsid),
352 BTRFS_FSID_SIZE);
353 }
354
355 void copy_extent_buffer_full(const struct extent_buffer *dst,
356 const struct extent_buffer *src);
357 void copy_extent_buffer(const struct extent_buffer *dst,
358 const struct extent_buffer *src,
359 unsigned long dst_offset, unsigned long src_offset,
360 unsigned long len);
361 void memcpy_extent_buffer(const struct extent_buffer *dst,
362 unsigned long dst_offset, unsigned long src_offset,
363 unsigned long len);
364 void memmove_extent_buffer(const struct extent_buffer *dst,
365 unsigned long dst_offset, unsigned long src_offset,
366 unsigned long len);
367 void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
368 unsigned long len);
369 bool extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
370 unsigned long pos);
371 void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
372 unsigned long pos, unsigned long len);
373 void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
374 unsigned long start, unsigned long pos,
375 unsigned long len);
376 void set_extent_buffer_dirty(struct extent_buffer *eb);
377 void set_extent_buffer_uptodate(struct extent_buffer *eb);
378 void clear_extent_buffer_uptodate(struct extent_buffer *eb);
379 void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
380 const struct folio *locked_folio,
381 struct extent_state **cached,
382 u32 bits_to_clear, unsigned long page_ops);
383 int extent_invalidate_folio(struct extent_io_tree *tree,
384 struct folio *folio, size_t offset);
385 void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
386 struct extent_buffer *buf);
387
388 int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array,
389 bool nofail);
390 int btrfs_alloc_folio_array(unsigned int nr_folios, unsigned int order,
391 struct folio **folio_array);
392
393 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
394 bool find_lock_delalloc_range(struct inode *inode,
395 struct folio *locked_folio, u64 *start,
396 u64 *end);
397 #endif
398 struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
399 u64 start);
400
401 #ifdef CONFIG_BTRFS_DEBUG
402 void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info);
403 #else
404 #define btrfs_extent_buffer_leak_debug_check(fs_info) do {} while (0)
405 #endif
406
407 void btrfs_inhibit_eb_writeback(struct btrfs_trans_handle *trans,
408 struct extent_buffer *eb);
409 void btrfs_uninhibit_all_eb_writeback(struct btrfs_trans_handle *trans);
410
411 #endif
412