1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
4 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
5 */
6
7 #include <linux/sched.h>
8 #include <linux/slab.h>
9 #include <linux/spinlock.h>
10 #include <linux/completion.h>
11 #include <linux/buffer_head.h>
12 #include <linux/mm.h>
13 #include <linux/pagemap.h>
14 #include <linux/writeback.h>
15 #include <linux/swap.h>
16 #include <linux/delay.h>
17 #include <linux/bio.h>
18 #include <linux/gfs2_ondisk.h>
19
20 #include "gfs2.h"
21 #include "incore.h"
22 #include "glock.h"
23 #include "glops.h"
24 #include "inode.h"
25 #include "log.h"
26 #include "lops.h"
27 #include "meta_io.h"
28 #include "rgrp.h"
29 #include "trans.h"
30 #include "util.h"
31 #include "trace_gfs2.h"
32
gfs2_aspace_write_folio(struct folio * folio,struct writeback_control * wbc)33 static void gfs2_aspace_write_folio(struct folio *folio,
34 struct writeback_control *wbc)
35 {
36 struct buffer_head *bh, *head;
37 int nr_underway = 0;
38 blk_opf_t write_flags = REQ_META | REQ_PRIO | wbc_to_write_flags(wbc);
39
40 BUG_ON(!folio_test_locked(folio));
41
42 head = folio_buffers(folio);
43 bh = head;
44
45 do {
46 if (!buffer_mapped(bh))
47 continue;
48 /*
49 * If it's a fully non-blocking write attempt and we cannot
50 * lock the buffer then redirty the page. Note that this can
51 * potentially cause a busy-wait loop from flusher thread and kswapd
52 * activity, but those code paths have their own higher-level
53 * throttling.
54 */
55 if (wbc->sync_mode != WB_SYNC_NONE) {
56 lock_buffer(bh);
57 } else if (!trylock_buffer(bh)) {
58 folio_redirty_for_writepage(wbc, folio);
59 continue;
60 }
61 if (test_clear_buffer_dirty(bh)) {
62 mark_buffer_async_write(bh);
63 } else {
64 unlock_buffer(bh);
65 }
66 } while ((bh = bh->b_this_page) != head);
67
68 /*
69 * The folio and its buffers are protected from truncation by
70 * the writeback flag, so we can drop the bh refcounts early.
71 */
72 BUG_ON(folio_test_writeback(folio));
73 folio_start_writeback(folio);
74
75 do {
76 struct buffer_head *next = bh->b_this_page;
77 if (buffer_async_write(bh)) {
78 submit_bh(REQ_OP_WRITE | write_flags, bh);
79 nr_underway++;
80 }
81 bh = next;
82 } while (bh != head);
83 folio_unlock(folio);
84
85 if (nr_underway == 0)
86 folio_end_writeback(folio);
87 }
88
gfs2_aspace_writepages(struct address_space * mapping,struct writeback_control * wbc)89 static int gfs2_aspace_writepages(struct address_space *mapping,
90 struct writeback_control *wbc)
91 {
92 struct folio *folio = NULL;
93 int error;
94
95 while ((folio = writeback_iter(mapping, wbc, folio, &error)))
96 gfs2_aspace_write_folio(folio, wbc);
97
98 return error;
99 }
100
101 const struct address_space_operations gfs2_meta_aops = {
102 .dirty_folio = block_dirty_folio,
103 .invalidate_folio = block_invalidate_folio,
104 .writepages = gfs2_aspace_writepages,
105 .release_folio = gfs2_release_folio,
106 };
107
108 const struct address_space_operations gfs2_rgrp_aops = {
109 .dirty_folio = block_dirty_folio,
110 .invalidate_folio = block_invalidate_folio,
111 .writepages = gfs2_aspace_writepages,
112 .release_folio = gfs2_release_folio,
113 };
114
115 /**
116 * gfs2_getbuf - Get a buffer with a given address space
117 * @gl: the glock
118 * @blkno: the block number (filesystem scope)
119 * @create: 1 if the buffer should be created
120 *
121 * Returns: the buffer
122 */
123
gfs2_getbuf(struct gfs2_glock * gl,u64 blkno,int create)124 struct buffer_head *gfs2_getbuf(struct gfs2_glock *gl, u64 blkno, int create)
125 {
126 struct address_space *mapping = gfs2_glock2aspace(gl);
127 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
128 struct folio *folio;
129 struct buffer_head *bh;
130 unsigned int shift;
131 unsigned long index;
132 unsigned int bufnum;
133
134 if (mapping == NULL)
135 mapping = &sdp->sd_aspace;
136
137 shift = PAGE_SHIFT - sdp->sd_sb.sb_bsize_shift;
138 index = blkno >> shift; /* convert block to page */
139 bufnum = blkno - (index << shift); /* block buf index within page */
140
141 if (create) {
142 folio = __filemap_get_folio(mapping, index,
143 FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
144 mapping_gfp_mask(mapping) | __GFP_NOFAIL);
145 bh = folio_buffers(folio);
146 if (!bh)
147 bh = create_empty_buffers(folio,
148 sdp->sd_sb.sb_bsize, 0);
149 } else {
150 folio = __filemap_get_folio(mapping, index,
151 FGP_LOCK | FGP_ACCESSED, 0);
152 if (IS_ERR(folio))
153 return NULL;
154 bh = folio_buffers(folio);
155 }
156
157 if (!bh)
158 goto out_unlock;
159
160 bh = get_nth_bh(bh, bufnum);
161 if (!buffer_mapped(bh))
162 map_bh(bh, sdp->sd_vfs, blkno);
163
164 out_unlock:
165 folio_unlock(folio);
166 folio_put(folio);
167
168 return bh;
169 }
170
meta_prep_new(struct buffer_head * bh)171 static void meta_prep_new(struct buffer_head *bh)
172 {
173 struct gfs2_meta_header *mh = (struct gfs2_meta_header *)bh->b_data;
174
175 lock_buffer(bh);
176 clear_buffer_dirty(bh);
177 set_buffer_uptodate(bh);
178 unlock_buffer(bh);
179
180 mh->mh_magic = cpu_to_be32(GFS2_MAGIC);
181 }
182
183 /**
184 * gfs2_meta_new - Get a block
185 * @gl: The glock associated with this block
186 * @blkno: The block number
187 *
188 * Returns: The buffer
189 */
190
gfs2_meta_new(struct gfs2_glock * gl,u64 blkno)191 struct buffer_head *gfs2_meta_new(struct gfs2_glock *gl, u64 blkno)
192 {
193 struct buffer_head *bh;
194 bh = gfs2_getbuf(gl, blkno, CREATE);
195 meta_prep_new(bh);
196 return bh;
197 }
198
gfs2_meta_read_endio(struct bio * bio)199 static void gfs2_meta_read_endio(struct bio *bio)
200 {
201 struct folio_iter fi;
202
203 bio_for_each_folio_all(fi, bio) {
204 struct folio *folio = fi.folio;
205 struct buffer_head *bh = folio_buffers(folio);
206 size_t len = fi.length;
207
208 while (bh_offset(bh) < fi.offset)
209 bh = bh->b_this_page;
210 do {
211 struct buffer_head *next = bh->b_this_page;
212 len -= bh->b_size;
213 bh->b_end_io(bh, !bio->bi_status);
214 bh = next;
215 } while (bh && len);
216 }
217 bio_put(bio);
218 }
219
220 /*
221 * Submit several consecutive buffer head I/O requests as a single bio I/O
222 * request. (See submit_bh_wbc.)
223 */
gfs2_submit_bhs(blk_opf_t opf,struct buffer_head * bhs[],int num)224 static void gfs2_submit_bhs(blk_opf_t opf, struct buffer_head *bhs[], int num)
225 {
226 while (num > 0) {
227 struct buffer_head *bh = *bhs;
228 struct bio *bio;
229
230 bio = bio_alloc(bh->b_bdev, num, opf, GFP_NOIO);
231 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
232 while (num > 0) {
233 bh = *bhs;
234 if (!bio_add_folio(bio, bh->b_folio, bh->b_size, bh_offset(bh))) {
235 BUG_ON(bio->bi_iter.bi_size == 0);
236 break;
237 }
238 bhs++;
239 num--;
240 }
241 bio->bi_end_io = gfs2_meta_read_endio;
242 submit_bio(bio);
243 }
244 }
245
246 /**
247 * gfs2_meta_read - Read a block from disk
248 * @gl: The glock covering the block
249 * @blkno: The block number
250 * @flags: flags
251 * @rahead: Do read-ahead
252 * @bhp: the place where the buffer is returned (NULL on failure)
253 *
254 * Returns: errno
255 */
256
gfs2_meta_read(struct gfs2_glock * gl,u64 blkno,int flags,int rahead,struct buffer_head ** bhp)257 int gfs2_meta_read(struct gfs2_glock *gl, u64 blkno, int flags,
258 int rahead, struct buffer_head **bhp)
259 {
260 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
261 struct buffer_head *bh, *bhs[2];
262 int num = 0;
263
264 if (gfs2_withdrawing_or_withdrawn(sdp) &&
265 !gfs2_withdraw_in_prog(sdp)) {
266 *bhp = NULL;
267 return -EIO;
268 }
269
270 *bhp = bh = gfs2_getbuf(gl, blkno, CREATE);
271
272 lock_buffer(bh);
273 if (buffer_uptodate(bh)) {
274 unlock_buffer(bh);
275 flags &= ~DIO_WAIT;
276 } else {
277 bh->b_end_io = end_buffer_read_sync;
278 get_bh(bh);
279 bhs[num++] = bh;
280 }
281
282 if (rahead) {
283 bh = gfs2_getbuf(gl, blkno + 1, CREATE);
284
285 lock_buffer(bh);
286 if (buffer_uptodate(bh)) {
287 unlock_buffer(bh);
288 brelse(bh);
289 } else {
290 bh->b_end_io = end_buffer_read_sync;
291 bhs[num++] = bh;
292 }
293 }
294
295 gfs2_submit_bhs(REQ_OP_READ | REQ_META | REQ_PRIO, bhs, num);
296 if (!(flags & DIO_WAIT))
297 return 0;
298
299 bh = *bhp;
300 wait_on_buffer(bh);
301 if (unlikely(!buffer_uptodate(bh))) {
302 struct gfs2_trans *tr = current->journal_info;
303 if (tr && test_bit(TR_TOUCHED, &tr->tr_flags))
304 gfs2_io_error_bh_wd(sdp, bh);
305 brelse(bh);
306 *bhp = NULL;
307 return -EIO;
308 }
309
310 return 0;
311 }
312
313 /**
314 * gfs2_meta_wait - Reread a block from disk
315 * @sdp: the filesystem
316 * @bh: The block to wait for
317 *
318 * Returns: errno
319 */
320
gfs2_meta_wait(struct gfs2_sbd * sdp,struct buffer_head * bh)321 int gfs2_meta_wait(struct gfs2_sbd *sdp, struct buffer_head *bh)
322 {
323 if (gfs2_withdrawing_or_withdrawn(sdp) &&
324 !gfs2_withdraw_in_prog(sdp))
325 return -EIO;
326
327 wait_on_buffer(bh);
328
329 if (!buffer_uptodate(bh)) {
330 struct gfs2_trans *tr = current->journal_info;
331 if (tr && test_bit(TR_TOUCHED, &tr->tr_flags))
332 gfs2_io_error_bh_wd(sdp, bh);
333 return -EIO;
334 }
335 if (gfs2_withdrawing_or_withdrawn(sdp) &&
336 !gfs2_withdraw_in_prog(sdp))
337 return -EIO;
338
339 return 0;
340 }
341
gfs2_remove_from_journal(struct buffer_head * bh,int meta)342 void gfs2_remove_from_journal(struct buffer_head *bh, int meta)
343 {
344 struct address_space *mapping = bh->b_folio->mapping;
345 struct gfs2_sbd *sdp = gfs2_mapping2sbd(mapping);
346 struct gfs2_bufdata *bd = bh->b_private;
347 struct gfs2_trans *tr = current->journal_info;
348 int was_pinned = 0;
349
350 if (test_clear_buffer_pinned(bh)) {
351 trace_gfs2_pin(bd, 0);
352 atomic_dec(&sdp->sd_log_pinned);
353 list_del_init(&bd->bd_list);
354 if (meta == REMOVE_META)
355 tr->tr_num_buf_rm++;
356 else
357 tr->tr_num_databuf_rm++;
358 set_bit(TR_TOUCHED, &tr->tr_flags);
359 was_pinned = 1;
360 brelse(bh);
361 }
362 if (bd) {
363 if (bd->bd_tr) {
364 gfs2_trans_add_revoke(sdp, bd);
365 } else if (was_pinned) {
366 bh->b_private = NULL;
367 kmem_cache_free(gfs2_bufdata_cachep, bd);
368 } else if (!list_empty(&bd->bd_ail_st_list) &&
369 !list_empty(&bd->bd_ail_gl_list)) {
370 gfs2_remove_from_ail(bd);
371 }
372 }
373 clear_buffer_dirty(bh);
374 clear_buffer_uptodate(bh);
375 }
376
377 /**
378 * gfs2_ail1_wipe - remove deleted/freed buffers from the ail1 list
379 * @sdp: superblock
380 * @bstart: starting block address of buffers to remove
381 * @blen: length of buffers to be removed
382 *
383 * This function is called from gfs2_journal wipe, whose job is to remove
384 * buffers, corresponding to deleted blocks, from the journal. If we find any
385 * bufdata elements on the system ail1 list, they haven't been written to
386 * the journal yet. So we remove them.
387 */
gfs2_ail1_wipe(struct gfs2_sbd * sdp,u64 bstart,u32 blen)388 static void gfs2_ail1_wipe(struct gfs2_sbd *sdp, u64 bstart, u32 blen)
389 {
390 struct gfs2_trans *tr, *s;
391 struct gfs2_bufdata *bd, *bs;
392 struct buffer_head *bh;
393 u64 end = bstart + blen;
394
395 gfs2_log_lock(sdp);
396 spin_lock(&sdp->sd_ail_lock);
397 list_for_each_entry_safe(tr, s, &sdp->sd_ail1_list, tr_list) {
398 list_for_each_entry_safe(bd, bs, &tr->tr_ail1_list,
399 bd_ail_st_list) {
400 bh = bd->bd_bh;
401 if (bh->b_blocknr < bstart || bh->b_blocknr >= end)
402 continue;
403
404 gfs2_remove_from_journal(bh, REMOVE_JDATA);
405 }
406 }
407 spin_unlock(&sdp->sd_ail_lock);
408 gfs2_log_unlock(sdp);
409 }
410
gfs2_getjdatabuf(struct gfs2_inode * ip,u64 blkno)411 static struct buffer_head *gfs2_getjdatabuf(struct gfs2_inode *ip, u64 blkno)
412 {
413 struct address_space *mapping = ip->i_inode.i_mapping;
414 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
415 struct folio *folio;
416 struct buffer_head *bh;
417 unsigned int shift = PAGE_SHIFT - sdp->sd_sb.sb_bsize_shift;
418 unsigned long index = blkno >> shift; /* convert block to page */
419 unsigned int bufnum = blkno - (index << shift);
420
421 folio = __filemap_get_folio(mapping, index, FGP_LOCK | FGP_ACCESSED, 0);
422 if (IS_ERR(folio))
423 return NULL;
424 bh = folio_buffers(folio);
425 if (bh)
426 bh = get_nth_bh(bh, bufnum);
427 folio_unlock(folio);
428 folio_put(folio);
429 return bh;
430 }
431
432 /**
433 * gfs2_journal_wipe - make inode's buffers so they aren't dirty/pinned anymore
434 * @ip: the inode who owns the buffers
435 * @bstart: the first buffer in the run
436 * @blen: the number of buffers in the run
437 *
438 */
439
gfs2_journal_wipe(struct gfs2_inode * ip,u64 bstart,u32 blen)440 void gfs2_journal_wipe(struct gfs2_inode *ip, u64 bstart, u32 blen)
441 {
442 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
443 struct buffer_head *bh;
444 int ty;
445
446 if (!ip->i_gl) {
447 /* This can only happen during incomplete inode creation. */
448 BUG_ON(!test_bit(GIF_ALLOC_FAILED, &ip->i_flags));
449 return;
450 }
451
452 gfs2_ail1_wipe(sdp, bstart, blen);
453 while (blen) {
454 ty = REMOVE_META;
455 bh = gfs2_getbuf(ip->i_gl, bstart, NO_CREATE);
456 if (!bh && gfs2_is_jdata(ip)) {
457 bh = gfs2_getjdatabuf(ip, bstart);
458 ty = REMOVE_JDATA;
459 }
460 if (bh) {
461 lock_buffer(bh);
462 gfs2_log_lock(sdp);
463 spin_lock(&sdp->sd_ail_lock);
464 gfs2_remove_from_journal(bh, ty);
465 spin_unlock(&sdp->sd_ail_lock);
466 gfs2_log_unlock(sdp);
467 unlock_buffer(bh);
468 brelse(bh);
469 }
470
471 bstart++;
472 blen--;
473 }
474 }
475
476 /**
477 * gfs2_meta_buffer - Get a metadata buffer
478 * @ip: The GFS2 inode
479 * @mtype: The block type (GFS2_METATYPE_*)
480 * @num: The block number (device relative) of the buffer
481 * @bhp: the buffer is returned here
482 *
483 * Returns: errno
484 */
485
gfs2_meta_buffer(struct gfs2_inode * ip,u32 mtype,u64 num,struct buffer_head ** bhp)486 int gfs2_meta_buffer(struct gfs2_inode *ip, u32 mtype, u64 num,
487 struct buffer_head **bhp)
488 {
489 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
490 struct gfs2_glock *gl = ip->i_gl;
491 struct buffer_head *bh;
492 int ret = 0;
493 int rahead = 0;
494
495 if (num == ip->i_no_addr)
496 rahead = ip->i_rahead;
497
498 ret = gfs2_meta_read(gl, num, DIO_WAIT, rahead, &bh);
499 if (ret == 0 && gfs2_metatype_check(sdp, bh, mtype)) {
500 brelse(bh);
501 ret = -EIO;
502 } else {
503 *bhp = bh;
504 }
505 return ret;
506 }
507
508 /**
509 * gfs2_meta_ra - start readahead on an extent of a file
510 * @gl: the glock the blocks belong to
511 * @dblock: the starting disk block
512 * @extlen: the number of blocks in the extent
513 *
514 * returns: the first buffer in the extent
515 */
516
gfs2_meta_ra(struct gfs2_glock * gl,u64 dblock,u32 extlen)517 struct buffer_head *gfs2_meta_ra(struct gfs2_glock *gl, u64 dblock, u32 extlen)
518 {
519 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
520 struct buffer_head *first_bh, *bh;
521 u32 max_ra = gfs2_tune_get(sdp, gt_max_readahead) >>
522 sdp->sd_sb.sb_bsize_shift;
523
524 BUG_ON(!extlen);
525
526 if (max_ra < 1)
527 max_ra = 1;
528 if (extlen > max_ra)
529 extlen = max_ra;
530
531 first_bh = gfs2_getbuf(gl, dblock, CREATE);
532
533 if (buffer_uptodate(first_bh))
534 goto out;
535 bh_read_nowait(first_bh, REQ_META | REQ_PRIO);
536
537 dblock++;
538 extlen--;
539
540 while (extlen) {
541 bh = gfs2_getbuf(gl, dblock, CREATE);
542
543 bh_readahead(bh, REQ_RAHEAD | REQ_META | REQ_PRIO);
544 brelse(bh);
545 dblock++;
546 extlen--;
547 if (!buffer_locked(first_bh) && buffer_uptodate(first_bh))
548 goto out;
549 }
550
551 wait_on_buffer(first_bh);
552 out:
553 return first_bh;
554 }
555
556