1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 1991, 1992 Linus Torvalds 4 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE 5 * Copyright (C) 2016 - 2020 Christoph Hellwig 6 */ 7 8 #include <linux/init.h> 9 #include <linux/mm.h> 10 #include <linux/slab.h> 11 #include <linux/kmod.h> 12 #include <linux/major.h> 13 #include <linux/device_cgroup.h> 14 #include <linux/blkdev.h> 15 #include <linux/blk-integrity.h> 16 #include <linux/backing-dev.h> 17 #include <linux/module.h> 18 #include <linux/blkpg.h> 19 #include <linux/magic.h> 20 #include <linux/buffer_head.h> 21 #include <linux/swap.h> 22 #include <linux/writeback.h> 23 #include <linux/mount.h> 24 #include <linux/pseudo_fs.h> 25 #include <linux/uio.h> 26 #include <linux/namei.h> 27 #include <linux/security.h> 28 #include <linux/part_stat.h> 29 #include <linux/uaccess.h> 30 #include <linux/stat.h> 31 #include "../fs/internal.h" 32 #include "blk.h" 33 34 /* Should we allow writing to mounted block devices? */ 35 static bool bdev_allow_write_mounted = IS_ENABLED(CONFIG_BLK_DEV_WRITE_MOUNTED); 36 37 struct bdev_inode { 38 struct block_device bdev; 39 struct inode vfs_inode; 40 }; 41 42 static inline struct bdev_inode *BDEV_I(struct inode *inode) 43 { 44 return container_of(inode, struct bdev_inode, vfs_inode); 45 } 46 47 static inline struct inode *BD_INODE(struct block_device *bdev) 48 { 49 return &container_of(bdev, struct bdev_inode, bdev)->vfs_inode; 50 } 51 52 struct block_device *I_BDEV(struct inode *inode) 53 { 54 return &BDEV_I(inode)->bdev; 55 } 56 EXPORT_SYMBOL(I_BDEV); 57 58 struct block_device *file_bdev(struct file *bdev_file) 59 { 60 return I_BDEV(bdev_file->f_mapping->host); 61 } 62 EXPORT_SYMBOL(file_bdev); 63 64 static void bdev_write_inode(struct block_device *bdev) 65 { 66 struct inode *inode = BD_INODE(bdev); 67 int ret; 68 69 spin_lock(&inode->i_lock); 70 while (inode->i_state & I_DIRTY) { 71 spin_unlock(&inode->i_lock); 72 ret = write_inode_now(inode, true); 73 if (ret) 74 pr_warn_ratelimited( 75 "VFS: Dirty inode writeback failed for block device %pg (err=%d).\n", 76 bdev, ret); 77 spin_lock(&inode->i_lock); 78 } 79 spin_unlock(&inode->i_lock); 80 } 81 82 /* Kill _all_ buffers and pagecache , dirty or not.. */ 83 static void kill_bdev(struct block_device *bdev) 84 { 85 struct address_space *mapping = bdev->bd_mapping; 86 87 if (mapping_empty(mapping)) 88 return; 89 90 invalidate_bh_lrus(); 91 truncate_inode_pages(mapping, 0); 92 } 93 94 /* Invalidate clean unused buffers and pagecache. */ 95 void invalidate_bdev(struct block_device *bdev) 96 { 97 struct address_space *mapping = bdev->bd_mapping; 98 99 if (mapping->nrpages) { 100 invalidate_bh_lrus(); 101 lru_add_drain_all(); /* make sure all lru add caches are flushed */ 102 invalidate_mapping_pages(mapping, 0, -1); 103 } 104 } 105 EXPORT_SYMBOL(invalidate_bdev); 106 107 /* 108 * Drop all buffers & page cache for given bdev range. This function bails 109 * with error if bdev has other exclusive owner (such as filesystem). 110 */ 111 int truncate_bdev_range(struct block_device *bdev, blk_mode_t mode, 112 loff_t lstart, loff_t lend) 113 { 114 /* 115 * If we don't hold exclusive handle for the device, upgrade to it 116 * while we discard the buffer cache to avoid discarding buffers 117 * under live filesystem. 118 */ 119 if (!(mode & BLK_OPEN_EXCL)) { 120 int err = bd_prepare_to_claim(bdev, truncate_bdev_range, NULL); 121 if (err) 122 goto invalidate; 123 } 124 125 truncate_inode_pages_range(bdev->bd_mapping, lstart, lend); 126 if (!(mode & BLK_OPEN_EXCL)) 127 bd_abort_claiming(bdev, truncate_bdev_range); 128 return 0; 129 130 invalidate: 131 /* 132 * Someone else has handle exclusively open. Try invalidating instead. 133 * The 'end' argument is inclusive so the rounding is safe. 134 */ 135 return invalidate_inode_pages2_range(bdev->bd_mapping, 136 lstart >> PAGE_SHIFT, 137 lend >> PAGE_SHIFT); 138 } 139 140 static void set_init_blocksize(struct block_device *bdev) 141 { 142 unsigned int bsize = bdev_logical_block_size(bdev); 143 loff_t size = i_size_read(BD_INODE(bdev)); 144 145 while (bsize < PAGE_SIZE) { 146 if (size & bsize) 147 break; 148 bsize <<= 1; 149 } 150 BD_INODE(bdev)->i_blkbits = blksize_bits(bsize); 151 mapping_set_folio_min_order(BD_INODE(bdev)->i_mapping, 152 get_order(bsize)); 153 } 154 155 int set_blocksize(struct file *file, int size) 156 { 157 struct inode *inode = file->f_mapping->host; 158 struct block_device *bdev = I_BDEV(inode); 159 160 if (blk_validate_block_size(size)) 161 return -EINVAL; 162 163 /* Size cannot be smaller than the size supported by the device */ 164 if (size < bdev_logical_block_size(bdev)) 165 return -EINVAL; 166 167 if (!file->private_data) 168 return -EINVAL; 169 170 /* Don't change the size if it is same as current */ 171 if (inode->i_blkbits != blksize_bits(size)) { 172 sync_blockdev(bdev); 173 inode->i_blkbits = blksize_bits(size); 174 mapping_set_folio_min_order(inode->i_mapping, get_order(size)); 175 kill_bdev(bdev); 176 } 177 return 0; 178 } 179 180 EXPORT_SYMBOL(set_blocksize); 181 182 int sb_set_blocksize(struct super_block *sb, int size) 183 { 184 if (!(sb->s_type->fs_flags & FS_LBS) && size > PAGE_SIZE) 185 return 0; 186 if (set_blocksize(sb->s_bdev_file, size)) 187 return 0; 188 /* If we get here, we know size is validated */ 189 sb->s_blocksize = size; 190 sb->s_blocksize_bits = blksize_bits(size); 191 return sb->s_blocksize; 192 } 193 194 EXPORT_SYMBOL(sb_set_blocksize); 195 196 int sb_min_blocksize(struct super_block *sb, int size) 197 { 198 int minsize = bdev_logical_block_size(sb->s_bdev); 199 if (size < minsize) 200 size = minsize; 201 return sb_set_blocksize(sb, size); 202 } 203 204 EXPORT_SYMBOL(sb_min_blocksize); 205 206 int sync_blockdev_nowait(struct block_device *bdev) 207 { 208 if (!bdev) 209 return 0; 210 return filemap_flush(bdev->bd_mapping); 211 } 212 EXPORT_SYMBOL_GPL(sync_blockdev_nowait); 213 214 /* 215 * Write out and wait upon all the dirty data associated with a block 216 * device via its mapping. Does not take the superblock lock. 217 */ 218 int sync_blockdev(struct block_device *bdev) 219 { 220 if (!bdev) 221 return 0; 222 return filemap_write_and_wait(bdev->bd_mapping); 223 } 224 EXPORT_SYMBOL(sync_blockdev); 225 226 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend) 227 { 228 return filemap_write_and_wait_range(bdev->bd_mapping, 229 lstart, lend); 230 } 231 EXPORT_SYMBOL(sync_blockdev_range); 232 233 /** 234 * bdev_freeze - lock a filesystem and force it into a consistent state 235 * @bdev: blockdevice to lock 236 * 237 * If a superblock is found on this device, we take the s_umount semaphore 238 * on it to make sure nobody unmounts until the snapshot creation is done. 239 * The reference counter (bd_fsfreeze_count) guarantees that only the last 240 * unfreeze process can unfreeze the frozen filesystem actually when multiple 241 * freeze requests arrive simultaneously. It counts up in bdev_freeze() and 242 * count down in bdev_thaw(). When it becomes 0, thaw_bdev() will unfreeze 243 * actually. 244 * 245 * Return: On success zero is returned, negative error code on failure. 246 */ 247 int bdev_freeze(struct block_device *bdev) 248 { 249 int error = 0; 250 251 mutex_lock(&bdev->bd_fsfreeze_mutex); 252 253 if (atomic_inc_return(&bdev->bd_fsfreeze_count) > 1) { 254 mutex_unlock(&bdev->bd_fsfreeze_mutex); 255 return 0; 256 } 257 258 mutex_lock(&bdev->bd_holder_lock); 259 if (bdev->bd_holder_ops && bdev->bd_holder_ops->freeze) { 260 error = bdev->bd_holder_ops->freeze(bdev); 261 lockdep_assert_not_held(&bdev->bd_holder_lock); 262 } else { 263 mutex_unlock(&bdev->bd_holder_lock); 264 error = sync_blockdev(bdev); 265 } 266 267 if (error) 268 atomic_dec(&bdev->bd_fsfreeze_count); 269 270 mutex_unlock(&bdev->bd_fsfreeze_mutex); 271 return error; 272 } 273 EXPORT_SYMBOL(bdev_freeze); 274 275 /** 276 * bdev_thaw - unlock filesystem 277 * @bdev: blockdevice to unlock 278 * 279 * Unlocks the filesystem and marks it writeable again after bdev_freeze(). 280 * 281 * Return: On success zero is returned, negative error code on failure. 282 */ 283 int bdev_thaw(struct block_device *bdev) 284 { 285 int error = -EINVAL, nr_freeze; 286 287 mutex_lock(&bdev->bd_fsfreeze_mutex); 288 289 /* 290 * If this returns < 0 it means that @bd_fsfreeze_count was 291 * already 0 and no decrement was performed. 292 */ 293 nr_freeze = atomic_dec_if_positive(&bdev->bd_fsfreeze_count); 294 if (nr_freeze < 0) 295 goto out; 296 297 error = 0; 298 if (nr_freeze > 0) 299 goto out; 300 301 mutex_lock(&bdev->bd_holder_lock); 302 if (bdev->bd_holder_ops && bdev->bd_holder_ops->thaw) { 303 error = bdev->bd_holder_ops->thaw(bdev); 304 lockdep_assert_not_held(&bdev->bd_holder_lock); 305 } else { 306 mutex_unlock(&bdev->bd_holder_lock); 307 } 308 309 if (error) 310 atomic_inc(&bdev->bd_fsfreeze_count); 311 out: 312 mutex_unlock(&bdev->bd_fsfreeze_mutex); 313 return error; 314 } 315 EXPORT_SYMBOL(bdev_thaw); 316 317 /* 318 * pseudo-fs 319 */ 320 321 static __cacheline_aligned_in_smp DEFINE_MUTEX(bdev_lock); 322 static struct kmem_cache *bdev_cachep __ro_after_init; 323 324 static struct inode *bdev_alloc_inode(struct super_block *sb) 325 { 326 struct bdev_inode *ei = alloc_inode_sb(sb, bdev_cachep, GFP_KERNEL); 327 328 if (!ei) 329 return NULL; 330 memset(&ei->bdev, 0, sizeof(ei->bdev)); 331 332 if (security_bdev_alloc(&ei->bdev)) { 333 kmem_cache_free(bdev_cachep, ei); 334 return NULL; 335 } 336 return &ei->vfs_inode; 337 } 338 339 static void bdev_free_inode(struct inode *inode) 340 { 341 struct block_device *bdev = I_BDEV(inode); 342 343 free_percpu(bdev->bd_stats); 344 kfree(bdev->bd_meta_info); 345 security_bdev_free(bdev); 346 347 if (!bdev_is_partition(bdev)) { 348 if (bdev->bd_disk && bdev->bd_disk->bdi) 349 bdi_put(bdev->bd_disk->bdi); 350 kfree(bdev->bd_disk); 351 } 352 353 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR) 354 blk_free_ext_minor(MINOR(bdev->bd_dev)); 355 356 kmem_cache_free(bdev_cachep, BDEV_I(inode)); 357 } 358 359 static void init_once(void *data) 360 { 361 struct bdev_inode *ei = data; 362 363 inode_init_once(&ei->vfs_inode); 364 } 365 366 static void bdev_evict_inode(struct inode *inode) 367 { 368 truncate_inode_pages_final(&inode->i_data); 369 invalidate_inode_buffers(inode); /* is it needed here? */ 370 clear_inode(inode); 371 } 372 373 static const struct super_operations bdev_sops = { 374 .statfs = simple_statfs, 375 .alloc_inode = bdev_alloc_inode, 376 .free_inode = bdev_free_inode, 377 .drop_inode = generic_delete_inode, 378 .evict_inode = bdev_evict_inode, 379 }; 380 381 static int bd_init_fs_context(struct fs_context *fc) 382 { 383 struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC); 384 if (!ctx) 385 return -ENOMEM; 386 fc->s_iflags |= SB_I_CGROUPWB; 387 ctx->ops = &bdev_sops; 388 return 0; 389 } 390 391 static struct file_system_type bd_type = { 392 .name = "bdev", 393 .init_fs_context = bd_init_fs_context, 394 .kill_sb = kill_anon_super, 395 }; 396 397 struct super_block *blockdev_superblock __ro_after_init; 398 static struct vfsmount *blockdev_mnt __ro_after_init; 399 EXPORT_SYMBOL_GPL(blockdev_superblock); 400 401 void __init bdev_cache_init(void) 402 { 403 int err; 404 405 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode), 406 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| 407 SLAB_ACCOUNT|SLAB_PANIC), 408 init_once); 409 err = register_filesystem(&bd_type); 410 if (err) 411 panic("Cannot register bdev pseudo-fs"); 412 blockdev_mnt = kern_mount(&bd_type); 413 if (IS_ERR(blockdev_mnt)) 414 panic("Cannot create bdev pseudo-fs"); 415 blockdev_superblock = blockdev_mnt->mnt_sb; /* For writeback */ 416 } 417 418 struct block_device *bdev_alloc(struct gendisk *disk, u8 partno) 419 { 420 struct block_device *bdev; 421 struct inode *inode; 422 423 inode = new_inode(blockdev_superblock); 424 if (!inode) 425 return NULL; 426 inode->i_mode = S_IFBLK; 427 inode->i_rdev = 0; 428 inode->i_data.a_ops = &def_blk_aops; 429 mapping_set_gfp_mask(&inode->i_data, GFP_USER); 430 431 bdev = I_BDEV(inode); 432 mutex_init(&bdev->bd_fsfreeze_mutex); 433 spin_lock_init(&bdev->bd_size_lock); 434 mutex_init(&bdev->bd_holder_lock); 435 atomic_set(&bdev->__bd_flags, partno); 436 bdev->bd_mapping = &inode->i_data; 437 bdev->bd_queue = disk->queue; 438 if (partno && bdev_test_flag(disk->part0, BD_HAS_SUBMIT_BIO)) 439 bdev_set_flag(bdev, BD_HAS_SUBMIT_BIO); 440 bdev->bd_stats = alloc_percpu(struct disk_stats); 441 if (!bdev->bd_stats) { 442 iput(inode); 443 return NULL; 444 } 445 bdev->bd_disk = disk; 446 return bdev; 447 } 448 449 void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors) 450 { 451 spin_lock(&bdev->bd_size_lock); 452 i_size_write(BD_INODE(bdev), (loff_t)sectors << SECTOR_SHIFT); 453 bdev->bd_nr_sectors = sectors; 454 spin_unlock(&bdev->bd_size_lock); 455 } 456 457 void bdev_add(struct block_device *bdev, dev_t dev) 458 { 459 struct inode *inode = BD_INODE(bdev); 460 if (bdev_stable_writes(bdev)) 461 mapping_set_stable_writes(bdev->bd_mapping); 462 bdev->bd_dev = dev; 463 inode->i_rdev = dev; 464 inode->i_ino = dev; 465 insert_inode_hash(inode); 466 } 467 468 void bdev_unhash(struct block_device *bdev) 469 { 470 remove_inode_hash(BD_INODE(bdev)); 471 } 472 473 void bdev_drop(struct block_device *bdev) 474 { 475 iput(BD_INODE(bdev)); 476 } 477 478 long nr_blockdev_pages(void) 479 { 480 struct inode *inode; 481 long ret = 0; 482 483 spin_lock(&blockdev_superblock->s_inode_list_lock); 484 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) 485 ret += inode->i_mapping->nrpages; 486 spin_unlock(&blockdev_superblock->s_inode_list_lock); 487 488 return ret; 489 } 490 491 /** 492 * bd_may_claim - test whether a block device can be claimed 493 * @bdev: block device of interest 494 * @holder: holder trying to claim @bdev 495 * @hops: holder ops 496 * 497 * Test whether @bdev can be claimed by @holder. 498 * 499 * RETURNS: 500 * %true if @bdev can be claimed, %false otherwise. 501 */ 502 static bool bd_may_claim(struct block_device *bdev, void *holder, 503 const struct blk_holder_ops *hops) 504 { 505 struct block_device *whole = bdev_whole(bdev); 506 507 lockdep_assert_held(&bdev_lock); 508 509 if (bdev->bd_holder) { 510 /* 511 * The same holder can always re-claim. 512 */ 513 if (bdev->bd_holder == holder) { 514 if (WARN_ON_ONCE(bdev->bd_holder_ops != hops)) 515 return false; 516 return true; 517 } 518 return false; 519 } 520 521 /* 522 * If the whole devices holder is set to bd_may_claim, a partition on 523 * the device is claimed, but not the whole device. 524 */ 525 if (whole != bdev && 526 whole->bd_holder && whole->bd_holder != bd_may_claim) 527 return false; 528 return true; 529 } 530 531 /** 532 * bd_prepare_to_claim - claim a block device 533 * @bdev: block device of interest 534 * @holder: holder trying to claim @bdev 535 * @hops: holder ops. 536 * 537 * Claim @bdev. This function fails if @bdev is already claimed by another 538 * holder and waits if another claiming is in progress. return, the caller 539 * has ownership of bd_claiming and bd_holder[s]. 540 * 541 * RETURNS: 542 * 0 if @bdev can be claimed, -EBUSY otherwise. 543 */ 544 int bd_prepare_to_claim(struct block_device *bdev, void *holder, 545 const struct blk_holder_ops *hops) 546 { 547 struct block_device *whole = bdev_whole(bdev); 548 549 if (WARN_ON_ONCE(!holder)) 550 return -EINVAL; 551 retry: 552 mutex_lock(&bdev_lock); 553 /* if someone else claimed, fail */ 554 if (!bd_may_claim(bdev, holder, hops)) { 555 mutex_unlock(&bdev_lock); 556 return -EBUSY; 557 } 558 559 /* if claiming is already in progress, wait for it to finish */ 560 if (whole->bd_claiming) { 561 wait_queue_head_t *wq = __var_waitqueue(&whole->bd_claiming); 562 DEFINE_WAIT(wait); 563 564 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); 565 mutex_unlock(&bdev_lock); 566 schedule(); 567 finish_wait(wq, &wait); 568 goto retry; 569 } 570 571 /* yay, all mine */ 572 whole->bd_claiming = holder; 573 mutex_unlock(&bdev_lock); 574 return 0; 575 } 576 EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */ 577 578 static void bd_clear_claiming(struct block_device *whole, void *holder) 579 { 580 lockdep_assert_held(&bdev_lock); 581 /* tell others that we're done */ 582 BUG_ON(whole->bd_claiming != holder); 583 whole->bd_claiming = NULL; 584 wake_up_var(&whole->bd_claiming); 585 } 586 587 /** 588 * bd_finish_claiming - finish claiming of a block device 589 * @bdev: block device of interest 590 * @holder: holder that has claimed @bdev 591 * @hops: block device holder operations 592 * 593 * Finish exclusive open of a block device. Mark the device as exlusively 594 * open by the holder and wake up all waiters for exclusive open to finish. 595 */ 596 static void bd_finish_claiming(struct block_device *bdev, void *holder, 597 const struct blk_holder_ops *hops) 598 { 599 struct block_device *whole = bdev_whole(bdev); 600 601 mutex_lock(&bdev_lock); 602 BUG_ON(!bd_may_claim(bdev, holder, hops)); 603 /* 604 * Note that for a whole device bd_holders will be incremented twice, 605 * and bd_holder will be set to bd_may_claim before being set to holder 606 */ 607 whole->bd_holders++; 608 whole->bd_holder = bd_may_claim; 609 bdev->bd_holders++; 610 mutex_lock(&bdev->bd_holder_lock); 611 bdev->bd_holder = holder; 612 bdev->bd_holder_ops = hops; 613 mutex_unlock(&bdev->bd_holder_lock); 614 bd_clear_claiming(whole, holder); 615 mutex_unlock(&bdev_lock); 616 } 617 618 /** 619 * bd_abort_claiming - abort claiming of a block device 620 * @bdev: block device of interest 621 * @holder: holder that has claimed @bdev 622 * 623 * Abort claiming of a block device when the exclusive open failed. This can be 624 * also used when exclusive open is not actually desired and we just needed 625 * to block other exclusive openers for a while. 626 */ 627 void bd_abort_claiming(struct block_device *bdev, void *holder) 628 { 629 mutex_lock(&bdev_lock); 630 bd_clear_claiming(bdev_whole(bdev), holder); 631 mutex_unlock(&bdev_lock); 632 } 633 EXPORT_SYMBOL(bd_abort_claiming); 634 635 static void bd_end_claim(struct block_device *bdev, void *holder) 636 { 637 struct block_device *whole = bdev_whole(bdev); 638 bool unblock = false; 639 640 /* 641 * Release a claim on the device. The holder fields are protected with 642 * bdev_lock. open_mutex is used to synchronize disk_holder unlinking. 643 */ 644 mutex_lock(&bdev_lock); 645 WARN_ON_ONCE(bdev->bd_holder != holder); 646 WARN_ON_ONCE(--bdev->bd_holders < 0); 647 WARN_ON_ONCE(--whole->bd_holders < 0); 648 if (!bdev->bd_holders) { 649 mutex_lock(&bdev->bd_holder_lock); 650 bdev->bd_holder = NULL; 651 bdev->bd_holder_ops = NULL; 652 mutex_unlock(&bdev->bd_holder_lock); 653 if (bdev_test_flag(bdev, BD_WRITE_HOLDER)) 654 unblock = true; 655 } 656 if (!whole->bd_holders) 657 whole->bd_holder = NULL; 658 mutex_unlock(&bdev_lock); 659 660 /* 661 * If this was the last claim, remove holder link and unblock evpoll if 662 * it was a write holder. 663 */ 664 if (unblock) { 665 disk_unblock_events(bdev->bd_disk); 666 bdev_clear_flag(bdev, BD_WRITE_HOLDER); 667 } 668 } 669 670 static void blkdev_flush_mapping(struct block_device *bdev) 671 { 672 WARN_ON_ONCE(bdev->bd_holders); 673 sync_blockdev(bdev); 674 kill_bdev(bdev); 675 bdev_write_inode(bdev); 676 } 677 678 static void blkdev_put_whole(struct block_device *bdev) 679 { 680 if (atomic_dec_and_test(&bdev->bd_openers)) 681 blkdev_flush_mapping(bdev); 682 if (bdev->bd_disk->fops->release) 683 bdev->bd_disk->fops->release(bdev->bd_disk); 684 } 685 686 static int blkdev_get_whole(struct block_device *bdev, blk_mode_t mode) 687 { 688 struct gendisk *disk = bdev->bd_disk; 689 int ret; 690 691 if (disk->fops->open) { 692 ret = disk->fops->open(disk, mode); 693 if (ret) { 694 /* avoid ghost partitions on a removed medium */ 695 if (ret == -ENOMEDIUM && 696 test_bit(GD_NEED_PART_SCAN, &disk->state)) 697 bdev_disk_changed(disk, true); 698 return ret; 699 } 700 } 701 702 if (!atomic_read(&bdev->bd_openers)) 703 set_init_blocksize(bdev); 704 atomic_inc(&bdev->bd_openers); 705 if (test_bit(GD_NEED_PART_SCAN, &disk->state)) { 706 /* 707 * Only return scanning errors if we are called from contexts 708 * that explicitly want them, e.g. the BLKRRPART ioctl. 709 */ 710 ret = bdev_disk_changed(disk, false); 711 if (ret && (mode & BLK_OPEN_STRICT_SCAN)) { 712 blkdev_put_whole(bdev); 713 return ret; 714 } 715 } 716 return 0; 717 } 718 719 static int blkdev_get_part(struct block_device *part, blk_mode_t mode) 720 { 721 struct gendisk *disk = part->bd_disk; 722 int ret; 723 724 ret = blkdev_get_whole(bdev_whole(part), mode); 725 if (ret) 726 return ret; 727 728 ret = -ENXIO; 729 if (!bdev_nr_sectors(part)) 730 goto out_blkdev_put; 731 732 if (!atomic_read(&part->bd_openers)) { 733 disk->open_partitions++; 734 set_init_blocksize(part); 735 } 736 atomic_inc(&part->bd_openers); 737 return 0; 738 739 out_blkdev_put: 740 blkdev_put_whole(bdev_whole(part)); 741 return ret; 742 } 743 744 int bdev_permission(dev_t dev, blk_mode_t mode, void *holder) 745 { 746 int ret; 747 748 ret = devcgroup_check_permission(DEVCG_DEV_BLOCK, 749 MAJOR(dev), MINOR(dev), 750 ((mode & BLK_OPEN_READ) ? DEVCG_ACC_READ : 0) | 751 ((mode & BLK_OPEN_WRITE) ? DEVCG_ACC_WRITE : 0)); 752 if (ret) 753 return ret; 754 755 /* Blocking writes requires exclusive opener */ 756 if (mode & BLK_OPEN_RESTRICT_WRITES && !holder) 757 return -EINVAL; 758 759 /* 760 * We're using error pointers to indicate to ->release() when we 761 * failed to open that block device. Also this doesn't make sense. 762 */ 763 if (WARN_ON_ONCE(IS_ERR(holder))) 764 return -EINVAL; 765 766 return 0; 767 } 768 769 static void blkdev_put_part(struct block_device *part) 770 { 771 struct block_device *whole = bdev_whole(part); 772 773 if (atomic_dec_and_test(&part->bd_openers)) { 774 blkdev_flush_mapping(part); 775 whole->bd_disk->open_partitions--; 776 } 777 blkdev_put_whole(whole); 778 } 779 780 struct block_device *blkdev_get_no_open(dev_t dev) 781 { 782 struct block_device *bdev; 783 struct inode *inode; 784 785 inode = ilookup(blockdev_superblock, dev); 786 if (!inode && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) { 787 blk_request_module(dev); 788 inode = ilookup(blockdev_superblock, dev); 789 if (inode) 790 pr_warn_ratelimited( 791 "block device autoloading is deprecated and will be removed.\n"); 792 } 793 if (!inode) 794 return NULL; 795 796 /* switch from the inode reference to a device mode one: */ 797 bdev = &BDEV_I(inode)->bdev; 798 if (!kobject_get_unless_zero(&bdev->bd_device.kobj)) 799 bdev = NULL; 800 iput(inode); 801 return bdev; 802 } 803 804 void blkdev_put_no_open(struct block_device *bdev) 805 { 806 put_device(&bdev->bd_device); 807 } 808 809 static bool bdev_writes_blocked(struct block_device *bdev) 810 { 811 return bdev->bd_writers < 0; 812 } 813 814 static void bdev_block_writes(struct block_device *bdev) 815 { 816 bdev->bd_writers--; 817 } 818 819 static void bdev_unblock_writes(struct block_device *bdev) 820 { 821 bdev->bd_writers++; 822 } 823 824 static bool bdev_may_open(struct block_device *bdev, blk_mode_t mode) 825 { 826 if (bdev_allow_write_mounted) 827 return true; 828 /* Writes blocked? */ 829 if (mode & BLK_OPEN_WRITE && bdev_writes_blocked(bdev)) 830 return false; 831 if (mode & BLK_OPEN_RESTRICT_WRITES && bdev->bd_writers > 0) 832 return false; 833 return true; 834 } 835 836 static void bdev_claim_write_access(struct block_device *bdev, blk_mode_t mode) 837 { 838 if (bdev_allow_write_mounted) 839 return; 840 841 /* Claim exclusive or shared write access. */ 842 if (mode & BLK_OPEN_RESTRICT_WRITES) 843 bdev_block_writes(bdev); 844 else if (mode & BLK_OPEN_WRITE) 845 bdev->bd_writers++; 846 } 847 848 static inline bool bdev_unclaimed(const struct file *bdev_file) 849 { 850 return bdev_file->private_data == BDEV_I(bdev_file->f_mapping->host); 851 } 852 853 static void bdev_yield_write_access(struct file *bdev_file) 854 { 855 struct block_device *bdev; 856 857 if (bdev_allow_write_mounted) 858 return; 859 860 if (bdev_unclaimed(bdev_file)) 861 return; 862 863 bdev = file_bdev(bdev_file); 864 865 if (bdev_file->f_mode & FMODE_WRITE_RESTRICTED) 866 bdev_unblock_writes(bdev); 867 else if (bdev_file->f_mode & FMODE_WRITE) 868 bdev->bd_writers--; 869 } 870 871 /** 872 * bdev_open - open a block device 873 * @bdev: block device to open 874 * @mode: open mode (BLK_OPEN_*) 875 * @holder: exclusive holder identifier 876 * @hops: holder operations 877 * @bdev_file: file for the block device 878 * 879 * Open the block device. If @holder is not %NULL, the block device is opened 880 * with exclusive access. Exclusive opens may nest for the same @holder. 881 * 882 * CONTEXT: 883 * Might sleep. 884 * 885 * RETURNS: 886 * zero on success, -errno on failure. 887 */ 888 int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder, 889 const struct blk_holder_ops *hops, struct file *bdev_file) 890 { 891 bool unblock_events = true; 892 struct gendisk *disk = bdev->bd_disk; 893 int ret; 894 895 if (holder) { 896 mode |= BLK_OPEN_EXCL; 897 ret = bd_prepare_to_claim(bdev, holder, hops); 898 if (ret) 899 return ret; 900 } else { 901 if (WARN_ON_ONCE(mode & BLK_OPEN_EXCL)) 902 return -EIO; 903 } 904 905 disk_block_events(disk); 906 907 mutex_lock(&disk->open_mutex); 908 ret = -ENXIO; 909 if (!disk_live(disk)) 910 goto abort_claiming; 911 if (!try_module_get(disk->fops->owner)) 912 goto abort_claiming; 913 ret = -EBUSY; 914 if (!bdev_may_open(bdev, mode)) 915 goto put_module; 916 if (bdev_is_partition(bdev)) 917 ret = blkdev_get_part(bdev, mode); 918 else 919 ret = blkdev_get_whole(bdev, mode); 920 if (ret) 921 goto put_module; 922 bdev_claim_write_access(bdev, mode); 923 if (holder) { 924 bd_finish_claiming(bdev, holder, hops); 925 926 /* 927 * Block event polling for write claims if requested. Any write 928 * holder makes the write_holder state stick until all are 929 * released. This is good enough and tracking individual 930 * writeable reference is too fragile given the way @mode is 931 * used in blkdev_get/put(). 932 */ 933 if ((mode & BLK_OPEN_WRITE) && 934 !bdev_test_flag(bdev, BD_WRITE_HOLDER) && 935 (disk->event_flags & DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE)) { 936 bdev_set_flag(bdev, BD_WRITE_HOLDER); 937 unblock_events = false; 938 } 939 } 940 mutex_unlock(&disk->open_mutex); 941 942 if (unblock_events) 943 disk_unblock_events(disk); 944 945 bdev_file->f_flags |= O_LARGEFILE; 946 bdev_file->f_mode |= FMODE_CAN_ODIRECT; 947 if (bdev_nowait(bdev)) 948 bdev_file->f_mode |= FMODE_NOWAIT; 949 if (mode & BLK_OPEN_RESTRICT_WRITES) 950 bdev_file->f_mode |= FMODE_WRITE_RESTRICTED; 951 bdev_file->f_mapping = bdev->bd_mapping; 952 bdev_file->f_wb_err = filemap_sample_wb_err(bdev_file->f_mapping); 953 bdev_file->private_data = holder; 954 955 return 0; 956 put_module: 957 module_put(disk->fops->owner); 958 abort_claiming: 959 if (holder) 960 bd_abort_claiming(bdev, holder); 961 mutex_unlock(&disk->open_mutex); 962 disk_unblock_events(disk); 963 return ret; 964 } 965 966 /* 967 * If BLK_OPEN_WRITE_IOCTL is set then this is a historical quirk 968 * associated with the floppy driver where it has allowed ioctls if the 969 * file was opened for writing, but does not allow reads or writes. 970 * Make sure that this quirk is reflected in @f_flags. 971 * 972 * It can also happen if a block device is opened as O_RDWR | O_WRONLY. 973 */ 974 static unsigned blk_to_file_flags(blk_mode_t mode) 975 { 976 unsigned int flags = 0; 977 978 if ((mode & (BLK_OPEN_READ | BLK_OPEN_WRITE)) == 979 (BLK_OPEN_READ | BLK_OPEN_WRITE)) 980 flags |= O_RDWR; 981 else if (mode & BLK_OPEN_WRITE_IOCTL) 982 flags |= O_RDWR | O_WRONLY; 983 else if (mode & BLK_OPEN_WRITE) 984 flags |= O_WRONLY; 985 else if (mode & BLK_OPEN_READ) 986 flags |= O_RDONLY; /* homeopathic, because O_RDONLY is 0 */ 987 else 988 WARN_ON_ONCE(true); 989 990 if (mode & BLK_OPEN_NDELAY) 991 flags |= O_NDELAY; 992 993 return flags; 994 } 995 996 struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder, 997 const struct blk_holder_ops *hops) 998 { 999 struct file *bdev_file; 1000 struct block_device *bdev; 1001 unsigned int flags; 1002 int ret; 1003 1004 ret = bdev_permission(dev, mode, holder); 1005 if (ret) 1006 return ERR_PTR(ret); 1007 1008 bdev = blkdev_get_no_open(dev); 1009 if (!bdev) 1010 return ERR_PTR(-ENXIO); 1011 1012 flags = blk_to_file_flags(mode); 1013 bdev_file = alloc_file_pseudo_noaccount(BD_INODE(bdev), 1014 blockdev_mnt, "", flags | O_LARGEFILE, &def_blk_fops); 1015 if (IS_ERR(bdev_file)) { 1016 blkdev_put_no_open(bdev); 1017 return bdev_file; 1018 } 1019 ihold(BD_INODE(bdev)); 1020 1021 ret = bdev_open(bdev, mode, holder, hops, bdev_file); 1022 if (ret) { 1023 /* We failed to open the block device. Let ->release() know. */ 1024 bdev_file->private_data = ERR_PTR(ret); 1025 fput(bdev_file); 1026 return ERR_PTR(ret); 1027 } 1028 return bdev_file; 1029 } 1030 EXPORT_SYMBOL(bdev_file_open_by_dev); 1031 1032 struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode, 1033 void *holder, 1034 const struct blk_holder_ops *hops) 1035 { 1036 struct file *file; 1037 dev_t dev; 1038 int error; 1039 1040 error = lookup_bdev(path, &dev); 1041 if (error) 1042 return ERR_PTR(error); 1043 1044 file = bdev_file_open_by_dev(dev, mode, holder, hops); 1045 if (!IS_ERR(file) && (mode & BLK_OPEN_WRITE)) { 1046 if (bdev_read_only(file_bdev(file))) { 1047 fput(file); 1048 file = ERR_PTR(-EACCES); 1049 } 1050 } 1051 1052 return file; 1053 } 1054 EXPORT_SYMBOL(bdev_file_open_by_path); 1055 1056 static inline void bd_yield_claim(struct file *bdev_file) 1057 { 1058 struct block_device *bdev = file_bdev(bdev_file); 1059 void *holder = bdev_file->private_data; 1060 1061 lockdep_assert_held(&bdev->bd_disk->open_mutex); 1062 1063 if (WARN_ON_ONCE(IS_ERR_OR_NULL(holder))) 1064 return; 1065 1066 if (!bdev_unclaimed(bdev_file)) 1067 bd_end_claim(bdev, holder); 1068 } 1069 1070 void bdev_release(struct file *bdev_file) 1071 { 1072 struct block_device *bdev = file_bdev(bdev_file); 1073 void *holder = bdev_file->private_data; 1074 struct gendisk *disk = bdev->bd_disk; 1075 1076 /* We failed to open that block device. */ 1077 if (IS_ERR(holder)) 1078 goto put_no_open; 1079 1080 /* 1081 * Sync early if it looks like we're the last one. If someone else 1082 * opens the block device between now and the decrement of bd_openers 1083 * then we did a sync that we didn't need to, but that's not the end 1084 * of the world and we want to avoid long (could be several minute) 1085 * syncs while holding the mutex. 1086 */ 1087 if (atomic_read(&bdev->bd_openers) == 1) 1088 sync_blockdev(bdev); 1089 1090 mutex_lock(&disk->open_mutex); 1091 bdev_yield_write_access(bdev_file); 1092 1093 if (holder) 1094 bd_yield_claim(bdev_file); 1095 1096 /* 1097 * Trigger event checking and tell drivers to flush MEDIA_CHANGE 1098 * event. This is to ensure detection of media removal commanded 1099 * from userland - e.g. eject(1). 1100 */ 1101 disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE); 1102 1103 if (bdev_is_partition(bdev)) 1104 blkdev_put_part(bdev); 1105 else 1106 blkdev_put_whole(bdev); 1107 mutex_unlock(&disk->open_mutex); 1108 1109 module_put(disk->fops->owner); 1110 put_no_open: 1111 blkdev_put_no_open(bdev); 1112 } 1113 1114 /** 1115 * bdev_fput - yield claim to the block device and put the file 1116 * @bdev_file: open block device 1117 * 1118 * Yield claim on the block device and put the file. Ensure that the 1119 * block device can be reclaimed before the file is closed which is a 1120 * deferred operation. 1121 */ 1122 void bdev_fput(struct file *bdev_file) 1123 { 1124 if (WARN_ON_ONCE(bdev_file->f_op != &def_blk_fops)) 1125 return; 1126 1127 if (bdev_file->private_data) { 1128 struct block_device *bdev = file_bdev(bdev_file); 1129 struct gendisk *disk = bdev->bd_disk; 1130 1131 mutex_lock(&disk->open_mutex); 1132 bdev_yield_write_access(bdev_file); 1133 bd_yield_claim(bdev_file); 1134 /* 1135 * Tell release we already gave up our hold on the 1136 * device and if write restrictions are available that 1137 * we already gave up write access to the device. 1138 */ 1139 bdev_file->private_data = BDEV_I(bdev_file->f_mapping->host); 1140 mutex_unlock(&disk->open_mutex); 1141 } 1142 1143 fput(bdev_file); 1144 } 1145 EXPORT_SYMBOL(bdev_fput); 1146 1147 /** 1148 * lookup_bdev() - Look up a struct block_device by name. 1149 * @pathname: Name of the block device in the filesystem. 1150 * @dev: Pointer to the block device's dev_t, if found. 1151 * 1152 * Lookup the block device's dev_t at @pathname in the current 1153 * namespace if possible and return it in @dev. 1154 * 1155 * Context: May sleep. 1156 * Return: 0 if succeeded, negative errno otherwise. 1157 */ 1158 int lookup_bdev(const char *pathname, dev_t *dev) 1159 { 1160 struct inode *inode; 1161 struct path path; 1162 int error; 1163 1164 if (!pathname || !*pathname) 1165 return -EINVAL; 1166 1167 error = kern_path(pathname, LOOKUP_FOLLOW, &path); 1168 if (error) 1169 return error; 1170 1171 inode = d_backing_inode(path.dentry); 1172 error = -ENOTBLK; 1173 if (!S_ISBLK(inode->i_mode)) 1174 goto out_path_put; 1175 error = -EACCES; 1176 if (!may_open_dev(&path)) 1177 goto out_path_put; 1178 1179 *dev = inode->i_rdev; 1180 error = 0; 1181 out_path_put: 1182 path_put(&path); 1183 return error; 1184 } 1185 EXPORT_SYMBOL(lookup_bdev); 1186 1187 /** 1188 * bdev_mark_dead - mark a block device as dead 1189 * @bdev: block device to operate on 1190 * @surprise: indicate a surprise removal 1191 * 1192 * Tell the file system that this devices or media is dead. If @surprise is set 1193 * to %true the device or media is already gone, if not we are preparing for an 1194 * orderly removal. 1195 * 1196 * This calls into the file system, which then typicall syncs out all dirty data 1197 * and writes back inodes and then invalidates any cached data in the inodes on 1198 * the file system. In addition we also invalidate the block device mapping. 1199 */ 1200 void bdev_mark_dead(struct block_device *bdev, bool surprise) 1201 { 1202 mutex_lock(&bdev->bd_holder_lock); 1203 if (bdev->bd_holder_ops && bdev->bd_holder_ops->mark_dead) 1204 bdev->bd_holder_ops->mark_dead(bdev, surprise); 1205 else { 1206 mutex_unlock(&bdev->bd_holder_lock); 1207 sync_blockdev(bdev); 1208 } 1209 1210 invalidate_bdev(bdev); 1211 } 1212 /* 1213 * New drivers should not use this directly. There are some drivers however 1214 * that needs this for historical reasons. For example, the DASD driver has 1215 * historically had a shutdown to offline mode that doesn't actually remove the 1216 * gendisk that otherwise looks a lot like a safe device removal. 1217 */ 1218 EXPORT_SYMBOL_GPL(bdev_mark_dead); 1219 1220 void sync_bdevs(bool wait) 1221 { 1222 struct inode *inode, *old_inode = NULL; 1223 1224 spin_lock(&blockdev_superblock->s_inode_list_lock); 1225 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) { 1226 struct address_space *mapping = inode->i_mapping; 1227 struct block_device *bdev; 1228 1229 spin_lock(&inode->i_lock); 1230 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) || 1231 mapping->nrpages == 0) { 1232 spin_unlock(&inode->i_lock); 1233 continue; 1234 } 1235 __iget(inode); 1236 spin_unlock(&inode->i_lock); 1237 spin_unlock(&blockdev_superblock->s_inode_list_lock); 1238 /* 1239 * We hold a reference to 'inode' so it couldn't have been 1240 * removed from s_inodes list while we dropped the 1241 * s_inode_list_lock We cannot iput the inode now as we can 1242 * be holding the last reference and we cannot iput it under 1243 * s_inode_list_lock. So we keep the reference and iput it 1244 * later. 1245 */ 1246 iput(old_inode); 1247 old_inode = inode; 1248 bdev = I_BDEV(inode); 1249 1250 mutex_lock(&bdev->bd_disk->open_mutex); 1251 if (!atomic_read(&bdev->bd_openers)) { 1252 ; /* skip */ 1253 } else if (wait) { 1254 /* 1255 * We keep the error status of individual mapping so 1256 * that applications can catch the writeback error using 1257 * fsync(2). See filemap_fdatawait_keep_errors() for 1258 * details. 1259 */ 1260 filemap_fdatawait_keep_errors(inode->i_mapping); 1261 } else { 1262 filemap_fdatawrite(inode->i_mapping); 1263 } 1264 mutex_unlock(&bdev->bd_disk->open_mutex); 1265 1266 spin_lock(&blockdev_superblock->s_inode_list_lock); 1267 } 1268 spin_unlock(&blockdev_superblock->s_inode_list_lock); 1269 iput(old_inode); 1270 } 1271 1272 /* 1273 * Handle STATX_{DIOALIGN, WRITE_ATOMIC} for block devices. 1274 */ 1275 void bdev_statx(struct path *path, struct kstat *stat, 1276 u32 request_mask) 1277 { 1278 struct inode *backing_inode; 1279 struct block_device *bdev; 1280 1281 backing_inode = d_backing_inode(path->dentry); 1282 1283 /* 1284 * Note that backing_inode is the inode of a block device node file, 1285 * not the block device's internal inode. Therefore it is *not* valid 1286 * to use I_BDEV() here; the block device has to be looked up by i_rdev 1287 * instead. 1288 */ 1289 bdev = blkdev_get_no_open(backing_inode->i_rdev); 1290 if (!bdev) 1291 return; 1292 1293 if (request_mask & STATX_DIOALIGN) { 1294 stat->dio_mem_align = bdev_dma_alignment(bdev) + 1; 1295 stat->dio_offset_align = bdev_logical_block_size(bdev); 1296 stat->result_mask |= STATX_DIOALIGN; 1297 } 1298 1299 if (request_mask & STATX_WRITE_ATOMIC && bdev_can_atomic_write(bdev)) { 1300 struct request_queue *bd_queue = bdev->bd_queue; 1301 1302 generic_fill_statx_atomic_writes(stat, 1303 queue_atomic_write_unit_min_bytes(bd_queue), 1304 queue_atomic_write_unit_max_bytes(bd_queue)); 1305 } 1306 1307 stat->blksize = bdev_io_min(bdev); 1308 1309 blkdev_put_no_open(bdev); 1310 } 1311 1312 bool disk_live(struct gendisk *disk) 1313 { 1314 return !inode_unhashed(BD_INODE(disk->part0)); 1315 } 1316 EXPORT_SYMBOL_GPL(disk_live); 1317 1318 unsigned int block_size(struct block_device *bdev) 1319 { 1320 return 1 << BD_INODE(bdev)->i_blkbits; 1321 } 1322 EXPORT_SYMBOL_GPL(block_size); 1323 1324 static int __init setup_bdev_allow_write_mounted(char *str) 1325 { 1326 if (kstrtobool(str, &bdev_allow_write_mounted)) 1327 pr_warn("Invalid option string for bdev_allow_write_mounted:" 1328 " '%s'\n", str); 1329 return 1; 1330 } 1331 __setup("bdev_allow_write_mounted=", setup_bdev_allow_write_mounted); 1332