Lines Matching +full:disk +full:- +full:activity
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright (C) 1994, Karl Keyte: Added support for disk statistics
7 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
8 * - July2000
9 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
19 #include <linux/blk-pm.h>
20 #include <linux/blk-integrity.h>
32 #include <linux/fault-inject.h>
37 #include <linux/t10-pi.h>
42 #include <linux/blk-crypto.h>
48 #include "blk-mq-sched.h"
49 #include "blk-pm.h"
50 #include "blk-cgroup.h"
51 #include "blk-throttle.h"
52 #include "blk-ioprio.h"
76 * blk_queue_flag_set - atomically set a queue flag
82 set_bit(flag, &q->queue_flags); in blk_queue_flag_set()
87 * blk_queue_flag_clear - atomically clear a queue flag
93 clear_bit(flag, &q->queue_flags); in blk_queue_flag_clear()
98 * blk_queue_flag_test_and_set - atomically test and set a queue flag
102 * Returns the previous value of @flag - 0 if the flag was not set and 1 if
107 return test_and_set_bit(flag, &q->queue_flags); in blk_queue_flag_test_and_set()
131 * blk_op_str - Return string XXX in the REQ_OP_XXX.
154 [BLK_STS_NOTSUPP] = { -EOPNOTSUPP, "operation not supported" },
155 [BLK_STS_TIMEOUT] = { -ETIMEDOUT, "timeout" },
156 [BLK_STS_NOSPC] = { -ENOSPC, "critical space allocation" },
157 [BLK_STS_TRANSPORT] = { -ENOLINK, "recoverable transport" },
158 [BLK_STS_TARGET] = { -EREMOTEIO, "critical target" },
159 [BLK_STS_RESV_CONFLICT] = { -EBADE, "reservation conflict" },
160 [BLK_STS_MEDIUM] = { -ENODATA, "critical medium" },
161 [BLK_STS_PROTECTION] = { -EILSEQ, "protection" },
162 [BLK_STS_RESOURCE] = { -ENOMEM, "kernel resource" },
163 [BLK_STS_DEV_RESOURCE] = { -EBUSY, "device resource" },
164 [BLK_STS_AGAIN] = { -EAGAIN, "nonblocking retry" },
165 [BLK_STS_OFFLINE] = { -ENODEV, "device offline" },
168 [BLK_STS_DM_REQUEUE] = { -EREMCHG, "dm internal retry" },
171 [BLK_STS_ZONE_OPEN_RESOURCE] = { -ETOOMANYREFS, "open zones exceeded" },
172 [BLK_STS_ZONE_ACTIVE_RESOURCE] = { -EOVERFLOW, "active zones exceeded" },
174 /* Command duration limit device-side timeout */
175 [BLK_STS_DURATION_LIMIT] = { -ETIME, "duration limit exceeded" },
178 [BLK_STS_IOERR] = { -EIO, "I/O" },
199 return -EIO; in blk_status_to_errno()
215 * blk_sync_queue - cancel any pending callbacks on a queue
219 * The block layer may perform asynchronous callback activity
222 * such activity is cancelled, thus allowing it to release resources
224 * that its ->submit_bio will not re-add plugging prior to calling
227 * This function does not cancel any asynchronous activity arising
234 del_timer_sync(&q->timeout); in blk_sync_queue()
235 cancel_work_sync(&q->timeout_work); in blk_sync_queue()
240 * blk_set_pm_only - increment pm_only counter
245 atomic_inc(&q->pm_only); in blk_set_pm_only()
253 pm_only = atomic_dec_return(&q->pm_only); in blk_clear_pm_only()
256 wake_up_all(&q->mq_freeze_wq); in blk_clear_pm_only()
265 percpu_ref_exit(&q->q_usage_counter); in blk_free_queue_rcu()
271 blk_free_queue_stats(q->stats); in blk_free_queue()
275 ida_free(&blk_queue_ida, q->id); in blk_free_queue()
276 call_rcu(&q->rcu_head, blk_free_queue_rcu); in blk_free_queue()
280 * blk_put_queue - decrement the request_queue refcount
288 if (refcount_dec_and_test(&q->refs)) in blk_put_queue()
304 wake_up_all(&q->mq_freeze_wq); in blk_queue_start_drain()
308 * blk_queue_enter() - try to increase q->q_usage_counter
318 return -EAGAIN; in blk_queue_enter()
328 wait_event(q->mq_freeze_wq, in blk_queue_enter()
329 (!q->mq_freeze_depth && in blk_queue_enter()
333 return -ENODEV; in blk_queue_enter()
342 struct gendisk *disk = bio->bi_bdev->bd_disk; in __bio_queue_enter() local
344 if (bio->bi_opf & REQ_NOWAIT) { in __bio_queue_enter()
345 if (test_bit(GD_DEAD, &disk->state)) in __bio_queue_enter()
348 return -EAGAIN; in __bio_queue_enter()
359 wait_event(q->mq_freeze_wq, in __bio_queue_enter()
360 (!q->mq_freeze_depth && in __bio_queue_enter()
362 test_bit(GD_DEAD, &disk->state)); in __bio_queue_enter()
363 if (test_bit(GD_DEAD, &disk->state)) in __bio_queue_enter()
370 return -ENODEV; in __bio_queue_enter()
375 percpu_ref_put(&q->q_usage_counter); in blk_queue_exit()
383 wake_up_all(&q->mq_freeze_wq); in blk_queue_usage_counter_release()
390 kblockd_schedule_work(&q->timeout_work); in blk_rq_timed_out_timer()
406 q->last_merge = NULL; in blk_alloc_queue()
408 q->id = ida_alloc(&blk_queue_ida, GFP_KERNEL); in blk_alloc_queue()
409 if (q->id < 0) in blk_alloc_queue()
412 q->stats = blk_alloc_queue_stats(); in blk_alloc_queue()
413 if (!q->stats) in blk_alloc_queue()
416 q->node = node_id; in blk_alloc_queue()
418 atomic_set(&q->nr_active_requests_shared_tags, 0); in blk_alloc_queue()
420 timer_setup(&q->timeout, blk_rq_timed_out_timer, 0); in blk_alloc_queue()
421 INIT_WORK(&q->timeout_work, blk_timeout_work); in blk_alloc_queue()
422 INIT_LIST_HEAD(&q->icq_list); in blk_alloc_queue()
424 refcount_set(&q->refs, 1); in blk_alloc_queue()
425 mutex_init(&q->debugfs_mutex); in blk_alloc_queue()
426 mutex_init(&q->sysfs_lock); in blk_alloc_queue()
427 mutex_init(&q->sysfs_dir_lock); in blk_alloc_queue()
428 mutex_init(&q->rq_qos_mutex); in blk_alloc_queue()
429 spin_lock_init(&q->queue_lock); in blk_alloc_queue()
431 init_waitqueue_head(&q->mq_freeze_wq); in blk_alloc_queue()
432 mutex_init(&q->mq_freeze_lock); in blk_alloc_queue()
438 if (percpu_ref_init(&q->q_usage_counter, in blk_alloc_queue()
443 blk_set_default_limits(&q->limits); in blk_alloc_queue()
444 q->nr_requests = BLKDEV_DEFAULT_RQ; in blk_alloc_queue()
449 blk_free_queue_stats(q->stats); in blk_alloc_queue()
451 ida_free(&blk_queue_ida, q->id); in blk_alloc_queue()
458 * blk_get_queue - increment the request_queue refcount
469 refcount_inc(&q->refs); in blk_get_queue()
486 return part->bd_make_it_fail && should_fail(&fail_make_request, bytes); in should_fail_request()
502 if (op_is_write(bio_op(bio)) && bdev_read_only(bio->bi_bdev)) { in bio_check_ro()
503 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) in bio_check_ro()
506 if (bio->bi_bdev->bd_ro_warned) in bio_check_ro()
509 bio->bi_bdev->bd_ro_warned = true; in bio_check_ro()
511 * Use ioctl to set underlying disk of raid/dm to read-only in bio_check_ro()
514 pr_warn("Trying to write to read-only block-device %pg\n", in bio_check_ro()
515 bio->bi_bdev); in bio_check_ro()
521 if (should_fail_request(bdev_whole(bio->bi_bdev), bio->bi_iter.bi_size)) in should_fail_bio()
522 return -EIO; in should_fail_bio()
529 * This may well happen - the kernel calls bread() without checking the size of
534 sector_t maxsector = bdev_nr_sectors(bio->bi_bdev); in bio_check_eod()
539 bio->bi_iter.bi_sector > maxsector - nr_sectors)) { in bio_check_eod()
542 current->comm, bio->bi_bdev, bio->bi_opf, in bio_check_eod()
543 bio->bi_iter.bi_sector, nr_sectors, maxsector); in bio_check_eod()
544 return -EIO; in bio_check_eod()
550 * Remap block n of partition p to block n+start(p) of the disk.
554 struct block_device *p = bio->bi_bdev; in blk_partition_remap()
556 if (unlikely(should_fail_request(p, bio->bi_iter.bi_size))) in blk_partition_remap()
557 return -EIO; in blk_partition_remap()
559 bio->bi_iter.bi_sector += p->bd_start_sect; in blk_partition_remap()
560 trace_block_bio_remap(bio, p->bd_dev, in blk_partition_remap()
561 bio->bi_iter.bi_sector - in blk_partition_remap()
562 p->bd_start_sect); in blk_partition_remap()
577 if (!bdev_is_zoned(bio->bi_bdev)) in blk_check_zone_append()
581 if (!bdev_is_zone_start(bio->bi_bdev, bio->bi_iter.bi_sector) || in blk_check_zone_append()
587 * split and could result in non-contiguous sectors being written in in blk_check_zone_append()
590 if (nr_sectors > q->limits.chunk_sectors) in blk_check_zone_append()
594 if (nr_sectors > q->limits.max_zone_append_sectors) in blk_check_zone_append()
597 bio->bi_opf |= REQ_NOMERGE; in blk_check_zone_append()
607 if (!bio->bi_bdev->bd_has_submit_bio) { in __submit_bio()
610 struct gendisk *disk = bio->bi_bdev->bd_disk; in __submit_bio() local
612 disk->fops->submit_bio(bio); in __submit_bio()
613 blk_queue_exit(disk->queue); in __submit_bio()
618 * The loop in this function may be a bit non-obvious, and so deserves some
621 * - Before entering the loop, bio->bi_next is NULL (as all callers ensure
623 * - We pretend that we have just taken it off a longer list, so we assign
625 * bio_list of new bios to be added. ->submit_bio() may indeed add some more
627 * non-NULL value in bio_list and re-enter the loop from the top.
628 * - In this case we really did just take the bio of the top of the list (no
629 * pretending) and so remove it from bio_list, and call into ->submit_bio()
632 * bio_list_on_stack[0] contains bios submitted by the current ->submit_bio.
634 * ->submit_bio, but that haven't been processed yet.
640 BUG_ON(bio->bi_next); in __submit_bio_noacct()
643 current->bio_list = bio_list_on_stack; in __submit_bio_noacct()
646 struct request_queue *q = bdev_get_queue(bio->bi_bdev); in __submit_bio_noacct()
664 if (q == bdev_get_queue(bio->bi_bdev)) in __submit_bio_noacct()
677 current->bio_list = NULL; in __submit_bio_noacct()
684 current->bio_list = bio_list; in __submit_bio_noacct_mq()
690 current->bio_list = NULL; in __submit_bio_noacct_mq()
708 * We only want one ->submit_bio to be active at a time, else stack in submit_bio_noacct_nocheck()
709 * usage with stacked devices could be a problem. Use current->bio_list in submit_bio_noacct_nocheck()
710 * to collect a list of requests submited by a ->submit_bio method while in submit_bio_noacct_nocheck()
713 if (current->bio_list) in submit_bio_noacct_nocheck()
714 bio_list_add(¤t->bio_list[0], bio); in submit_bio_noacct_nocheck()
715 else if (!bio->bi_bdev->bd_has_submit_bio) in submit_bio_noacct_nocheck()
722 * submit_bio_noacct - re-submit a bio to the block device layer for I/O
732 struct block_device *bdev = bio->bi_bdev; in submit_bio_noacct()
739 * For a REQ_NOWAIT based request, return -EOPNOTSUPP in submit_bio_noacct()
742 if ((bio->bi_opf & REQ_NOWAIT) && !bdev_nowait(bdev)) in submit_bio_noacct()
751 if (bdev->bd_partno && unlikely(blk_partition_remap(bio))) in submit_bio_noacct()
759 if (op_is_flush(bio->bi_opf)) { in submit_bio_noacct()
763 if (!test_bit(QUEUE_FLAG_WC, &q->queue_flags)) { in submit_bio_noacct()
764 bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA); in submit_bio_noacct()
772 if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) in submit_bio_noacct()
799 if (!q->limits.max_write_zeroes_sectors) in submit_bio_noacct()
806 if (!bdev_is_zoned(bio->bi_bdev)) in submit_bio_noacct()
810 if (!bdev_is_zoned(bio->bi_bdev) || !blk_queue_zone_resetall(q)) in submit_bio_noacct()
832 bio->bi_status = status; in submit_bio_noacct()
840 if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) == IOPRIO_CLASS_NONE) in bio_set_ioprio()
841 bio->bi_ioprio = get_current_ioprio(); in bio_set_ioprio()
846 * submit_bio - submit a bio to the block device layer for I/O
854 * completion, is delivered asynchronously through the ->bi_end_io() callback
855 * in @bio. The bio must NOT be touched by the caller until ->bi_end_io() has
861 task_io_account_read(bio->bi_iter.bi_size); in submit_bio()
873 * bio_poll - poll for BIO completions
886 blk_qc_t cookie = READ_ONCE(bio->bi_cookie); in bio_poll()
891 bdev = READ_ONCE(bio->bi_bdev); in bio_poll()
897 !test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) in bio_poll()
906 blk_flush_plug(current->plug, false); in bio_poll()
917 if (!percpu_ref_tryget(&q->q_usage_counter)) in bio_poll()
922 struct gendisk *disk = q->disk; in bio_poll() local
924 if (disk && disk->fops->poll_bio) in bio_poll()
925 ret = disk->fops->poll_bio(bio, iob, flags); in bio_poll()
934 * in iocb->private, and cleared before freeing the bio.
958 * device bdev and thus disk, the disk is also still valid. Grabbing in iocb_bio_iopoll()
963 bio = READ_ONCE(kiocb->private); in iocb_bio_iopoll()
976 stamp = READ_ONCE(part->bd_stamp); in update_io_ticks()
978 if (likely(try_cmpxchg(&part->bd_stamp, &stamp, now))) in update_io_ticks()
979 __part_stat_add(part, io_ticks, end ? now - stamp : 1); in update_io_ticks()
981 if (part->bd_partno) { in update_io_ticks()
1000 * bio_start_io_acct - start I/O accounting for bio based drivers
1007 return bdev_start_io_acct(bio->bi_bdev, bio_op(bio), jiffies); in bio_start_io_acct()
1016 unsigned long duration = now - start_time; in bdev_end_io_acct()
1036 * blk_lld_busy - Check if underlying low-level drivers of a device are busy
1040 * Check if underlying low-level drivers of a device are busy.
1051 * 0 - Not busy (The request stacking driver should dispatch request)
1052 * 1 - Busy (The request stacking driver should stop dispatching request)
1056 if (queue_is_mq(q) && q->mq_ops->busy) in blk_lld_busy()
1057 return q->mq_ops->busy(q); in blk_lld_busy()
1083 if (tsk->plug) in blk_start_plug_nr_ios()
1086 plug->mq_list = NULL; in blk_start_plug_nr_ios()
1087 plug->cached_rq = NULL; in blk_start_plug_nr_ios()
1088 plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT); in blk_start_plug_nr_ios()
1089 plug->rq_count = 0; in blk_start_plug_nr_ios()
1090 plug->multiple_queues = false; in blk_start_plug_nr_ios()
1091 plug->has_elevator = false; in blk_start_plug_nr_ios()
1092 INIT_LIST_HEAD(&plug->cb_list); in blk_start_plug_nr_ios()
1098 tsk->plug = plug; in blk_start_plug_nr_ios()
1102 * blk_start_plug - initialize blk_plug and track it inside the task_struct
1115 * Tracking blk_plug inside the task_struct will help with auto-flushing the
1134 while (!list_empty(&plug->cb_list)) { in flush_plug_callbacks()
1135 list_splice_init(&plug->cb_list, &callbacks); in flush_plug_callbacks()
1141 list_del(&cb->list); in flush_plug_callbacks()
1142 cb->callback(cb, from_schedule); in flush_plug_callbacks()
1150 struct blk_plug *plug = current->plug; in blk_check_plugged()
1156 list_for_each_entry(cb, &plug->cb_list, list) in blk_check_plugged()
1157 if (cb->callback == unplug && cb->data == data) in blk_check_plugged()
1164 cb->data = data; in blk_check_plugged()
1165 cb->callback = unplug; in blk_check_plugged()
1166 list_add(&cb->list, &plug->cb_list); in blk_check_plugged()
1174 if (!list_empty(&plug->cb_list)) in __blk_flush_plug()
1183 if (unlikely(!rq_list_empty(plug->cached_rq))) in __blk_flush_plug()
1188 * blk_finish_plug - mark the end of a batch of submitted I/O
1199 if (plug == current->plug) { in blk_finish_plug()
1201 current->plug = NULL; in blk_finish_plug()
1226 /* used for unplugging and affects IO latency/throughput - HIGHPRI */ in blk_dev_init()