1 // SPDX-License-Identifier: GPL-2.0
10 #include <linux/backing-dev.h>
15 #include "blk-mq.h"
16 #include "blk-mq-debugfs.h"
17 #include "blk-mq-sched.h"
18 #include "blk-rq-qos.h"
19 #include "blk-wbt.h"
20 #include "blk-cgroup.h"
21 #include "blk-throttle.h"
30 queue_var_show(unsigned long var, char *page)  in queue_var_show()  argument
32 	return sprintf(page, "%lu\n", var);  in queue_var_show()
36 queue_var_store(unsigned long *var, const char *page, size_t count)  in queue_var_store()  argument
41 	err = kstrtoul(page, 10, &v);  in queue_var_store()
43 		return -EINVAL;  in queue_var_store()
50 static ssize_t queue_requests_show(struct request_queue *q, char *page)  in queue_requests_show()  argument
52 	return queue_var_show(q->nr_requests, page);  in queue_requests_show()
56 queue_requests_store(struct request_queue *q, const char *page, size_t count)  in queue_requests_store()  argument
62 		return -EINVAL;  in queue_requests_store()
64 	ret = queue_var_store(&nr, page, count);  in queue_requests_store()
78 static ssize_t queue_ra_show(struct request_queue *q, char *page)  in queue_ra_show()  argument
82 	if (!q->disk)  in queue_ra_show()
83 		return -EINVAL;  in queue_ra_show()
84 	ra_kb = q->disk->bdi->ra_pages << (PAGE_SHIFT - 10);  in queue_ra_show()
85 	return queue_var_show(ra_kb, page);  in queue_ra_show()
89 queue_ra_store(struct request_queue *q, const char *page, size_t count)  in queue_ra_store()  argument
94 	if (!q->disk)  in queue_ra_store()
95 		return -EINVAL;  in queue_ra_store()
96 	ret = queue_var_store(&ra_kb, page, count);  in queue_ra_store()
99 	q->disk->bdi->ra_pages = ra_kb >> (PAGE_SHIFT - 10);  in queue_ra_store()
103 static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)  in queue_max_sectors_show()  argument
107 	return queue_var_show(max_sectors_kb, page);  in queue_max_sectors_show()
110 static ssize_t queue_max_segments_show(struct request_queue *q, char *page)  in queue_max_segments_show()  argument
112 	return queue_var_show(queue_max_segments(q), page);  in queue_max_segments_show()
116 		char *page)  in queue_max_discard_segments_show()  argument
118 	return queue_var_show(queue_max_discard_segments(q), page);  in queue_max_discard_segments_show()
121 static ssize_t queue_max_integrity_segments_show(struct request_queue *q, char *page)  in queue_max_integrity_segments_show()  argument
123 	return queue_var_show(q->limits.max_integrity_segments, page);  in queue_max_integrity_segments_show()
126 static ssize_t queue_max_segment_size_show(struct request_queue *q, char *page)  in queue_max_segment_size_show()  argument
128 	return queue_var_show(queue_max_segment_size(q), page);  in queue_max_segment_size_show()
131 static ssize_t queue_logical_block_size_show(struct request_queue *q, char *page)  in queue_logical_block_size_show()  argument
133 	return queue_var_show(queue_logical_block_size(q), page);  in queue_logical_block_size_show()
136 static ssize_t queue_physical_block_size_show(struct request_queue *q, char *page)  in queue_physical_block_size_show()  argument
138 	return queue_var_show(queue_physical_block_size(q), page);  in queue_physical_block_size_show()
141 static ssize_t queue_chunk_sectors_show(struct request_queue *q, char *page)  in queue_chunk_sectors_show()  argument
143 	return queue_var_show(q->limits.chunk_sectors, page);  in queue_chunk_sectors_show()
146 static ssize_t queue_io_min_show(struct request_queue *q, char *page)  in queue_io_min_show()  argument
148 	return queue_var_show(queue_io_min(q), page);  in queue_io_min_show()
151 static ssize_t queue_io_opt_show(struct request_queue *q, char *page)  in queue_io_opt_show()  argument
153 	return queue_var_show(queue_io_opt(q), page);  in queue_io_opt_show()
156 static ssize_t queue_discard_granularity_show(struct request_queue *q, char *page)  in queue_discard_granularity_show()  argument
158 	return queue_var_show(q->limits.discard_granularity, page);  in queue_discard_granularity_show()
161 static ssize_t queue_discard_max_hw_show(struct request_queue *q, char *page)  in queue_discard_max_hw_show()  argument
164 	return sprintf(page, "%llu\n",  in queue_discard_max_hw_show()
165 		(unsigned long long)q->limits.max_hw_discard_sectors << 9);  in queue_discard_max_hw_show()
168 static ssize_t queue_discard_max_show(struct request_queue *q, char *page)  in queue_discard_max_show()  argument
170 	return sprintf(page, "%llu\n",  in queue_discard_max_show()
171 		       (unsigned long long)q->limits.max_discard_sectors << 9);  in queue_discard_max_show()
175 				       const char *page, size_t count)  in queue_discard_max_store()  argument
178 	ssize_t ret = queue_var_store(&max_discard, page, count);  in queue_discard_max_store()
183 	if (max_discard & (q->limits.discard_granularity - 1))  in queue_discard_max_store()
184 		return -EINVAL;  in queue_discard_max_store()
188 		return -EINVAL;  in queue_discard_max_store()
190 	if (max_discard > q->limits.max_hw_discard_sectors)  in queue_discard_max_store()
191 		max_discard = q->limits.max_hw_discard_sectors;  in queue_discard_max_store()
193 	q->limits.max_discard_sectors = max_discard;  in queue_discard_max_store()
197 static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page)  in queue_discard_zeroes_data_show()  argument
199 	return queue_var_show(0, page);  in queue_discard_zeroes_data_show()
202 static ssize_t queue_write_same_max_show(struct request_queue *q, char *page)  in queue_write_same_max_show()  argument
204 	return queue_var_show(0, page);  in queue_write_same_max_show()
207 static ssize_t queue_write_zeroes_max_show(struct request_queue *q, char *page)  in queue_write_zeroes_max_show()  argument
209 	return sprintf(page, "%llu\n",  in queue_write_zeroes_max_show()
210 		(unsigned long long)q->limits.max_write_zeroes_sectors << 9);  in queue_write_zeroes_max_show()
214 						 char *page)  in queue_zone_write_granularity_show()  argument
216 	return queue_var_show(queue_zone_write_granularity(q), page);  in queue_zone_write_granularity_show()
219 static ssize_t queue_zone_append_max_show(struct request_queue *q, char *page)  in queue_zone_append_max_show()  argument
221 	unsigned long long max_sectors = q->limits.max_zone_append_sectors;  in queue_zone_append_max_show()
223 	return sprintf(page, "%llu\n", max_sectors << SECTOR_SHIFT);  in queue_zone_append_max_show()
227 queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)  in queue_max_sectors_store()  argument
232 			page_kb = 1 << (PAGE_SHIFT - 10);  in queue_max_sectors_store()
233 	ssize_t ret = queue_var_store(&var, page, count);  in queue_max_sectors_store()
240 					 q->limits.max_dev_sectors >> 1);  in queue_max_sectors_store()
242 		q->limits.max_user_sectors = 0;  in queue_max_sectors_store()
248 			return -EINVAL;  in queue_max_sectors_store()
249 		q->limits.max_user_sectors = max_sectors_kb << 1;  in queue_max_sectors_store()
252 	spin_lock_irq(&q->queue_lock);  in queue_max_sectors_store()
253 	q->limits.max_sectors = max_sectors_kb << 1;  in queue_max_sectors_store()
254 	if (q->disk)  in queue_max_sectors_store()
255 		q->disk->bdi->io_pages = max_sectors_kb >> (PAGE_SHIFT - 10);  in queue_max_sectors_store()
256 	spin_unlock_irq(&q->queue_lock);  in queue_max_sectors_store()
261 static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page)  in queue_max_hw_sectors_show()  argument
265 	return queue_var_show(max_hw_sectors_kb, page);  in queue_max_hw_sectors_show()
268 static ssize_t queue_virt_boundary_mask_show(struct request_queue *q, char *page)  in queue_virt_boundary_mask_show()  argument
270 	return queue_var_show(q->limits.virt_boundary_mask, page);  in queue_virt_boundary_mask_show()
273 static ssize_t queue_dma_alignment_show(struct request_queue *q, char *page)  in queue_dma_alignment_show()  argument
275 	return queue_var_show(queue_dma_alignment(q), page);  in queue_dma_alignment_show()
280 queue_##name##_show(struct request_queue *q, char *page)		\
283 	bit = test_bit(QUEUE_FLAG_##flag, &q->queue_flags);		\
284 	return queue_var_show(neg ? !bit : bit, page);			\
287 queue_##name##_store(struct request_queue *q, const char *page, size_t count) \
291 	ret = queue_var_store(&val, page, count);			\
310 static ssize_t queue_zoned_show(struct request_queue *q, char *page)  in queue_zoned_show()  argument
313 		return sprintf(page, "host-managed\n");  in queue_zoned_show()
314 	return sprintf(page, "none\n");  in queue_zoned_show()
317 static ssize_t queue_nr_zones_show(struct request_queue *q, char *page)  in queue_nr_zones_show()  argument
319 	return queue_var_show(disk_nr_zones(q->disk), page);  in queue_nr_zones_show()
322 static ssize_t queue_max_open_zones_show(struct request_queue *q, char *page)  in queue_max_open_zones_show()  argument
324 	return queue_var_show(bdev_max_open_zones(q->disk->part0), page);  in queue_max_open_zones_show()
327 static ssize_t queue_max_active_zones_show(struct request_queue *q, char *page)  in queue_max_active_zones_show()  argument
329 	return queue_var_show(bdev_max_active_zones(q->disk->part0), page);  in queue_max_active_zones_show()
332 static ssize_t queue_nomerges_show(struct request_queue *q, char *page)  in queue_nomerges_show()  argument
335 			       blk_queue_noxmerges(q), page);  in queue_nomerges_show()
338 static ssize_t queue_nomerges_store(struct request_queue *q, const char *page,  in queue_nomerges_store()  argument
342 	ssize_t ret = queue_var_store(&nm, page, count);  in queue_nomerges_store()
357 static ssize_t queue_rq_affinity_show(struct request_queue *q, char *page)  in queue_rq_affinity_show()  argument
359 	bool set = test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags);  in queue_rq_affinity_show()
360 	bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags);  in queue_rq_affinity_show()
362 	return queue_var_show(set << force, page);  in queue_rq_affinity_show()
366 queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)  in queue_rq_affinity_store()  argument
368 	ssize_t ret = -EINVAL;  in queue_rq_affinity_store()
372 	ret = queue_var_store(&val, page, count);  in queue_rq_affinity_store()
390 static ssize_t queue_poll_delay_show(struct request_queue *q, char *page)  in queue_poll_delay_show()  argument
392 	return sprintf(page, "%d\n", -1);  in queue_poll_delay_show()
395 static ssize_t queue_poll_delay_store(struct request_queue *q, const char *page,  in queue_poll_delay_store()  argument
401 static ssize_t queue_poll_show(struct request_queue *q, char *page)  in queue_poll_show()  argument
403 	return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);  in queue_poll_show()
406 static ssize_t queue_poll_store(struct request_queue *q, const char *page,  in queue_poll_store()  argument
409 	if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))  in queue_poll_store()
410 		return -EINVAL;  in queue_poll_store()
416 static ssize_t queue_io_timeout_show(struct request_queue *q, char *page)  in queue_io_timeout_show()  argument
418 	return sprintf(page, "%u\n", jiffies_to_msecs(q->rq_timeout));  in queue_io_timeout_show()
421 static ssize_t queue_io_timeout_store(struct request_queue *q, const char *page,  in queue_io_timeout_store()  argument
427 	err = kstrtou32(page, 10, &val);  in queue_io_timeout_store()
429 		return -EINVAL;  in queue_io_timeout_store()
436 static ssize_t queue_wc_show(struct request_queue *q, char *page)  in queue_wc_show()  argument
438 	if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))  in queue_wc_show()
439 		return sprintf(page, "write back\n");  in queue_wc_show()
441 	return sprintf(page, "write through\n");  in queue_wc_show()
444 static ssize_t queue_wc_store(struct request_queue *q, const char *page,  in queue_wc_store()  argument
447 	if (!strncmp(page, "write back", 10)) {  in queue_wc_store()
448 		if (!test_bit(QUEUE_FLAG_HW_WC, &q->queue_flags))  in queue_wc_store()
449 			return -EINVAL;  in queue_wc_store()
451 	} else if (!strncmp(page, "write through", 13) ||  in queue_wc_store()
452 		 !strncmp(page, "none", 4)) {  in queue_wc_store()
455 		return -EINVAL;  in queue_wc_store()
461 static ssize_t queue_fua_show(struct request_queue *q, char *page)  in queue_fua_show()  argument
463 	return sprintf(page, "%u\n", test_bit(QUEUE_FLAG_FUA, &q->queue_flags));  in queue_fua_show()
466 static ssize_t queue_dax_show(struct request_queue *q, char *page)  in queue_dax_show()  argument
468 	return queue_var_show(blk_queue_dax(q), page);  in queue_dax_show()
542 static ssize_t queue_var_store64(s64 *var, const char *page)  in queue_var_store64()  argument
547 	err = kstrtos64(page, 10, &v);  in queue_var_store64()
555 static ssize_t queue_wb_lat_show(struct request_queue *q, char *page)  in queue_wb_lat_show()  argument
558 		return -EINVAL;  in queue_wb_lat_show()
561 		return sprintf(page, "0\n");  in queue_wb_lat_show()
563 	return sprintf(page, "%llu\n", div_u64(wbt_get_min_lat(q), 1000));  in queue_wb_lat_show()
566 static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,  in queue_wb_lat_store()  argument
573 	ret = queue_var_store64(&val, page);  in queue_wb_lat_store()
576 	if (val < -1)  in queue_wb_lat_store()
577 		return -EINVAL;  in queue_wb_lat_store()
581 		ret = wbt_init(q->disk);  in queue_wb_lat_store()
586 	if (val == -1)  in queue_wb_lat_store()
613 /* Common attributes for bio-based and request-based queues. */
658 /* Request-based queue attributes that are not relevant for bio-based queues. */
674 	struct request_queue *q = disk->queue;  in queue_attr_visible()
681 	return attr->mode;  in queue_attr_visible()
688 	struct request_queue *q = disk->queue;  in blk_mq_queue_attr_visible()
693 	if (attr == &queue_io_timeout_entry.attr && !q->mq_ops->timeout)  in blk_mq_queue_attr_visible()
696 	return attr->mode;  in blk_mq_queue_attr_visible()
712 queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)  in queue_attr_show()  argument
716 	struct request_queue *q = disk->queue;  in queue_attr_show()
719 	if (!entry->show)  in queue_attr_show()
720 		return -EIO;  in queue_attr_show()
721 	mutex_lock(&q->sysfs_lock);  in queue_attr_show()
722 	res = entry->show(q, page);  in queue_attr_show()
723 	mutex_unlock(&q->sysfs_lock);  in queue_attr_show()
729 		    const char *page, size_t length)  in queue_attr_store()  argument
733 	struct request_queue *q = disk->queue;  in queue_attr_store()
736 	if (!entry->store)  in queue_attr_store()
737 		return -EIO;  in queue_attr_store()
739 	mutex_lock(&q->sysfs_lock);  in queue_attr_store()
740 	res = entry->store(q, page, length);  in queue_attr_store()
741 	mutex_unlock(&q->sysfs_lock);  in queue_attr_store()
769 	struct request_queue *q = disk->queue;  in blk_debugfs_remove()
771 	mutex_lock(&q->debugfs_mutex);  in blk_debugfs_remove()
773 	debugfs_remove_recursive(q->debugfs_dir);  in blk_debugfs_remove()
774 	q->debugfs_dir = NULL;  in blk_debugfs_remove()
775 	q->sched_debugfs_dir = NULL;  in blk_debugfs_remove()
776 	q->rqos_debugfs_dir = NULL;  in blk_debugfs_remove()
777 	mutex_unlock(&q->debugfs_mutex);  in blk_debugfs_remove()
781  * blk_register_queue - register a block layer queue with sysfs
786 	struct request_queue *q = disk->queue;  in blk_register_queue()
789 	mutex_lock(&q->sysfs_dir_lock);  in blk_register_queue()
790 	kobject_init(&disk->queue_kobj, &blk_queue_ktype);  in blk_register_queue()
791 	ret = kobject_add(&disk->queue_kobj, &disk_to_dev(disk)->kobj, "queue");  in blk_register_queue()
800 	mutex_lock(&q->sysfs_lock);  in blk_register_queue()
802 	mutex_lock(&q->debugfs_mutex);  in blk_register_queue()
803 	q->debugfs_dir = debugfs_create_dir(disk->disk_name, blk_debugfs_root);  in blk_register_queue()
806 	mutex_unlock(&q->debugfs_mutex);  in blk_register_queue()
812 	if (q->elevator) {  in blk_register_queue()
827 	kobject_uevent(&disk->queue_kobj, KOBJ_ADD);  in blk_register_queue()
828 	if (q->elevator)  in blk_register_queue()
829 		kobject_uevent(&q->elevator->kobj, KOBJ_ADD);  in blk_register_queue()
830 	mutex_unlock(&q->sysfs_lock);  in blk_register_queue()
831 	mutex_unlock(&q->sysfs_dir_lock);  in blk_register_queue()
835 	 * request_queues for non-existent devices.  Shutting down a fully  in blk_register_queue()
840 	 * request_queues for non-existent devices never get registered.  in blk_register_queue()
844 		percpu_ref_switch_to_percpu(&q->q_usage_counter);  in blk_register_queue()
855 	mutex_unlock(&q->sysfs_lock);  in blk_register_queue()
857 	kobject_put(&disk->queue_kobj);  in blk_register_queue()
858 	mutex_unlock(&q->sysfs_dir_lock);  in blk_register_queue()
863  * blk_unregister_queue - counterpart of blk_register_queue()
871 	struct request_queue *q = disk->queue;  in blk_unregister_queue()
876 	/* Return early if disk->queue was never registered. */  in blk_unregister_queue()
885 	mutex_lock(&q->sysfs_lock);  in blk_unregister_queue()
887 	mutex_unlock(&q->sysfs_lock);  in blk_unregister_queue()
889 	mutex_lock(&q->sysfs_dir_lock);  in blk_unregister_queue()
898 	mutex_lock(&q->sysfs_lock);  in blk_unregister_queue()
901 	mutex_unlock(&q->sysfs_lock);  in blk_unregister_queue()
904 	kobject_uevent(&disk->queue_kobj, KOBJ_REMOVE);  in blk_unregister_queue()
905 	kobject_del(&disk->queue_kobj);  in blk_unregister_queue()
906 	mutex_unlock(&q->sysfs_dir_lock);  in blk_unregister_queue()