275 static void null_del_dev(struct nullb *nullb);
277 static struct nullb *null_find_dev_by_name(const char *name);
382 	if (!dev->nullb)  in nullb_update_nr_hw_queues()
407 	set = dev->nullb->tag_set;  in nullb_update_nr_hw_queues()
513 			null_del_dev(dev->nullb);  in nullb_device_power_store()
712 		null_del_dev(dev->nullb);  in nullb_group_drop_item()
772 static inline int null_cache_active(struct nullb *nullb)  in null_cache_active()  argument
774 	return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);  in null_cache_active()
900 static void null_free_sector(struct nullb *nullb, sector_t sector,  in null_free_sector()  argument
908 	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;  in null_free_sector()
921 				nullb->dev->curr_cache -= PAGE_SIZE;  in null_free_sector()
926 static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,  in null_radix_tree_insert()  argument
931 	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;  in null_radix_tree_insert()
938 		nullb->dev->curr_cache += PAGE_SIZE;  in null_radix_tree_insert()
972 static struct nullb_page *__null_lookup_page(struct nullb *nullb,  in __null_lookup_page()  argument
983 	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;  in __null_lookup_page()
993 static struct nullb_page *null_lookup_page(struct nullb *nullb,  in null_lookup_page()  argument
999 		page = __null_lookup_page(nullb, sector, for_write, true);  in null_lookup_page()
1002 	return __null_lookup_page(nullb, sector, for_write, false);  in null_lookup_page()
1005 static struct nullb_page *null_insert_page(struct nullb *nullb,  in null_insert_page()  argument
1007 	__releases(&nullb->lock)  in null_insert_page()
1008 	__acquires(&nullb->lock)  in null_insert_page()
1013 	t_page = null_lookup_page(nullb, sector, true, ignore_cache);  in null_insert_page()
1017 	spin_unlock_irq(&nullb->lock);  in null_insert_page()
1026 	spin_lock_irq(&nullb->lock);  in null_insert_page()
1029 	t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);  in null_insert_page()
1036 	spin_lock_irq(&nullb->lock);  in null_insert_page()
1037 	return null_lookup_page(nullb, sector, true, ignore_cache);  in null_insert_page()
1040 static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)  in null_flush_cache_page()  argument
1050 	t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);  in null_flush_cache_page()
1056 			ret = radix_tree_delete_item(&nullb->dev->data,  in null_flush_cache_page()
1070 			i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {  in null_flush_cache_page()
1074 				nullb->dev->blocksize);  in null_flush_cache_page()
1082 	ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);  in null_flush_cache_page()
1084 	nullb->dev->curr_cache -= PAGE_SIZE;  in null_flush_cache_page()
1089 static int null_make_cache_space(struct nullb *nullb, unsigned long n)  in null_make_cache_space()  argument
1096 	if ((nullb->dev->cache_size * 1024 * 1024) >  in null_make_cache_space()
1097 	     nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)  in null_make_cache_space()
1100 	nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,  in null_make_cache_space()
1101 			(void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);  in null_make_cache_space()
1107 		nullb->cache_flush_pos = c_pages[i]->page->private;  in null_make_cache_space()
1122 		err = null_flush_cache_page(nullb, c_pages[i]);  in null_make_cache_space()
1131 			nullb->cache_flush_pos = 0;  in null_make_cache_space()
1134 			spin_unlock_irq(&nullb->lock);  in null_make_cache_space()
1135 			spin_lock_irq(&nullb->lock);  in null_make_cache_space()
1142 static blk_status_t copy_to_nullb(struct nullb *nullb, void *source,  in copy_to_nullb()  argument
1150 		temp = min3(nullb->dev->blocksize, n - count,  in copy_to_nullb()
1154 		if (null_cache_active(nullb) && !is_fua)  in copy_to_nullb()
1155 			null_make_cache_space(nullb, PAGE_SIZE);  in copy_to_nullb()
1157 		t_page = null_insert_page(nullb, sector,  in copy_to_nullb()
1158 			!null_cache_active(nullb) || is_fua);  in copy_to_nullb()
1168 			null_free_sector(nullb, sector, true);  in copy_to_nullb()
1176 static void copy_from_nullb(struct nullb *nullb, void *dest, loff_t pos,  in copy_from_nullb()  argument
1184 		temp = min3(nullb->dev->blocksize, n - count,  in copy_from_nullb()
1188 		t_page = null_lookup_page(nullb, sector, false,  in copy_from_nullb()
1189 			!null_cache_active(nullb));  in copy_from_nullb()
1204 	struct nullb *nullb = dev->nullb;  in null_handle_discard()  local
1208 	spin_lock_irq(&nullb->lock);  in null_handle_discard()
1211 		null_free_sector(nullb, sector, false);  in null_handle_discard()
1212 		if (null_cache_active(nullb))  in null_handle_discard()
1213 			null_free_sector(nullb, sector, true);  in null_handle_discard()
1217 	spin_unlock_irq(&nullb->lock);  in null_handle_discard()
1222 static blk_status_t null_handle_flush(struct nullb *nullb)  in null_handle_flush()  argument
1226 	if (!null_cache_active(nullb))  in null_handle_flush()
1229 	spin_lock_irq(&nullb->lock);  in null_handle_flush()
1231 		err = null_make_cache_space(nullb,  in null_handle_flush()
1232 			nullb->dev->cache_size * 1024 * 1024);  in null_handle_flush()
1233 		if (err || nullb->dev->curr_cache == 0)  in null_handle_flush()
1237 	WARN_ON(!radix_tree_empty(&nullb->dev->cache));  in null_handle_flush()
1238 	spin_unlock_irq(&nullb->lock);  in null_handle_flush()
1242 static blk_status_t null_transfer(struct nullb *nullb, struct page *page,  in null_transfer()  argument
1246 	struct nullb_device *dev = nullb->dev;  in null_transfer()
1254 			valid_len = null_zone_valid_read_len(nullb,  in null_transfer()
1261 			copy_from_nullb(nullb, p, pos, valid_len);  in null_transfer()
1271 		err = copy_to_nullb(nullb, p, pos, len, is_fua);  in null_transfer()
1286 	struct nullb *nullb = cmd->nq->dev->nullb;  in null_handle_data_transfer()  local
1295 	spin_lock_irq(&nullb->lock);  in null_handle_data_transfer()
1300 		err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,  in null_handle_data_transfer()
1310 	spin_unlock_irq(&nullb->lock);  in null_handle_data_transfer()
1318 	struct nullb *nullb = dev->nullb;  in null_handle_throttled()  local
1322 	if (!hrtimer_active(&nullb->bw_timer))  in null_handle_throttled()
1323 		hrtimer_restart(&nullb->bw_timer);  in null_handle_throttled()
1325 	if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) {  in null_handle_throttled()
1326 		blk_mq_stop_hw_queues(nullb->q);  in null_handle_throttled()
1328 		if (atomic_long_read(&nullb->cur_bytes) > 0)  in null_handle_throttled()
1329 			blk_mq_start_stopped_hw_queues(nullb->q, true);  in null_handle_throttled()
1446 	struct nullb *nullb = dev->nullb;  in null_handle_cmd()  local
1450 		cmd->error = null_handle_flush(nullb);  in null_handle_cmd()
1469 	struct nullb *nullb = container_of(timer, struct nullb, bw_timer);  in nullb_bwtimer_fn()  local
1471 	unsigned int mbps = nullb->dev->mbps;  in nullb_bwtimer_fn()
1473 	if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))  in nullb_bwtimer_fn()
1476 	atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));  in nullb_bwtimer_fn()
1477 	blk_mq_start_stopped_hw_queues(nullb->q, true);  in nullb_bwtimer_fn()
1479 	hrtimer_forward_now(&nullb->bw_timer, timer_interval);  in nullb_bwtimer_fn()
1484 static void nullb_setup_bwtimer(struct nullb *nullb)  in nullb_setup_bwtimer()  argument
1488 	hrtimer_setup(&nullb->bw_timer, nullb_bwtimer_fn, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in nullb_setup_bwtimer()
1489 	atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));  in nullb_setup_bwtimer()
1490 	hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);  in nullb_setup_bwtimer()
1537 	struct nullb *nullb = set->driver_data;  in null_map_queues()  local
1542 	if (nullb) {  in null_map_queues()
1543 		struct nullb_device *dev = nullb->dev;  in null_map_queues()
1725 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)  in null_init_queue()  argument
1727 	nq->dev = nullb->dev;  in null_init_queue()
1735 	struct nullb *nullb = hctx->queue->queuedata;  in null_init_hctx()  local
1738 	if (should_init_hctx_fail(nullb->dev))  in null_init_hctx()
1741 	nq = &nullb->queues[hctx_idx];  in null_init_hctx()
1743 	null_init_queue(nullb, nq);  in null_init_hctx()
1758 static void null_del_dev(struct nullb *nullb)  in null_del_dev()  argument
1762 	if (!nullb)  in null_del_dev()
1765 	dev = nullb->dev;  in null_del_dev()
1767 	ida_free(&nullb_indexes, nullb->index);  in null_del_dev()
1769 	list_del_init(&nullb->list);  in null_del_dev()
1771 	del_gendisk(nullb->disk);  in null_del_dev()
1773 	if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {  in null_del_dev()
1774 		hrtimer_cancel(&nullb->bw_timer);  in null_del_dev()
1775 		atomic_long_set(&nullb->cur_bytes, LONG_MAX);  in null_del_dev()
1776 		blk_mq_start_stopped_hw_queues(nullb->q, true);  in null_del_dev()
1779 	put_disk(nullb->disk);  in null_del_dev()
1780 	if (nullb->tag_set == &nullb->__tag_set)  in null_del_dev()
1781 		blk_mq_free_tag_set(nullb->tag_set);  in null_del_dev()
1782 	kfree(nullb->queues);  in null_del_dev()
1783 	if (null_cache_active(nullb))  in null_del_dev()
1784 		null_free_device_storage(nullb->dev, true);  in null_del_dev()
1785 	kfree(nullb);  in null_del_dev()
1786 	dev->nullb = NULL;  in null_del_dev()
1789 static void null_config_discard(struct nullb *nullb, struct queue_limits *lim)  in null_config_discard()  argument
1791 	if (nullb->dev->discard == false)  in null_config_discard()
1794 	if (!nullb->dev->memory_backed) {  in null_config_discard()
1795 		nullb->dev->discard = false;  in null_config_discard()
1800 	if (nullb->dev->zoned) {  in null_config_discard()
1801 		nullb->dev->discard = false;  in null_config_discard()
1814 static int setup_queues(struct nullb *nullb)  in setup_queues()  argument
1821 	nullb->queues = kzalloc_objs(struct nullb_queue, nqueues);  in setup_queues()
1822 	if (!nullb->queues)  in setup_queues()
1864 static int null_setup_tagset(struct nullb *nullb)  in null_setup_tagset()  argument
1866 	if (nullb->dev->shared_tags) {  in null_setup_tagset()
1867 		nullb->tag_set = &tag_set;  in null_setup_tagset()
1871 	nullb->tag_set = &nullb->__tag_set;  in null_setup_tagset()
1872 	nullb->tag_set->driver_data = nullb;  in null_setup_tagset()
1873 	nullb->tag_set->nr_hw_queues = nullb->dev->submit_queues;  in null_setup_tagset()
1874 	nullb->tag_set->queue_depth = nullb->dev->hw_queue_depth;  in null_setup_tagset()
1875 	nullb->tag_set->numa_node = nullb->dev->home_node;  in null_setup_tagset()
1876 	if (nullb->dev->no_sched)  in null_setup_tagset()
1877 		nullb->tag_set->flags |= BLK_MQ_F_NO_SCHED_BY_DEFAULT;  in null_setup_tagset()
1878 	if (nullb->dev->shared_tag_bitmap)  in null_setup_tagset()
1879 		nullb->tag_set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;  in null_setup_tagset()
1880 	if (nullb->dev->blocking)  in null_setup_tagset()
1881 		nullb->tag_set->flags |= BLK_MQ_F_BLOCKING;  in null_setup_tagset()
1882 	return null_init_tag_set(nullb->tag_set, nullb->dev->poll_queues);  in null_setup_tagset()
1964 	struct nullb *nullb;  in null_add_dev()  local
1971 	nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);  in null_add_dev()
1972 	if (!nullb) {  in null_add_dev()
1976 	nullb->dev = dev;  in null_add_dev()
1977 	dev->nullb = nullb;  in null_add_dev()
1979 	spin_lock_init(&nullb->lock);  in null_add_dev()
1981 	rv = setup_queues(nullb);  in null_add_dev()
1985 	rv = null_setup_tagset(nullb);  in null_add_dev()
1991 	null_config_discard(nullb, &lim);  in null_add_dev()
1999 		set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);  in null_add_dev()
2008 	nullb->disk = blk_mq_alloc_disk(nullb->tag_set, &lim, nullb);  in null_add_dev()
2009 	if (IS_ERR(nullb->disk)) {  in null_add_dev()
2010 		rv = PTR_ERR(nullb->disk);  in null_add_dev()
2013 	nullb->q = nullb->disk->queue;  in null_add_dev()
2017 		nullb_setup_bwtimer(nullb);  in null_add_dev()
2020 	nullb->q->queuedata = nullb;  in null_add_dev()
2026 	nullb->index = rv;  in null_add_dev()
2031 		snprintf(nullb->disk_name, sizeof(nullb->disk_name),  in null_add_dev()
2034 		sprintf(nullb->disk_name, "nullb%d", nullb->index);  in null_add_dev()
2037 	set_capacity(nullb->disk,  in null_add_dev()
2038 		((sector_t)nullb->dev->size * SZ_1M) >> SECTOR_SHIFT);  in null_add_dev()
2039 	nullb->disk->major = null_major;  in null_add_dev()
2040 	nullb->disk->first_minor = nullb->index;  in null_add_dev()
2041 	nullb->disk->minors = 1;  in null_add_dev()
2042 	nullb->disk->fops = &null_ops;  in null_add_dev()
2043 	nullb->disk->private_data = nullb;  in null_add_dev()
2044 	strscpy(nullb->disk->disk_name, nullb->disk_name);  in null_add_dev()
2046 	if (nullb->dev->zoned) {  in null_add_dev()
2047 		rv = null_register_zoned_dev(nullb);  in null_add_dev()
2052 	rv = add_disk(nullb->disk);  in null_add_dev()
2056 	list_add_tail(&nullb->list, &nullb_list);  in null_add_dev()
2058 	pr_info("disk %s created\n", nullb->disk_name);  in null_add_dev()
2063 	ida_free(&nullb_indexes, nullb->index);  in null_add_dev()
2065 	put_disk(nullb->disk);  in null_add_dev()
2069 	if (nullb->tag_set == &nullb->__tag_set)  in null_add_dev()
2070 		blk_mq_free_tag_set(nullb->tag_set);  in null_add_dev()
2072 	kfree(nullb->queues);  in null_add_dev()
2074 	kfree(nullb);  in null_add_dev()
2075 	dev->nullb = NULL;  in null_add_dev()
2080 static struct nullb *null_find_dev_by_name(const char *name)  in null_find_dev_by_name()
2082 	struct nullb *nullb = NULL, *nb;  in null_find_dev_by_name()  local
2087 			nullb = nb;  in null_find_dev_by_name()
2093 	return nullb;  in null_find_dev_by_name()
2116 static void null_destroy_dev(struct nullb *nullb)  in null_destroy_dev()  argument
2118 	struct nullb_device *dev = nullb->dev;  in null_destroy_dev()
2120 	null_del_dev(nullb);  in null_destroy_dev()
2129 	struct nullb *nullb;  in null_init()  local
2188 		nullb = list_entry(nullb_list.next, struct nullb, list);  in null_init()
2189 		null_destroy_dev(nullb);  in null_init()
2199 	struct nullb *nullb;  in null_exit()  local
2207 		nullb = list_entry(nullb_list.next, struct nullb, list);  in null_exit()
2208 		null_destroy_dev(nullb);  in null_exit()