70 	struct nbd_device *nbd;  member
149 	struct nbd_device *nbd;  member
163 #define nbd_name(nbd) ((nbd)->disk->disk_name)  argument
171 static int nbd_dev_dbg_init(struct nbd_device *nbd);
172 static void nbd_dev_dbg_close(struct nbd_device *nbd);
173 static void nbd_config_put(struct nbd_device *nbd);
177 static void nbd_disconnect_and_put(struct nbd_device *nbd);
179 static inline struct device *nbd_to_dev(struct nbd_device *nbd)  in nbd_to_dev()  argument
181 	return disk_to_dev(nbd->disk);  in nbd_to_dev()
240 	struct nbd_device *nbd = disk->private_data;  in pid_show()  local
242 	return sprintf(buf, "%d\n", nbd->pid);  in pid_show()
254 	struct nbd_device *nbd = disk->private_data;  in backend_show()  local
256 	return sprintf(buf, "%s\n", nbd->backend ?: "");  in backend_show()
264 static void nbd_dev_remove(struct nbd_device *nbd)  in nbd_dev_remove()  argument
266 	struct gendisk *disk = nbd->disk;  in nbd_dev_remove()
269 	blk_mq_free_tag_set(&nbd->tag_set);  in nbd_dev_remove()
276 	idr_remove(&nbd_index_idr, nbd->index);  in nbd_dev_remove()
278 	destroy_workqueue(nbd->recv_workq);  in nbd_dev_remove()
287 static void nbd_put(struct nbd_device *nbd)  in nbd_put()  argument
289 	if (!refcount_dec_and_test(&nbd->refs))  in nbd_put()
293 	if (test_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags))  in nbd_put()
294 		queue_work(nbd_del_wq, &nbd->remove_work);  in nbd_put()
296 		nbd_dev_remove(nbd);  in nbd_put()
305 static void nbd_mark_nsock_dead(struct nbd_device *nbd, struct nbd_sock *nsock,  in nbd_mark_nsock_dead()  argument
308 	if (!nsock->dead && notify && !nbd_disconnected(nbd->config)) {  in nbd_mark_nsock_dead()
313 			args->index = nbd->index;  in nbd_mark_nsock_dead()
319 		if (atomic_dec_return(&nbd->config->live_connections) == 0) {  in nbd_mark_nsock_dead()
321 					       &nbd->config->runtime_flags)) {  in nbd_mark_nsock_dead()
323 					&nbd->config->runtime_flags);  in nbd_mark_nsock_dead()
324 				dev_info(nbd_to_dev(nbd),  in nbd_mark_nsock_dead()
334 static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize, loff_t blksize)  in nbd_set_size()  argument
348 	nbd->config->bytesize = bytesize;  in nbd_set_size()
349 	nbd->config->blksize_bits = __ffs(blksize);  in nbd_set_size()
351 	if (!nbd->pid)  in nbd_set_size()
354 	lim = queue_limits_start_update(nbd->disk->queue);  in nbd_set_size()
355 	if (nbd->config->flags & NBD_FLAG_SEND_TRIM)  in nbd_set_size()
359 	if (!(nbd->config->flags & NBD_FLAG_SEND_FLUSH)) {  in nbd_set_size()
361 	} else if (nbd->config->flags & NBD_FLAG_SEND_FUA) {  in nbd_set_size()
367 	if (nbd->config->flags & NBD_FLAG_ROTATIONAL)  in nbd_set_size()
369 	if (nbd->config->flags & NBD_FLAG_SEND_WRITE_ZEROES)  in nbd_set_size()
374 	error = queue_limits_commit_update_frozen(nbd->disk->queue, &lim);  in nbd_set_size()
379 		set_bit(GD_NEED_PART_SCAN, &nbd->disk->state);  in nbd_set_size()
380 	if (!set_capacity_and_notify(nbd->disk, bytesize >> 9))  in nbd_set_size()
381 		kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE);  in nbd_set_size()
389 	dev_dbg(nbd_to_dev(cmd->nbd), "request %p: %s\n", req,  in nbd_complete_rq()
398 static void sock_shutdown(struct nbd_device *nbd)  in sock_shutdown()  argument
400 	struct nbd_config *config = nbd->config;  in sock_shutdown()
411 		nbd_mark_nsock_dead(nbd, nsock, 0);  in sock_shutdown()
414 	dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n");  in sock_shutdown()
435 static struct nbd_config *nbd_get_config_unlocked(struct nbd_device *nbd)  in nbd_get_config_unlocked()  argument
437 	if (refcount_inc_not_zero(&nbd->config_refs)) {  in nbd_get_config_unlocked()
445 		return nbd->config;  in nbd_get_config_unlocked()
454 	struct nbd_device *nbd = cmd->nbd;  in nbd_xmit_timeout()  local
471 	config = nbd_get_config_unlocked(nbd);  in nbd_xmit_timeout()
480 	    (config->num_connections == 1 && nbd->tag_set.timeout)) {  in nbd_xmit_timeout()
481 		dev_err_ratelimited(nbd_to_dev(nbd),  in nbd_xmit_timeout()
503 					nbd_mark_nsock_dead(nbd, nsock, 1);  in nbd_xmit_timeout()
508 			nbd_config_put(nbd);  in nbd_xmit_timeout()
513 	if (!nbd->tag_set.timeout) {  in nbd_xmit_timeout()
520 …dev_info(nbd_to_dev(nbd), "Possible stuck request %p: control (%s@%llu,%uB). Runtime %u seconds\n",  in nbd_xmit_timeout()
530 			nbd_config_put(nbd);  in nbd_xmit_timeout()
535 		nbd_config_put(nbd);  in nbd_xmit_timeout()
539 	dev_err_ratelimited(nbd_to_dev(nbd), "Connection timed out\n");  in nbd_xmit_timeout()
544 	sock_shutdown(nbd);  in nbd_xmit_timeout()
545 	nbd_config_put(nbd);  in nbd_xmit_timeout()
551 static int __sock_xmit(struct nbd_device *nbd, struct socket *sock, int send,  in __sock_xmit()  argument
559 		dev_err_ratelimited(disk_to_dev(nbd->disk),  in __sock_xmit()
599 static int sock_xmit(struct nbd_device *nbd, int index, int send,  in sock_xmit()  argument
602 	struct nbd_config *config = nbd->config;  in sock_xmit()
605 	return __sock_xmit(nbd, sock, send, iter, msg_flags, sent);  in sock_xmit()
625 static void nbd_sched_pending_work(struct nbd_device *nbd,  in nbd_sched_pending_work()  argument
637 	refcount_inc(&nbd->config_refs);  in nbd_sched_pending_work()
645 static blk_status_t nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd,  in nbd_send_cmd()  argument
649 	struct nbd_config *config = nbd->config;  in nbd_send_cmd()
672 		dev_err_ratelimited(disk_to_dev(nbd->disk),  in nbd_send_cmd()
710 	trace_nbd_send_request(&request, nbd->index, blk_mq_rq_from_pdu(cmd));  in nbd_send_cmd()
712 	dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n",  in nbd_send_cmd()
715 	result = sock_xmit(nbd, index, 1, &from,  in nbd_send_cmd()
726 				nbd_sched_pending_work(nbd, nsock, cmd, sent);  in nbd_send_cmd()
732 		dev_err_ratelimited(disk_to_dev(nbd->disk),  in nbd_send_cmd()
750 			dev_dbg(nbd_to_dev(nbd), "request %p: sending %d bytes data\n",  in nbd_send_cmd()
761 			result = sock_xmit(nbd, index, 1, &from, flags, &sent);  in nbd_send_cmd()
764 					nbd_sched_pending_work(nbd, nsock, cmd, sent);  in nbd_send_cmd()
767 				dev_err(disk_to_dev(nbd->disk),  in nbd_send_cmd()
800 	dev_err_ratelimited(disk_to_dev(nbd->disk),  in nbd_send_cmd()
802 	nbd_mark_nsock_dead(nbd, nsock, 1);  in nbd_send_cmd()
813 	struct nbd_device *nbd = cmd->nbd;  in nbd_pending_cmd_work()  local
825 		nbd_send_cmd(nbd, cmd, cmd->index);  in nbd_pending_cmd_work()
842 	nbd_config_put(nbd);  in nbd_pending_cmd_work()
845 static int nbd_read_reply(struct nbd_device *nbd, struct socket *sock,  in nbd_read_reply()  argument
854 	result = __sock_xmit(nbd, sock, 0, &to, MSG_WAITALL, NULL);  in nbd_read_reply()
856 		if (!nbd_disconnected(nbd->config))  in nbd_read_reply()
857 			dev_err(disk_to_dev(nbd->disk),  in nbd_read_reply()
863 		dev_err(disk_to_dev(nbd->disk), "Wrong magic (0x%lx)\n",  in nbd_read_reply()
872 static struct nbd_cmd *nbd_handle_reply(struct nbd_device *nbd, int index,  in nbd_handle_reply()  argument
886 	if (hwq < nbd->tag_set.nr_hw_queues)  in nbd_handle_reply()
887 		req = blk_mq_tag_to_rq(nbd->tag_set.tags[hwq],  in nbd_handle_reply()
890 		dev_err(disk_to_dev(nbd->disk), "Unexpected reply (%d) %p\n",  in nbd_handle_reply()
899 		dev_err(disk_to_dev(nbd->disk), "Suspicious reply %d (status %u flags %lu)",  in nbd_handle_reply()
905 		dev_err(disk_to_dev(nbd->disk), "Unexpected reply %d from different sock %d (expected %d)",  in nbd_handle_reply()
911 		dev_err(disk_to_dev(nbd->disk), "Double reply on req %p, cmd_cookie %u, handle cookie %u\n",  in nbd_handle_reply()
917 		dev_err(disk_to_dev(nbd->disk), "Command already handled %p\n",  in nbd_handle_reply()
923 		dev_err(disk_to_dev(nbd->disk), "Raced with timeout on req %p\n",  in nbd_handle_reply()
929 		dev_err(disk_to_dev(nbd->disk), "Other side returned error (%d)\n",  in nbd_handle_reply()
935 	dev_dbg(nbd_to_dev(nbd), "request %p: got reply\n", req);  in nbd_handle_reply()
943 			result = sock_xmit(nbd, index, 0, &to, MSG_WAITALL, NULL);  in nbd_handle_reply()
945 				dev_err(disk_to_dev(nbd->disk), "Receive data failed (result %d)\n",  in nbd_handle_reply()
953 				if (nbd_disconnected(nbd->config)) {  in nbd_handle_reply()
960 			dev_dbg(nbd_to_dev(nbd), "request %p: got %d bytes data\n",  in nbd_handle_reply()
975 	struct nbd_device *nbd = args->nbd;  in recv_work()  local
976 	struct nbd_config *config = nbd->config;  in recv_work()
977 	struct request_queue *q = nbd->disk->queue;  in recv_work()
985 		if (nbd_read_reply(nbd, nsock->sock, &reply))  in recv_work()
995 			dev_err(disk_to_dev(nbd->disk), "%s: no io inflight\n",  in recv_work()
1000 		cmd = nbd_handle_reply(nbd, args->index, &reply);  in recv_work()
1021 	nbd_mark_nsock_dead(nbd, nsock, 1);  in recv_work()
1026 	nbd_config_put(nbd);  in recv_work()
1050 static void nbd_clear_que(struct nbd_device *nbd)  in nbd_clear_que()  argument
1052 	blk_mq_quiesce_queue(nbd->disk->queue);  in nbd_clear_que()
1053 	blk_mq_tagset_busy_iter(&nbd->tag_set, nbd_clear_req, NULL);  in nbd_clear_que()
1054 	blk_mq_unquiesce_queue(nbd->disk->queue);  in nbd_clear_que()
1055 	dev_dbg(disk_to_dev(nbd->disk), "queue cleared\n");  in nbd_clear_que()
1058 static int find_fallback(struct nbd_device *nbd, int index)  in find_fallback()  argument
1060 	struct nbd_config *config = nbd->config;  in find_fallback()
1069 		dev_err_ratelimited(disk_to_dev(nbd->disk),  in find_fallback()
1092 			dev_err_ratelimited(disk_to_dev(nbd->disk),  in find_fallback()
1101 static int wait_for_reconnect(struct nbd_device *nbd)  in wait_for_reconnect()  argument
1103 	struct nbd_config *config = nbd->config;  in wait_for_reconnect()
1120 	struct nbd_device *nbd = cmd->nbd;  in nbd_handle_cmd()  local
1127 	config = nbd_get_config_unlocked(nbd);  in nbd_handle_cmd()
1129 		dev_err_ratelimited(disk_to_dev(nbd->disk),  in nbd_handle_cmd()
1135 		dev_err_ratelimited(disk_to_dev(nbd->disk),  in nbd_handle_cmd()
1137 		nbd_config_put(nbd);  in nbd_handle_cmd()
1146 		index = find_fallback(nbd, index);  in nbd_handle_cmd()
1149 			if (wait_for_reconnect(nbd)) {  in nbd_handle_cmd()
1159 			sock_shutdown(nbd);  in nbd_handle_cmd()
1160 			nbd_config_put(nbd);  in nbd_handle_cmd()
1177 	ret = nbd_send_cmd(nbd, cmd, index);  in nbd_handle_cmd()
1180 	nbd_config_put(nbd);  in nbd_handle_cmd()
1213 static struct socket *nbd_get_socket(struct nbd_device *nbd, unsigned long fd,  in nbd_get_socket()  argument
1225 		dev_err(disk_to_dev(nbd->disk), "Unsupported socket: should be TCP or UNIX.\n");  in nbd_get_socket()
1232 		dev_err(disk_to_dev(nbd->disk), "Unsupported socket: shutdown callout must be supported.\n");  in nbd_get_socket()
1241 static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg,  in nbd_add_socket()  argument
1244 	struct nbd_config *config = nbd->config;  in nbd_add_socket()
1254 	sock = nbd_get_socket(nbd, arg, &err);  in nbd_add_socket()
1262 	memflags = blk_mq_freeze_queue(nbd->disk->queue);  in nbd_add_socket()
1264 	if (!netlink && !nbd->task_setup &&  in nbd_add_socket()
1266 		nbd->task_setup = current;  in nbd_add_socket()
1269 	    (nbd->task_setup != current ||  in nbd_add_socket()
1271 		dev_err(disk_to_dev(nbd->disk),  in nbd_add_socket()
1303 	blk_mq_unfreeze_queue(nbd->disk->queue, memflags);  in nbd_add_socket()
1308 	blk_mq_unfreeze_queue(nbd->disk->queue, memflags);  in nbd_add_socket()
1313 static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)  in nbd_reconnect_socket()  argument
1315 	struct nbd_config *config = nbd->config;  in nbd_reconnect_socket()
1321 	sock = nbd_get_socket(nbd, arg, &err);  in nbd_reconnect_socket()
1343 		if (nbd->tag_set.timeout)  in nbd_reconnect_socket()
1344 			sock->sk->sk_sndtimeo = nbd->tag_set.timeout;  in nbd_reconnect_socket()
1346 		refcount_inc(&nbd->config_refs);  in nbd_reconnect_socket()
1353 		args->nbd = nbd;  in nbd_reconnect_socket()
1364 		queue_work(nbd->recv_workq, &args->work);  in nbd_reconnect_socket()
1375 static void nbd_bdev_reset(struct nbd_device *nbd)  in nbd_bdev_reset()  argument
1377 	if (disk_openers(nbd->disk) > 1)  in nbd_bdev_reset()
1379 	set_capacity(nbd->disk, 0);  in nbd_bdev_reset()
1382 static void nbd_parse_flags(struct nbd_device *nbd)  in nbd_parse_flags()  argument
1384 	if (nbd->config->flags & NBD_FLAG_READ_ONLY)  in nbd_parse_flags()
1385 		set_disk_ro(nbd->disk, true);  in nbd_parse_flags()
1387 		set_disk_ro(nbd->disk, false);  in nbd_parse_flags()
1390 static void send_disconnects(struct nbd_device *nbd)  in send_disconnects()  argument
1392 	struct nbd_config *config = nbd->config;  in send_disconnects()
1406 		ret = sock_xmit(nbd, i, 1, &from, 0, NULL);  in send_disconnects()
1408 			dev_err(disk_to_dev(nbd->disk),  in send_disconnects()
1414 static int nbd_disconnect(struct nbd_device *nbd)  in nbd_disconnect()  argument
1416 	struct nbd_config *config = nbd->config;  in nbd_disconnect()
1418 	dev_info(disk_to_dev(nbd->disk), "NBD_DISCONNECT\n");  in nbd_disconnect()
1420 	set_bit(NBD_DISCONNECT_REQUESTED, &nbd->flags);  in nbd_disconnect()
1421 	send_disconnects(nbd);  in nbd_disconnect()
1425 static void nbd_clear_sock(struct nbd_device *nbd)  in nbd_clear_sock()  argument
1427 	sock_shutdown(nbd);  in nbd_clear_sock()
1428 	nbd_clear_que(nbd);  in nbd_clear_sock()
1429 	nbd->task_setup = NULL;  in nbd_clear_sock()
1432 static void nbd_config_put(struct nbd_device *nbd)  in nbd_config_put()  argument
1434 	if (refcount_dec_and_mutex_lock(&nbd->config_refs,  in nbd_config_put()
1435 					&nbd->config_lock)) {  in nbd_config_put()
1436 		struct nbd_config *config = nbd->config;  in nbd_config_put()
1437 		nbd_dev_dbg_close(nbd);  in nbd_config_put()
1438 		invalidate_disk(nbd->disk);  in nbd_config_put()
1439 		if (nbd->config->bytesize)  in nbd_config_put()
1440 			kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE);  in nbd_config_put()
1443 			device_remove_file(disk_to_dev(nbd->disk), &pid_attr);  in nbd_config_put()
1444 		nbd->pid = 0;  in nbd_config_put()
1447 			device_remove_file(disk_to_dev(nbd->disk), &backend_attr);  in nbd_config_put()
1448 			kfree(nbd->backend);  in nbd_config_put()
1449 			nbd->backend = NULL;  in nbd_config_put()
1451 		nbd_clear_sock(nbd);  in nbd_config_put()
1460 		kfree(nbd->config);  in nbd_config_put()
1461 		nbd->config = NULL;  in nbd_config_put()
1463 		nbd->tag_set.timeout = 0;  in nbd_config_put()
1465 		mutex_unlock(&nbd->config_lock);  in nbd_config_put()
1466 		nbd_put(nbd);  in nbd_config_put()
1471 static int nbd_start_device(struct nbd_device *nbd)  in nbd_start_device()  argument
1473 	struct nbd_config *config = nbd->config;  in nbd_start_device()
1477 	if (nbd->pid)  in nbd_start_device()
1483 		dev_err(disk_to_dev(nbd->disk), "server does not support multiple connections per device.\n");  in nbd_start_device()
1488 	mutex_unlock(&nbd->config_lock);  in nbd_start_device()
1489 	blk_mq_update_nr_hw_queues(&nbd->tag_set, num_connections);  in nbd_start_device()
1490 	mutex_lock(&nbd->config_lock);  in nbd_start_device()
1498 	nbd->pid = task_pid_nr(current);  in nbd_start_device()
1500 	nbd_parse_flags(nbd);  in nbd_start_device()
1502 	error = device_create_file(disk_to_dev(nbd->disk), &pid_attr);  in nbd_start_device()
1504 		dev_err(disk_to_dev(nbd->disk), "device_create_file failed for pid!\n");  in nbd_start_device()
1509 	nbd_dev_dbg_init(nbd);  in nbd_start_device()
1515 			sock_shutdown(nbd);  in nbd_start_device()
1525 				flush_workqueue(nbd->recv_workq);  in nbd_start_device()
1529 		if (nbd->tag_set.timeout)  in nbd_start_device()
1531 				nbd->tag_set.timeout;  in nbd_start_device()
1533 		refcount_inc(&nbd->config_refs);  in nbd_start_device()
1535 		args->nbd = nbd;  in nbd_start_device()
1538 		queue_work(nbd->recv_workq, &args->work);  in nbd_start_device()
1540 	return nbd_set_size(nbd, config->bytesize, nbd_blksize(config));  in nbd_start_device()
1543 static int nbd_start_device_ioctl(struct nbd_device *nbd)  in nbd_start_device_ioctl()  argument
1545 	struct nbd_config *config = nbd->config;  in nbd_start_device_ioctl()
1548 	ret = nbd_start_device(nbd);  in nbd_start_device_ioctl()
1553 		set_bit(GD_NEED_PART_SCAN, &nbd->disk->state);  in nbd_start_device_ioctl()
1554 	mutex_unlock(&nbd->config_lock);  in nbd_start_device_ioctl()
1558 		sock_shutdown(nbd);  in nbd_start_device_ioctl()
1559 		nbd_clear_que(nbd);  in nbd_start_device_ioctl()
1562 	flush_workqueue(nbd->recv_workq);  in nbd_start_device_ioctl()
1563 	mutex_lock(&nbd->config_lock);  in nbd_start_device_ioctl()
1564 	nbd_bdev_reset(nbd);  in nbd_start_device_ioctl()
1573 static void nbd_clear_sock_ioctl(struct nbd_device *nbd)  in nbd_clear_sock_ioctl()  argument
1575 	nbd_clear_sock(nbd);  in nbd_clear_sock_ioctl()
1576 	disk_force_media_change(nbd->disk);  in nbd_clear_sock_ioctl()
1577 	nbd_bdev_reset(nbd);  in nbd_clear_sock_ioctl()
1579 			       &nbd->config->runtime_flags))  in nbd_clear_sock_ioctl()
1580 		nbd_config_put(nbd);  in nbd_clear_sock_ioctl()
1583 static void nbd_set_cmd_timeout(struct nbd_device *nbd, u64 timeout)  in nbd_set_cmd_timeout()  argument
1585 	nbd->tag_set.timeout = timeout * HZ;  in nbd_set_cmd_timeout()
1587 		blk_queue_rq_timeout(nbd->disk->queue, timeout * HZ);  in nbd_set_cmd_timeout()
1589 		blk_queue_rq_timeout(nbd->disk->queue, 30 * HZ);  in nbd_set_cmd_timeout()
1593 static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,  in __nbd_ioctl()  argument
1596 	struct nbd_config *config = nbd->config;  in __nbd_ioctl()
1601 		return nbd_disconnect(nbd);  in __nbd_ioctl()
1603 		nbd_clear_sock_ioctl(nbd);  in __nbd_ioctl()
1606 		return nbd_add_socket(nbd, arg, false);  in __nbd_ioctl()
1608 		return nbd_set_size(nbd, config->bytesize, arg);  in __nbd_ioctl()
1610 		return nbd_set_size(nbd, arg, nbd_blksize(config));  in __nbd_ioctl()
1614 		return nbd_set_size(nbd, bytesize, nbd_blksize(config));  in __nbd_ioctl()
1616 		nbd_set_cmd_timeout(nbd, arg);  in __nbd_ioctl()
1623 		return nbd_start_device_ioctl(nbd);  in __nbd_ioctl()
1643 	struct nbd_device *nbd = bdev->bd_disk->private_data;  in nbd_ioctl()  local
1644 	struct nbd_config *config = nbd->config;  in nbd_ioctl()
1656 	mutex_lock(&nbd->config_lock);  in nbd_ioctl()
1663 		error = __nbd_ioctl(bdev, nbd, cmd, arg);  in nbd_ioctl()
1665 		dev_err(nbd_to_dev(nbd), "Cannot use ioctl interface on a netlink controlled device.\n");  in nbd_ioctl()
1666 	mutex_unlock(&nbd->config_lock);  in nbd_ioctl()
1670 static int nbd_alloc_and_init_config(struct nbd_device *nbd)  in nbd_alloc_and_init_config()  argument
1674 	if (WARN_ON(nbd->config))  in nbd_alloc_and_init_config()
1692 	nbd->config = config;  in nbd_alloc_and_init_config()
1700 	refcount_set(&nbd->config_refs, 1);  in nbd_alloc_and_init_config()
1707 	struct nbd_device *nbd;  in nbd_open()  local
1712 	nbd = disk->private_data;  in nbd_open()
1713 	if (!nbd) {  in nbd_open()
1717 	if (!refcount_inc_not_zero(&nbd->refs)) {  in nbd_open()
1722 	config = nbd_get_config_unlocked(nbd);  in nbd_open()
1724 		mutex_lock(&nbd->config_lock);  in nbd_open()
1725 		if (refcount_inc_not_zero(&nbd->config_refs)) {  in nbd_open()
1726 			mutex_unlock(&nbd->config_lock);  in nbd_open()
1729 		ret = nbd_alloc_and_init_config(nbd);  in nbd_open()
1731 			mutex_unlock(&nbd->config_lock);  in nbd_open()
1735 		refcount_inc(&nbd->refs);  in nbd_open()
1736 		mutex_unlock(&nbd->config_lock);  in nbd_open()
1750 	struct nbd_device *nbd = disk->private_data;  in nbd_release()  local
1752 	if (test_bit(NBD_RT_DISCONNECT_ON_CLOSE, &nbd->config->runtime_flags) &&  in nbd_release()
1754 		nbd_disconnect_and_put(nbd);  in nbd_release()
1756 	nbd_config_put(nbd);  in nbd_release()
1757 	nbd_put(nbd);  in nbd_release()
1762 	struct nbd_device *nbd = disk->private_data;  in nbd_free_disk()  local
1764 	kfree(nbd);  in nbd_free_disk()
1781 	struct nbd_device *nbd = s->private;  in nbd_dbg_tasks_show()  local
1783 	if (nbd->pid)  in nbd_dbg_tasks_show()
1784 		seq_printf(s, "recv: %d\n", nbd->pid);  in nbd_dbg_tasks_show()
1793 	struct nbd_device *nbd = s->private;  in nbd_dbg_flags_show()  local
1794 	u32 flags = nbd->config->flags;  in nbd_dbg_flags_show()
1820 static int nbd_dev_dbg_init(struct nbd_device *nbd)  in nbd_dev_dbg_init()  argument
1823 	struct nbd_config *config = nbd->config;  in nbd_dev_dbg_init()
1828 	dir = debugfs_create_dir(nbd_name(nbd), nbd_dbg_dir);  in nbd_dev_dbg_init()
1830 		dev_err(nbd_to_dev(nbd), "Failed to create debugfs dir for '%s'\n",  in nbd_dev_dbg_init()
1831 			nbd_name(nbd));  in nbd_dev_dbg_init()
1836 	debugfs_create_file("tasks", 0444, dir, nbd, &nbd_dbg_tasks_fops);  in nbd_dev_dbg_init()
1838 	debugfs_create_u32("timeout", 0444, dir, &nbd->tag_set.timeout);  in nbd_dev_dbg_init()
1840 	debugfs_create_file("flags", 0444, dir, nbd, &nbd_dbg_flags_fops);  in nbd_dev_dbg_init()
1845 static void nbd_dev_dbg_close(struct nbd_device *nbd)  in nbd_dev_dbg_close()  argument
1847 	debugfs_remove_recursive(nbd->config->dbg_dir);  in nbd_dev_dbg_close()
1870 static int nbd_dev_dbg_init(struct nbd_device *nbd)  in nbd_dev_dbg_init()  argument
1875 static void nbd_dev_dbg_close(struct nbd_device *nbd)  in nbd_dev_dbg_close()  argument
1894 	cmd->nbd = set->driver_data;  in nbd_init_request()
1915 	struct nbd_device *nbd;  in nbd_dev_add()  local
1919 	nbd = kzalloc_obj(struct nbd_device);  in nbd_dev_add()
1920 	if (!nbd)  in nbd_dev_add()
1923 	nbd->tag_set.ops = &nbd_mq_ops;  in nbd_dev_add()
1924 	nbd->tag_set.nr_hw_queues = 1;  in nbd_dev_add()
1925 	nbd->tag_set.queue_depth = 128;  in nbd_dev_add()
1926 	nbd->tag_set.numa_node = NUMA_NO_NODE;  in nbd_dev_add()
1927 	nbd->tag_set.cmd_size = sizeof(struct nbd_cmd);  in nbd_dev_add()
1928 	nbd->tag_set.flags = BLK_MQ_F_BLOCKING;  in nbd_dev_add()
1929 	nbd->tag_set.driver_data = nbd;  in nbd_dev_add()
1930 	INIT_WORK(&nbd->remove_work, nbd_dev_remove_work);  in nbd_dev_add()
1931 	nbd->backend = NULL;  in nbd_dev_add()
1933 	err = blk_mq_alloc_tag_set(&nbd->tag_set);  in nbd_dev_add()
1939 		err = idr_alloc(&nbd_index_idr, nbd, index, index + 1,  in nbd_dev_add()
1944 		err = idr_alloc(&nbd_index_idr, nbd, 0,  in nbd_dev_add()
1949 	nbd->index = index;  in nbd_dev_add()
1954 	disk = blk_mq_alloc_disk(&nbd->tag_set, &lim, NULL);  in nbd_dev_add()
1959 	nbd->disk = disk;  in nbd_dev_add()
1961 	nbd->recv_workq = alloc_workqueue("nbd%d-recv",  in nbd_dev_add()
1963 					  WQ_UNBOUND, 0, nbd->index);  in nbd_dev_add()
1964 	if (!nbd->recv_workq) {  in nbd_dev_add()
1965 		dev_err(disk_to_dev(nbd->disk), "Could not allocate knbd recv work queue.\n");  in nbd_dev_add()
1970 	mutex_init(&nbd->config_lock);  in nbd_dev_add()
1971 	refcount_set(&nbd->config_refs, 0);  in nbd_dev_add()
1976 	refcount_set(&nbd->refs, 0);  in nbd_dev_add()
1977 	INIT_LIST_HEAD(&nbd->list);  in nbd_dev_add()
1982 	disk->private_data = nbd;  in nbd_dev_add()
1991 	refcount_set(&nbd->refs, refs);  in nbd_dev_add()
1993 	return nbd;  in nbd_dev_add()
1996 	destroy_workqueue(nbd->recv_workq);  in nbd_dev_add()
2004 	blk_mq_free_tag_set(&nbd->tag_set);  in nbd_dev_add()
2006 	kfree(nbd);  in nbd_dev_add()
2013 	struct nbd_device *nbd;  in nbd_find_get_unused()  local
2018 	idr_for_each_entry(&nbd_index_idr, nbd, id) {  in nbd_find_get_unused()
2019 		if (refcount_read(&nbd->config_refs) ||  in nbd_find_get_unused()
2020 		    test_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags))  in nbd_find_get_unused()
2022 		if (refcount_inc_not_zero(&nbd->refs))  in nbd_find_get_unused()
2023 			return nbd;  in nbd_find_get_unused()
2056 static int nbd_genl_size_set(struct genl_info *info, struct nbd_device *nbd)  in nbd_genl_size_set()  argument
2058 	struct nbd_config *config = nbd->config;  in nbd_genl_size_set()
2069 		return nbd_set_size(nbd, bytes, bsize);  in nbd_genl_size_set()
2075 	struct nbd_device *nbd;  in nbd_genl_connect()  local
2108 		nbd = nbd_find_get_unused();  in nbd_genl_connect()
2110 		nbd = idr_find(&nbd_index_idr, index);  in nbd_genl_connect()
2111 		if (nbd) {  in nbd_genl_connect()
2112 			if ((test_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags) &&  in nbd_genl_connect()
2113 			     test_bit(NBD_DISCONNECT_REQUESTED, &nbd->flags)) ||  in nbd_genl_connect()
2114 			    !refcount_inc_not_zero(&nbd->refs)) {  in nbd_genl_connect()
2124 	if (!nbd) {  in nbd_genl_connect()
2125 		nbd = nbd_dev_add(index, 2);  in nbd_genl_connect()
2126 		if (IS_ERR(nbd)) {  in nbd_genl_connect()
2128 			return PTR_ERR(nbd);  in nbd_genl_connect()
2132 	mutex_lock(&nbd->config_lock);  in nbd_genl_connect()
2133 	if (refcount_read(&nbd->config_refs)) {  in nbd_genl_connect()
2134 		mutex_unlock(&nbd->config_lock);  in nbd_genl_connect()
2135 		nbd_put(nbd);  in nbd_genl_connect()
2142 	ret = nbd_alloc_and_init_config(nbd);  in nbd_genl_connect()
2144 		mutex_unlock(&nbd->config_lock);  in nbd_genl_connect()
2145 		nbd_put(nbd);  in nbd_genl_connect()
2150 	config = nbd->config;  in nbd_genl_connect()
2152 	ret = nbd_genl_size_set(info, nbd);  in nbd_genl_connect()
2157 		nbd_set_cmd_timeout(nbd,  in nbd_genl_connect()
2179 					      &nbd->flags))  in nbd_genl_connect()
2183 					       &nbd->flags))  in nbd_genl_connect()
2184 				refcount_inc(&nbd->refs);  in nbd_genl_connect()
2217 			ret = nbd_add_socket(nbd, fd, true);  in nbd_genl_connect()
2224 		nbd->backend = nla_strdup(info->attrs[NBD_ATTR_BACKEND_IDENTIFIER],  in nbd_genl_connect()
2226 		if (!nbd->backend) {  in nbd_genl_connect()
2231 	ret = device_create_file(disk_to_dev(nbd->disk), &backend_attr);  in nbd_genl_connect()
2233 		dev_err(disk_to_dev(nbd->disk),  in nbd_genl_connect()
2239 	ret = nbd_start_device(nbd);  in nbd_genl_connect()
2243 		refcount_inc(&nbd->config_refs);  in nbd_genl_connect()
2244 		nbd_connect_reply(info, nbd->index);  in nbd_genl_connect()
2246 	mutex_unlock(&nbd->config_lock);  in nbd_genl_connect()
2248 	nbd_config_put(nbd);  in nbd_genl_connect()
2250 		nbd_put(nbd);  in nbd_genl_connect()
2254 static void nbd_disconnect_and_put(struct nbd_device *nbd)  in nbd_disconnect_and_put()  argument
2256 	mutex_lock(&nbd->config_lock);  in nbd_disconnect_and_put()
2257 	nbd_disconnect(nbd);  in nbd_disconnect_and_put()
2258 	sock_shutdown(nbd);  in nbd_disconnect_and_put()
2259 	wake_up(&nbd->config->conn_wait);  in nbd_disconnect_and_put()
2264 	flush_workqueue(nbd->recv_workq);  in nbd_disconnect_and_put()
2265 	nbd_clear_que(nbd);  in nbd_disconnect_and_put()
2266 	nbd->task_setup = NULL;  in nbd_disconnect_and_put()
2267 	clear_bit(NBD_RT_BOUND, &nbd->config->runtime_flags);  in nbd_disconnect_and_put()
2268 	mutex_unlock(&nbd->config_lock);  in nbd_disconnect_and_put()
2271 			       &nbd->config->runtime_flags))  in nbd_disconnect_and_put()
2272 		nbd_config_put(nbd);  in nbd_disconnect_and_put()
2277 	struct nbd_device *nbd;  in nbd_genl_disconnect()  local
2289 	nbd = idr_find(&nbd_index_idr, index);  in nbd_genl_disconnect()
2290 	if (!nbd) {  in nbd_genl_disconnect()
2295 	if (!refcount_inc_not_zero(&nbd->refs)) {  in nbd_genl_disconnect()
2301 	if (!refcount_inc_not_zero(&nbd->config_refs))  in nbd_genl_disconnect()
2303 	nbd_disconnect_and_put(nbd);  in nbd_genl_disconnect()
2304 	nbd_config_put(nbd);  in nbd_genl_disconnect()
2306 	nbd_put(nbd);  in nbd_genl_disconnect()
2312 	struct nbd_device *nbd = NULL;  in nbd_genl_reconfigure()  local
2327 	nbd = idr_find(&nbd_index_idr, index);  in nbd_genl_reconfigure()
2328 	if (!nbd) {  in nbd_genl_reconfigure()
2333 	if (nbd->backend) {  in nbd_genl_reconfigure()
2336 				       nbd->backend)) {  in nbd_genl_reconfigure()
2338 				dev_err(nbd_to_dev(nbd),  in nbd_genl_reconfigure()
2340 					nbd->backend);  in nbd_genl_reconfigure()
2345 			dev_err(nbd_to_dev(nbd), "must specify backend\n");  in nbd_genl_reconfigure()
2349 	if (!refcount_inc_not_zero(&nbd->refs)) {  in nbd_genl_reconfigure()
2356 	config = nbd_get_config_unlocked(nbd);  in nbd_genl_reconfigure()
2358 		dev_err(nbd_to_dev(nbd),  in nbd_genl_reconfigure()
2360 		nbd_put(nbd);  in nbd_genl_reconfigure()
2364 	mutex_lock(&nbd->config_lock);  in nbd_genl_reconfigure()
2366 	    !nbd->pid) {  in nbd_genl_reconfigure()
2367 		dev_err(nbd_to_dev(nbd),  in nbd_genl_reconfigure()
2373 	ret = nbd_genl_size_set(info, nbd);  in nbd_genl_reconfigure()
2378 		nbd_set_cmd_timeout(nbd,  in nbd_genl_reconfigure()
2389 					      &nbd->flags))  in nbd_genl_reconfigure()
2393 					       &nbd->flags))  in nbd_genl_reconfigure()
2394 				refcount_inc(&nbd->refs);  in nbd_genl_reconfigure()
2431 			ret = nbd_reconnect_socket(nbd, fd);  in nbd_genl_reconfigure()
2437 			dev_info(nbd_to_dev(nbd), "reconnected socket\n");  in nbd_genl_reconfigure()
2441 	mutex_unlock(&nbd->config_lock);  in nbd_genl_reconfigure()
2442 	nbd_config_put(nbd);  in nbd_genl_reconfigure()
2443 	nbd_put(nbd);  in nbd_genl_reconfigure()
2445 		nbd_put(nbd);  in nbd_genl_reconfigure()
2492 static int populate_nbd_status(struct nbd_device *nbd, struct sk_buff *reply)  in populate_nbd_status()  argument
2505 	if (refcount_read(&nbd->config_refs))  in populate_nbd_status()
2510 	ret = nla_put_u32(reply, NBD_DEVICE_INDEX, nbd->index);  in populate_nbd_status()
2523 	struct nbd_device *nbd = ptr;  in status_cb()  local
2524 	return populate_nbd_status(nbd, (struct sk_buff *)data);  in status_cb()
2569 		struct nbd_device *nbd;  in nbd_genl_status()  local
2570 		nbd = idr_find(&nbd_index_idr, index);  in nbd_genl_status()
2571 		if (nbd) {  in nbd_genl_status()
2572 			ret = populate_nbd_status(nbd, reply);  in nbd_genl_status()
2699 	struct nbd_device *nbd = ptr;  in nbd_exit_cb()  local
2702 	if (refcount_read(&nbd->refs))  in nbd_exit_cb()
2703 		list_add_tail(&nbd->list, list);  in nbd_exit_cb()
2710 	struct nbd_device *nbd;  in nbd_cleanup()  local
2726 		nbd = list_first_entry(&del_list, struct nbd_device, list);  in nbd_cleanup()
2727 		list_del_init(&nbd->list);  in nbd_cleanup()
2728 		if (refcount_read(&nbd->config_refs))  in nbd_cleanup()
2730 					refcount_read(&nbd->config_refs));  in nbd_cleanup()
2731 		if (refcount_read(&nbd->refs) != 1)  in nbd_cleanup()
2733 		nbd_put(nbd);  in nbd_cleanup()