85 	struct nvme_ns *ns = req->q->queuedata;  in nvme_failover_req()  local
90 	nvme_mpath_clear_current_path(ns);  in nvme_failover_req()
97 	if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) {  in nvme_failover_req()
98 		set_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_failover_req()
99 		queue_work(nvme_wq, &ns->ctrl->ana_work);  in nvme_failover_req()
102 	spin_lock_irqsave(&ns->head->requeue_lock, flags);  in nvme_failover_req()
104 		bio_set_dev(bio, ns->head->disk->part0);  in nvme_failover_req()
118 	blk_steal_bios(&ns->head->requeue_list, req);  in nvme_failover_req()
119 	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);  in nvme_failover_req()
122 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_failover_req()
127 	struct nvme_ns *ns = rq->q->queuedata;  in nvme_mpath_start_request()  local
128 	struct gendisk *disk = ns->head->disk;  in nvme_mpath_start_request()
141 	struct nvme_ns *ns = rq->q->queuedata;  in nvme_mpath_end_request()  local
145 	bdev_end_io_acct(ns->head->disk->part0, req_op(rq),  in nvme_mpath_end_request()
152 	struct nvme_ns *ns;  in nvme_kick_requeue_lists()  local
155 	list_for_each_entry(ns, &ctrl->namespaces, list) {  in nvme_kick_requeue_lists()
156 		if (!ns->head->disk)  in nvme_kick_requeue_lists()
158 		kblockd_schedule_work(&ns->head->requeue_work);  in nvme_kick_requeue_lists()
159 		if (nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE)  in nvme_kick_requeue_lists()
160 			disk_uevent(ns->head->disk, KOBJ_CHANGE);  in nvme_kick_requeue_lists()
174 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)  in nvme_mpath_clear_current_path()  argument
176 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_clear_current_path()
184 		if (ns == rcu_access_pointer(head->current_path[node])) {  in nvme_mpath_clear_current_path()
195 	struct nvme_ns *ns;  in nvme_mpath_clear_ctrl_paths()  local
198 	list_for_each_entry(ns, &ctrl->namespaces, list) {  in nvme_mpath_clear_ctrl_paths()
199 		nvme_mpath_clear_current_path(ns);  in nvme_mpath_clear_ctrl_paths()
200 		kblockd_schedule_work(&ns->head->requeue_work);  in nvme_mpath_clear_ctrl_paths()
205 void nvme_mpath_revalidate_paths(struct nvme_ns *ns)  in nvme_mpath_revalidate_paths()  argument
207 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_revalidate_paths()
213 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in nvme_mpath_revalidate_paths()
214 		if (capacity != get_capacity(ns->disk))  in nvme_mpath_revalidate_paths()
215 			clear_bit(NVME_NS_READY, &ns->flags);  in nvme_mpath_revalidate_paths()
224 static bool nvme_path_is_disabled(struct nvme_ns *ns)  in nvme_path_is_disabled()  argument
226 	enum nvme_ctrl_state state = nvme_ctrl_state(ns->ctrl);  in nvme_path_is_disabled()
235 	if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||  in nvme_path_is_disabled()
236 	    !test_bit(NVME_NS_READY, &ns->flags))  in nvme_path_is_disabled()
244 	struct nvme_ns *found = NULL, *fallback = NULL, *ns;  in __nvme_find_path()  local
246 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in __nvme_find_path()
247 		if (nvme_path_is_disabled(ns))  in __nvme_find_path()
251 			distance = node_distance(node, ns->ctrl->numa_node);  in __nvme_find_path()
255 		switch (ns->ana_state) {  in __nvme_find_path()
259 				found = ns;  in __nvme_find_path()
265 				fallback = ns;  in __nvme_find_path()
281 		struct nvme_ns *ns)  in nvme_next_ns()  argument
283 	ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,  in nvme_next_ns()
285 	if (ns)  in nvme_next_ns()
286 		return ns;  in nvme_next_ns()
293 	struct nvme_ns *ns, *found = NULL;  in nvme_round_robin_path()  local
301 	for (ns = nvme_next_ns(head, old);  in nvme_round_robin_path()
302 	     ns && ns != old;  in nvme_round_robin_path()
303 	     ns = nvme_next_ns(head, ns)) {  in nvme_round_robin_path()
304 		if (nvme_path_is_disabled(ns))  in nvme_round_robin_path()
307 		if (ns->ana_state == NVME_ANA_OPTIMIZED) {  in nvme_round_robin_path()
308 			found = ns;  in nvme_round_robin_path()
311 		if (ns->ana_state == NVME_ANA_NONOPTIMIZED)  in nvme_round_robin_path()
312 			found = ns;  in nvme_round_robin_path()
333 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)  in nvme_path_is_optimized()  argument
335 	return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE &&  in nvme_path_is_optimized()
336 		ns->ana_state == NVME_ANA_OPTIMIZED;  in nvme_path_is_optimized()
342 	struct nvme_ns *ns;  in nvme_find_path()  local
344 	ns = srcu_dereference(head->current_path[node], &head->srcu);  in nvme_find_path()
345 	if (unlikely(!ns))  in nvme_find_path()
349 		return nvme_round_robin_path(head, node, ns);  in nvme_find_path()
350 	if (unlikely(!nvme_path_is_optimized(ns)))  in nvme_find_path()
352 	return ns;  in nvme_find_path()
357 	struct nvme_ns *ns;  in nvme_available_path()  local
359 	list_for_each_entry_rcu(ns, &head->list, siblings) {  in nvme_available_path()
360 		if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags))  in nvme_available_path()
362 		switch (nvme_ctrl_state(ns->ctrl)) {  in nvme_available_path()
379 	struct nvme_ns *ns;  in nvme_ns_head_submit_bio()  local
392 	ns = nvme_find_path(head);  in nvme_ns_head_submit_bio()
393 	if (likely(ns)) {  in nvme_ns_head_submit_bio()
394 		bio_set_dev(bio, ns->disk->part0);  in nvme_ns_head_submit_bio()
396 		trace_block_bio_remap(bio, disk_devt(ns->head->disk),  in nvme_ns_head_submit_bio()
431 	struct nvme_ns *ns;  in nvme_ns_head_report_zones()  local
435 	ns = nvme_find_path(head);  in nvme_ns_head_report_zones()
436 	if (ns)  in nvme_ns_head_report_zones()
437 		ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data);  in nvme_ns_head_report_zones()
568 static void nvme_mpath_set_live(struct nvme_ns *ns)  in nvme_mpath_set_live()  argument
570 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_set_live()
585 			clear_bit(NVME_NSHEAD_DISK_LIVE, &ns->flags);  in nvme_mpath_set_live()
592 	if (nvme_path_is_optimized(ns)) {  in nvme_mpath_set_live()
656 		struct nvme_ns *ns)  in nvme_update_ns_ana_state()  argument
658 	ns->ana_grpid = le32_to_cpu(desc->grpid);  in nvme_update_ns_ana_state()
659 	ns->ana_state = desc->state;  in nvme_update_ns_ana_state()
660 	clear_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_update_ns_ana_state()
670 	if (nvme_state_is_live(ns->ana_state) &&  in nvme_update_ns_ana_state()
671 	    nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE)  in nvme_update_ns_ana_state()
672 		nvme_mpath_set_live(ns);  in nvme_update_ns_ana_state()
680 	struct nvme_ns *ns;  in nvme_update_ana_state()  local
693 	list_for_each_entry(ns, &ctrl->namespaces, list) {  in nvme_update_ana_state()
697 		if (ns->head->ns_id < nsid)  in nvme_update_ana_state()
699 		if (ns->head->ns_id == nsid)  in nvme_update_ana_state()
700 			nvme_update_ns_ana_state(desc, ns);  in nvme_update_ana_state()
703 		if (ns->head->ns_id > nsid)  in nvme_update_ana_state()
829 	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);  in ana_state_show()  local
831 	return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);  in ana_state_show()
847 void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid)  in nvme_mpath_add_disk()  argument
849 	if (nvme_ctrl_use_ana(ns->ctrl)) {  in nvme_mpath_add_disk()
855 		mutex_lock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
856 		ns->ana_grpid = le32_to_cpu(anagrpid);  in nvme_mpath_add_disk()
857 		nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc);  in nvme_mpath_add_disk()
858 		mutex_unlock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
861 			nvme_update_ns_ana_state(&desc, ns);  in nvme_mpath_add_disk()
864 			set_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_mpath_add_disk()
865 			queue_work(nvme_wq, &ns->ctrl->ana_work);  in nvme_mpath_add_disk()
868 		ns->ana_state = NVME_ANA_OPTIMIZED;  in nvme_mpath_add_disk()
869 		nvme_mpath_set_live(ns);  in nvme_mpath_add_disk()
872 	if (blk_queue_stable_writes(ns->queue) && ns->head->disk)  in nvme_mpath_add_disk()
874 				   ns->head->disk->queue);  in nvme_mpath_add_disk()
876 	if (blk_queue_is_zoned(ns->queue) && ns->head->disk)  in nvme_mpath_add_disk()
877 		ns->head->disk->nr_zones = ns->disk->nr_zones;  in nvme_mpath_add_disk()