xref: /linux/drivers/nvme/host/fc.c (revision ee59e3820ca92a9f4307ae23dfc7229dc8b8d400)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2016 Avago Technologies.  All rights reserved.
4  */
5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6 #include <linux/module.h>
7 #include <linux/parser.h>
8 #include <uapi/scsi/fc/fc_fs.h>
9 #include <uapi/scsi/fc/fc_els.h>
10 #include <linux/delay.h>
11 #include <linux/overflow.h>
12 #include <linux/blk-cgroup.h>
13 #include "nvme.h"
14 #include "fabrics.h"
15 #include <linux/nvme-fc-driver.h>
16 #include <linux/nvme-fc.h>
17 #include "fc.h"
18 #include <scsi/scsi_transport_fc.h>
19 
20 /* *************************** Data Structures/Defines ****************** */
21 
22 
23 enum nvme_fc_queue_flags {
24 	NVME_FC_Q_CONNECTED = 0,
25 	NVME_FC_Q_LIVE,
26 };
27 
28 #define NVME_FC_DEFAULT_DEV_LOSS_TMO	60	/* seconds */
29 #define NVME_FC_DEFAULT_RECONNECT_TMO	2	/* delay between reconnects
30 						 * when connected and a
31 						 * connection failure.
32 						 */
33 
34 struct nvme_fc_queue {
35 	struct nvme_fc_ctrl	*ctrl;
36 	struct device		*dev;
37 	struct blk_mq_hw_ctx	*hctx;
38 	void			*lldd_handle;
39 	size_t			cmnd_capsule_len;
40 	u32			qnum;
41 	u32			rqcnt;
42 	u32			seqno;
43 
44 	u64			connection_id;
45 	atomic_t		csn;
46 
47 	unsigned long		flags;
48 } __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
49 
50 enum nvme_fcop_flags {
51 	FCOP_FLAGS_TERMIO	= (1 << 0),
52 	FCOP_FLAGS_AEN		= (1 << 1),
53 };
54 
55 struct nvmefc_ls_req_op {
56 	struct nvmefc_ls_req	ls_req;
57 
58 	struct nvme_fc_rport	*rport;
59 	struct nvme_fc_queue	*queue;
60 	struct request		*rq;
61 	u32			flags;
62 
63 	int			ls_error;
64 	struct completion	ls_done;
65 	struct list_head	lsreq_list;	/* rport->ls_req_list */
66 	bool			req_queued;
67 };
68 
69 struct nvmefc_ls_rcv_op {
70 	struct nvme_fc_rport		*rport;
71 	struct nvmefc_ls_rsp		*lsrsp;
72 	union nvmefc_ls_requests	*rqstbuf;
73 	union nvmefc_ls_responses	*rspbuf;
74 	u16				rqstdatalen;
75 	bool				handled;
76 	dma_addr_t			rspdma;
77 	struct list_head		lsrcv_list;	/* rport->ls_rcv_list */
78 } __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
79 
80 enum nvme_fcpop_state {
81 	FCPOP_STATE_UNINIT	= 0,
82 	FCPOP_STATE_IDLE	= 1,
83 	FCPOP_STATE_ACTIVE	= 2,
84 	FCPOP_STATE_ABORTED	= 3,
85 	FCPOP_STATE_COMPLETE	= 4,
86 };
87 
88 struct nvme_fc_fcp_op {
89 	struct nvme_request	nreq;		/*
90 						 * nvme/host/core.c
91 						 * requires this to be
92 						 * the 1st element in the
93 						 * private structure
94 						 * associated with the
95 						 * request.
96 						 */
97 	struct nvmefc_fcp_req	fcp_req;
98 
99 	struct nvme_fc_ctrl	*ctrl;
100 	struct nvme_fc_queue	*queue;
101 	struct request		*rq;
102 
103 	atomic_t		state;
104 	u32			flags;
105 	u32			rqno;
106 	u32			nents;
107 
108 	struct nvme_fc_cmd_iu	cmd_iu;
109 	struct nvme_fc_ersp_iu	rsp_iu;
110 };
111 
112 struct nvme_fcp_op_w_sgl {
113 	struct nvme_fc_fcp_op	op;
114 	struct scatterlist	sgl[NVME_INLINE_SG_CNT];
115 	uint8_t			priv[];
116 };
117 
118 struct nvme_fc_lport {
119 	struct nvme_fc_local_port	localport;
120 
121 	struct ida			endp_cnt;
122 	struct list_head		port_list;	/* nvme_fc_port_list */
123 	struct list_head		endp_list;
124 	struct device			*dev;	/* physical device for dma */
125 	struct nvme_fc_port_template	*ops;
126 	struct kref			ref;
127 	atomic_t                        act_rport_cnt;
128 } __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
129 
130 struct nvme_fc_rport {
131 	struct nvme_fc_remote_port	remoteport;
132 
133 	struct list_head		endp_list; /* for lport->endp_list */
134 	struct list_head		ctrl_list;
135 	struct list_head		ls_req_list;
136 	struct list_head		ls_rcv_list;
137 	struct list_head		disc_list;
138 	struct device			*dev;	/* physical device for dma */
139 	struct nvme_fc_lport		*lport;
140 	spinlock_t			lock;
141 	struct kref			ref;
142 	atomic_t                        act_ctrl_cnt;
143 	unsigned long			dev_loss_end;
144 	struct work_struct		lsrcv_work;
145 } __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
146 
147 /* fc_ctrl flags values - specified as bit positions */
148 #define ASSOC_ACTIVE		0
149 #define ASSOC_FAILED		1
150 #define FCCTRL_TERMIO		2
151 
152 struct nvme_fc_ctrl {
153 	spinlock_t		lock;
154 	struct nvme_fc_queue	*queues;
155 	struct device		*dev;
156 	struct nvme_fc_lport	*lport;
157 	struct nvme_fc_rport	*rport;
158 	u32			cnum;
159 
160 	bool			ioq_live;
161 	u64			association_id;
162 	struct nvmefc_ls_rcv_op	*rcv_disconn;
163 
164 	struct list_head	ctrl_list;	/* rport->ctrl_list */
165 
166 	struct blk_mq_tag_set	admin_tag_set;
167 	struct blk_mq_tag_set	tag_set;
168 
169 	struct work_struct	ioerr_work;
170 	struct delayed_work	connect_work;
171 
172 	struct kref		ref;
173 	unsigned long		flags;
174 	u32			iocnt;
175 	wait_queue_head_t	ioabort_wait;
176 
177 	struct nvme_fc_fcp_op	aen_ops[NVME_NR_AEN_COMMANDS];
178 
179 	struct nvme_ctrl	ctrl;
180 };
181 
182 static inline struct nvme_fc_ctrl *
183 to_fc_ctrl(struct nvme_ctrl *ctrl)
184 {
185 	return container_of(ctrl, struct nvme_fc_ctrl, ctrl);
186 }
187 
188 static inline struct nvme_fc_lport *
189 localport_to_lport(struct nvme_fc_local_port *portptr)
190 {
191 	return container_of(portptr, struct nvme_fc_lport, localport);
192 }
193 
194 static inline struct nvme_fc_rport *
195 remoteport_to_rport(struct nvme_fc_remote_port *portptr)
196 {
197 	return container_of(portptr, struct nvme_fc_rport, remoteport);
198 }
199 
200 static inline struct nvmefc_ls_req_op *
201 ls_req_to_lsop(struct nvmefc_ls_req *lsreq)
202 {
203 	return container_of(lsreq, struct nvmefc_ls_req_op, ls_req);
204 }
205 
206 static inline struct nvme_fc_fcp_op *
207 fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq)
208 {
209 	return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req);
210 }
211 
212 
213 
214 /* *************************** Globals **************************** */
215 
216 
217 static DEFINE_SPINLOCK(nvme_fc_lock);
218 
219 static LIST_HEAD(nvme_fc_lport_list);
220 static DEFINE_IDA(nvme_fc_local_port_cnt);
221 static DEFINE_IDA(nvme_fc_ctrl_cnt);
222 
223 /*
224  * These items are short-term. They will eventually be moved into
225  * a generic FC class. See comments in module init.
226  */
227 static struct device *fc_udev_device;
228 
229 static void nvme_fc_complete_rq(struct request *rq);
230 
231 /* *********************** FC-NVME Port Management ************************ */
232 
233 static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
234 			struct nvme_fc_queue *, unsigned int);
235 
236 static void nvme_fc_handle_ls_rqst_work(struct work_struct *work);
237 
238 
239 static void
240 nvme_fc_free_lport(struct kref *ref)
241 {
242 	struct nvme_fc_lport *lport =
243 		container_of(ref, struct nvme_fc_lport, ref);
244 	unsigned long flags;
245 
246 	WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
247 	WARN_ON(!list_empty(&lport->endp_list));
248 
249 	/* remove from transport list */
250 	spin_lock_irqsave(&nvme_fc_lock, flags);
251 	list_del(&lport->port_list);
252 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
253 
254 	ida_free(&nvme_fc_local_port_cnt, lport->localport.port_num);
255 	ida_destroy(&lport->endp_cnt);
256 
257 	put_device(lport->dev);
258 
259 	kfree(lport);
260 }
261 
262 static void
263 nvme_fc_lport_put(struct nvme_fc_lport *lport)
264 {
265 	kref_put(&lport->ref, nvme_fc_free_lport);
266 }
267 
268 static int
269 nvme_fc_lport_get(struct nvme_fc_lport *lport)
270 {
271 	return kref_get_unless_zero(&lport->ref);
272 }
273 
274 
275 static struct nvme_fc_lport *
276 nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo,
277 			struct nvme_fc_port_template *ops,
278 			struct device *dev)
279 {
280 	struct nvme_fc_lport *lport;
281 	unsigned long flags;
282 
283 	spin_lock_irqsave(&nvme_fc_lock, flags);
284 
285 	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
286 		if (lport->localport.node_name != pinfo->node_name ||
287 		    lport->localport.port_name != pinfo->port_name)
288 			continue;
289 
290 		if (lport->dev != dev) {
291 			lport = ERR_PTR(-EXDEV);
292 			goto out_done;
293 		}
294 
295 		if (lport->localport.port_state != FC_OBJSTATE_DELETED) {
296 			lport = ERR_PTR(-EEXIST);
297 			goto out_done;
298 		}
299 
300 		if (!nvme_fc_lport_get(lport)) {
301 			/*
302 			 * fails if ref cnt already 0. If so,
303 			 * act as if lport already deleted
304 			 */
305 			lport = NULL;
306 			goto out_done;
307 		}
308 
309 		/* resume the lport */
310 
311 		lport->ops = ops;
312 		lport->localport.port_role = pinfo->port_role;
313 		lport->localport.port_id = pinfo->port_id;
314 		lport->localport.port_state = FC_OBJSTATE_ONLINE;
315 
316 		spin_unlock_irqrestore(&nvme_fc_lock, flags);
317 
318 		return lport;
319 	}
320 
321 	lport = NULL;
322 
323 out_done:
324 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
325 
326 	return lport;
327 }
328 
329 /**
330  * nvme_fc_register_localport - transport entry point called by an
331  *                              LLDD to register the existence of a NVME
332  *                              host FC port.
333  * @pinfo:     pointer to information about the port to be registered
334  * @template:  LLDD entrypoints and operational parameters for the port
335  * @dev:       physical hardware device node port corresponds to. Will be
336  *             used for DMA mappings
337  * @portptr:   pointer to a local port pointer. Upon success, the routine
338  *             will allocate a nvme_fc_local_port structure and place its
339  *             address in the local port pointer. Upon failure, local port
340  *             pointer will be set to 0.
341  *
342  * Returns:
343  * a completion status. Must be 0 upon success; a negative errno
344  * (ex: -ENXIO) upon failure.
345  */
346 int
347 nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
348 			struct nvme_fc_port_template *template,
349 			struct device *dev,
350 			struct nvme_fc_local_port **portptr)
351 {
352 	struct nvme_fc_lport *newrec;
353 	unsigned long flags;
354 	int ret, idx;
355 
356 	if (!template->localport_delete || !template->remoteport_delete ||
357 	    !template->ls_req || !template->fcp_io ||
358 	    !template->ls_abort || !template->fcp_abort ||
359 	    !template->max_hw_queues || !template->max_sgl_segments ||
360 	    !template->max_dif_sgl_segments || !template->dma_boundary) {
361 		ret = -EINVAL;
362 		goto out_reghost_failed;
363 	}
364 
365 	/*
366 	 * look to see if there is already a localport that had been
367 	 * deregistered and in the process of waiting for all the
368 	 * references to fully be removed.  If the references haven't
369 	 * expired, we can simply re-enable the localport. Remoteports
370 	 * and controller reconnections should resume naturally.
371 	 */
372 	newrec = nvme_fc_attach_to_unreg_lport(pinfo, template, dev);
373 
374 	/* found an lport, but something about its state is bad */
375 	if (IS_ERR(newrec)) {
376 		ret = PTR_ERR(newrec);
377 		goto out_reghost_failed;
378 
379 	/* found existing lport, which was resumed */
380 	} else if (newrec) {
381 		*portptr = &newrec->localport;
382 		return 0;
383 	}
384 
385 	/* nothing found - allocate a new localport struct */
386 
387 	newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz),
388 			 GFP_KERNEL);
389 	if (!newrec) {
390 		ret = -ENOMEM;
391 		goto out_reghost_failed;
392 	}
393 
394 	idx = ida_alloc(&nvme_fc_local_port_cnt, GFP_KERNEL);
395 	if (idx < 0) {
396 		ret = -ENOSPC;
397 		goto out_fail_kfree;
398 	}
399 
400 	if (!get_device(dev) && dev) {
401 		ret = -ENODEV;
402 		goto out_ida_put;
403 	}
404 
405 	INIT_LIST_HEAD(&newrec->port_list);
406 	INIT_LIST_HEAD(&newrec->endp_list);
407 	kref_init(&newrec->ref);
408 	atomic_set(&newrec->act_rport_cnt, 0);
409 	newrec->ops = template;
410 	newrec->dev = dev;
411 	ida_init(&newrec->endp_cnt);
412 	if (template->local_priv_sz)
413 		newrec->localport.private = &newrec[1];
414 	else
415 		newrec->localport.private = NULL;
416 	newrec->localport.node_name = pinfo->node_name;
417 	newrec->localport.port_name = pinfo->port_name;
418 	newrec->localport.port_role = pinfo->port_role;
419 	newrec->localport.port_id = pinfo->port_id;
420 	newrec->localport.port_state = FC_OBJSTATE_ONLINE;
421 	newrec->localport.port_num = idx;
422 
423 	spin_lock_irqsave(&nvme_fc_lock, flags);
424 	list_add_tail(&newrec->port_list, &nvme_fc_lport_list);
425 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
426 
427 	if (dev)
428 		dma_set_seg_boundary(dev, template->dma_boundary);
429 
430 	*portptr = &newrec->localport;
431 	return 0;
432 
433 out_ida_put:
434 	ida_free(&nvme_fc_local_port_cnt, idx);
435 out_fail_kfree:
436 	kfree(newrec);
437 out_reghost_failed:
438 	*portptr = NULL;
439 
440 	return ret;
441 }
442 EXPORT_SYMBOL_GPL(nvme_fc_register_localport);
443 
444 /**
445  * nvme_fc_unregister_localport - transport entry point called by an
446  *                              LLDD to deregister/remove a previously
447  *                              registered a NVME host FC port.
448  * @portptr: pointer to the (registered) local port that is to be deregistered.
449  *
450  * Returns:
451  * a completion status. Must be 0 upon success; a negative errno
452  * (ex: -ENXIO) upon failure.
453  */
454 int
455 nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr)
456 {
457 	struct nvme_fc_lport *lport = localport_to_lport(portptr);
458 	unsigned long flags;
459 
460 	if (!portptr)
461 		return -EINVAL;
462 
463 	spin_lock_irqsave(&nvme_fc_lock, flags);
464 
465 	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
466 		spin_unlock_irqrestore(&nvme_fc_lock, flags);
467 		return -EINVAL;
468 	}
469 	portptr->port_state = FC_OBJSTATE_DELETED;
470 
471 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
472 
473 	if (atomic_read(&lport->act_rport_cnt) == 0)
474 		lport->ops->localport_delete(&lport->localport);
475 
476 	nvme_fc_lport_put(lport);
477 
478 	return 0;
479 }
480 EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport);
481 
482 /*
483  * TRADDR strings, per FC-NVME are fixed format:
484  *   "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters
485  * udev event will only differ by prefix of what field is
486  * being specified:
487  *    "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters
488  *  19 + 43 + null_fudge = 64 characters
489  */
490 #define FCNVME_TRADDR_LENGTH		64
491 
492 static void
493 nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport,
494 		struct nvme_fc_rport *rport)
495 {
496 	char hostaddr[FCNVME_TRADDR_LENGTH];	/* NVMEFC_HOST_TRADDR=...*/
497 	char tgtaddr[FCNVME_TRADDR_LENGTH];	/* NVMEFC_TRADDR=...*/
498 	char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL };
499 
500 	if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY))
501 		return;
502 
503 	snprintf(hostaddr, sizeof(hostaddr),
504 		"NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx",
505 		lport->localport.node_name, lport->localport.port_name);
506 	snprintf(tgtaddr, sizeof(tgtaddr),
507 		"NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx",
508 		rport->remoteport.node_name, rport->remoteport.port_name);
509 	kobject_uevent_env(&fc_udev_device->kobj, KOBJ_CHANGE, envp);
510 }
511 
512 static void
513 nvme_fc_free_rport(struct kref *ref)
514 {
515 	struct nvme_fc_rport *rport =
516 		container_of(ref, struct nvme_fc_rport, ref);
517 	struct nvme_fc_lport *lport =
518 			localport_to_lport(rport->remoteport.localport);
519 	unsigned long flags;
520 
521 	WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED);
522 	WARN_ON(!list_empty(&rport->ctrl_list));
523 
524 	/* remove from lport list */
525 	spin_lock_irqsave(&nvme_fc_lock, flags);
526 	list_del(&rport->endp_list);
527 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
528 
529 	WARN_ON(!list_empty(&rport->disc_list));
530 	ida_free(&lport->endp_cnt, rport->remoteport.port_num);
531 
532 	kfree(rport);
533 
534 	nvme_fc_lport_put(lport);
535 }
536 
537 static void
538 nvme_fc_rport_put(struct nvme_fc_rport *rport)
539 {
540 	kref_put(&rport->ref, nvme_fc_free_rport);
541 }
542 
543 static int
544 nvme_fc_rport_get(struct nvme_fc_rport *rport)
545 {
546 	return kref_get_unless_zero(&rport->ref);
547 }
548 
549 static void
550 nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
551 {
552 	switch (nvme_ctrl_state(&ctrl->ctrl)) {
553 	case NVME_CTRL_NEW:
554 	case NVME_CTRL_CONNECTING:
555 		/*
556 		 * As all reconnects were suppressed, schedule a
557 		 * connect.
558 		 */
559 		dev_info(ctrl->ctrl.device,
560 			"NVME-FC{%d}: connectivity re-established. "
561 			"Attempting reconnect\n", ctrl->cnum);
562 
563 		queue_delayed_work(nvme_wq, &ctrl->connect_work, 0);
564 		break;
565 
566 	case NVME_CTRL_RESETTING:
567 		/*
568 		 * Controller is already in the process of terminating the
569 		 * association. No need to do anything further. The reconnect
570 		 * step will naturally occur after the reset completes.
571 		 */
572 		break;
573 
574 	default:
575 		/* no action to take - let it delete */
576 		break;
577 	}
578 }
579 
580 static struct nvme_fc_rport *
581 nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport,
582 				struct nvme_fc_port_info *pinfo)
583 {
584 	struct nvme_fc_rport *rport;
585 	struct nvme_fc_ctrl *ctrl;
586 	unsigned long flags;
587 
588 	spin_lock_irqsave(&nvme_fc_lock, flags);
589 
590 	list_for_each_entry(rport, &lport->endp_list, endp_list) {
591 		if (rport->remoteport.node_name != pinfo->node_name ||
592 		    rport->remoteport.port_name != pinfo->port_name)
593 			continue;
594 
595 		if (!nvme_fc_rport_get(rport)) {
596 			rport = ERR_PTR(-ENOLCK);
597 			goto out_done;
598 		}
599 
600 		spin_unlock_irqrestore(&nvme_fc_lock, flags);
601 
602 		spin_lock_irqsave(&rport->lock, flags);
603 
604 		/* has it been unregistered */
605 		if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) {
606 			/* means lldd called us twice */
607 			spin_unlock_irqrestore(&rport->lock, flags);
608 			nvme_fc_rport_put(rport);
609 			return ERR_PTR(-ESTALE);
610 		}
611 
612 		rport->remoteport.port_role = pinfo->port_role;
613 		rport->remoteport.port_id = pinfo->port_id;
614 		rport->remoteport.port_state = FC_OBJSTATE_ONLINE;
615 		rport->dev_loss_end = 0;
616 
617 		/*
618 		 * kick off a reconnect attempt on all associations to the
619 		 * remote port. A successful reconnects will resume i/o.
620 		 */
621 		list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list)
622 			nvme_fc_resume_controller(ctrl);
623 
624 		spin_unlock_irqrestore(&rport->lock, flags);
625 
626 		return rport;
627 	}
628 
629 	rport = NULL;
630 
631 out_done:
632 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
633 
634 	return rport;
635 }
636 
637 static inline void
638 __nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport,
639 			struct nvme_fc_port_info *pinfo)
640 {
641 	if (pinfo->dev_loss_tmo)
642 		rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo;
643 	else
644 		rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO;
645 }
646 
647 /**
648  * nvme_fc_register_remoteport - transport entry point called by an
649  *                              LLDD to register the existence of a NVME
650  *                              subsystem FC port on its fabric.
651  * @localport: pointer to the (registered) local port that the remote
652  *             subsystem port is connected to.
653  * @pinfo:     pointer to information about the port to be registered
654  * @portptr:   pointer to a remote port pointer. Upon success, the routine
655  *             will allocate a nvme_fc_remote_port structure and place its
656  *             address in the remote port pointer. Upon failure, remote port
657  *             pointer will be set to 0.
658  *
659  * Returns:
660  * a completion status. Must be 0 upon success; a negative errno
661  * (ex: -ENXIO) upon failure.
662  */
663 int
664 nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
665 				struct nvme_fc_port_info *pinfo,
666 				struct nvme_fc_remote_port **portptr)
667 {
668 	struct nvme_fc_lport *lport = localport_to_lport(localport);
669 	struct nvme_fc_rport *newrec;
670 	unsigned long flags;
671 	int ret, idx;
672 
673 	if (!nvme_fc_lport_get(lport)) {
674 		ret = -ESHUTDOWN;
675 		goto out_reghost_failed;
676 	}
677 
678 	/*
679 	 * look to see if there is already a remoteport that is waiting
680 	 * for a reconnect (within dev_loss_tmo) with the same WWN's.
681 	 * If so, transition to it and reconnect.
682 	 */
683 	newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo);
684 
685 	/* found an rport, but something about its state is bad */
686 	if (IS_ERR(newrec)) {
687 		ret = PTR_ERR(newrec);
688 		goto out_lport_put;
689 
690 	/* found existing rport, which was resumed */
691 	} else if (newrec) {
692 		nvme_fc_lport_put(lport);
693 		__nvme_fc_set_dev_loss_tmo(newrec, pinfo);
694 		nvme_fc_signal_discovery_scan(lport, newrec);
695 		*portptr = &newrec->remoteport;
696 		return 0;
697 	}
698 
699 	/* nothing found - allocate a new remoteport struct */
700 
701 	newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz),
702 			 GFP_KERNEL);
703 	if (!newrec) {
704 		ret = -ENOMEM;
705 		goto out_lport_put;
706 	}
707 
708 	idx = ida_alloc(&lport->endp_cnt, GFP_KERNEL);
709 	if (idx < 0) {
710 		ret = -ENOSPC;
711 		goto out_kfree_rport;
712 	}
713 
714 	INIT_LIST_HEAD(&newrec->endp_list);
715 	INIT_LIST_HEAD(&newrec->ctrl_list);
716 	INIT_LIST_HEAD(&newrec->ls_req_list);
717 	INIT_LIST_HEAD(&newrec->disc_list);
718 	kref_init(&newrec->ref);
719 	atomic_set(&newrec->act_ctrl_cnt, 0);
720 	spin_lock_init(&newrec->lock);
721 	newrec->remoteport.localport = &lport->localport;
722 	INIT_LIST_HEAD(&newrec->ls_rcv_list);
723 	newrec->dev = lport->dev;
724 	newrec->lport = lport;
725 	if (lport->ops->remote_priv_sz)
726 		newrec->remoteport.private = &newrec[1];
727 	else
728 		newrec->remoteport.private = NULL;
729 	newrec->remoteport.port_role = pinfo->port_role;
730 	newrec->remoteport.node_name = pinfo->node_name;
731 	newrec->remoteport.port_name = pinfo->port_name;
732 	newrec->remoteport.port_id = pinfo->port_id;
733 	newrec->remoteport.port_state = FC_OBJSTATE_ONLINE;
734 	newrec->remoteport.port_num = idx;
735 	__nvme_fc_set_dev_loss_tmo(newrec, pinfo);
736 	INIT_WORK(&newrec->lsrcv_work, nvme_fc_handle_ls_rqst_work);
737 
738 	spin_lock_irqsave(&nvme_fc_lock, flags);
739 	list_add_tail(&newrec->endp_list, &lport->endp_list);
740 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
741 
742 	nvme_fc_signal_discovery_scan(lport, newrec);
743 
744 	*portptr = &newrec->remoteport;
745 	return 0;
746 
747 out_kfree_rport:
748 	kfree(newrec);
749 out_lport_put:
750 	nvme_fc_lport_put(lport);
751 out_reghost_failed:
752 	*portptr = NULL;
753 	return ret;
754 }
755 EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);
756 
757 static int
758 nvme_fc_abort_lsops(struct nvme_fc_rport *rport)
759 {
760 	struct nvmefc_ls_req_op *lsop;
761 	unsigned long flags;
762 
763 restart:
764 	spin_lock_irqsave(&rport->lock, flags);
765 
766 	list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) {
767 		if (!(lsop->flags & FCOP_FLAGS_TERMIO)) {
768 			lsop->flags |= FCOP_FLAGS_TERMIO;
769 			spin_unlock_irqrestore(&rport->lock, flags);
770 			rport->lport->ops->ls_abort(&rport->lport->localport,
771 						&rport->remoteport,
772 						&lsop->ls_req);
773 			goto restart;
774 		}
775 	}
776 	spin_unlock_irqrestore(&rport->lock, flags);
777 
778 	return 0;
779 }
780 
781 static void
782 nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
783 {
784 	enum nvme_ctrl_state state;
785 	unsigned long flags;
786 
787 	dev_info(ctrl->ctrl.device,
788 		"NVME-FC{%d}: controller connectivity lost. Awaiting "
789 		"Reconnect", ctrl->cnum);
790 
791 	spin_lock_irqsave(&ctrl->lock, flags);
792 	set_bit(ASSOC_FAILED, &ctrl->flags);
793 	state = nvme_ctrl_state(&ctrl->ctrl);
794 	spin_unlock_irqrestore(&ctrl->lock, flags);
795 
796 	switch (state) {
797 	case NVME_CTRL_NEW:
798 	case NVME_CTRL_LIVE:
799 		/*
800 		 * Schedule a controller reset. The reset will terminate the
801 		 * association and schedule the reconnect timer.  Reconnects
802 		 * will be attempted until either the ctlr_loss_tmo
803 		 * (max_retries * connect_delay) expires or the remoteport's
804 		 * dev_loss_tmo expires.
805 		 */
806 		if (nvme_reset_ctrl(&ctrl->ctrl)) {
807 			dev_warn(ctrl->ctrl.device,
808 				"NVME-FC{%d}: Couldn't schedule reset.\n",
809 				ctrl->cnum);
810 			nvme_delete_ctrl(&ctrl->ctrl);
811 		}
812 		break;
813 
814 	case NVME_CTRL_CONNECTING:
815 		/*
816 		 * The association has already been terminated and the
817 		 * controller is attempting reconnects.  No need to do anything
818 		 * futher.  Reconnects will be attempted until either the
819 		 * ctlr_loss_tmo (max_retries * connect_delay) expires or the
820 		 * remoteport's dev_loss_tmo expires.
821 		 */
822 		break;
823 
824 	case NVME_CTRL_RESETTING:
825 		/*
826 		 * Controller is already in the process of terminating the
827 		 * association.  No need to do anything further. The reconnect
828 		 * step will kick in naturally after the association is
829 		 * terminated.
830 		 */
831 		break;
832 
833 	case NVME_CTRL_DELETING:
834 	case NVME_CTRL_DELETING_NOIO:
835 	default:
836 		/* no action to take - let it delete */
837 		break;
838 	}
839 }
840 
841 /**
842  * nvme_fc_unregister_remoteport - transport entry point called by an
843  *                              LLDD to deregister/remove a previously
844  *                              registered a NVME subsystem FC port.
845  * @portptr: pointer to the (registered) remote port that is to be
846  *           deregistered.
847  *
848  * Returns:
849  * a completion status. Must be 0 upon success; a negative errno
850  * (ex: -ENXIO) upon failure.
851  */
852 int
853 nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
854 {
855 	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
856 	struct nvme_fc_ctrl *ctrl;
857 	unsigned long flags;
858 
859 	if (!portptr)
860 		return -EINVAL;
861 
862 	spin_lock_irqsave(&rport->lock, flags);
863 
864 	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
865 		spin_unlock_irqrestore(&rport->lock, flags);
866 		return -EINVAL;
867 	}
868 	portptr->port_state = FC_OBJSTATE_DELETED;
869 
870 	rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ);
871 
872 	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
873 		/* if dev_loss_tmo==0, dev loss is immediate */
874 		if (!portptr->dev_loss_tmo) {
875 			dev_warn(ctrl->ctrl.device,
876 				"NVME-FC{%d}: controller connectivity lost.\n",
877 				ctrl->cnum);
878 			nvme_delete_ctrl(&ctrl->ctrl);
879 		} else
880 			nvme_fc_ctrl_connectivity_loss(ctrl);
881 	}
882 
883 	spin_unlock_irqrestore(&rport->lock, flags);
884 
885 	nvme_fc_abort_lsops(rport);
886 
887 	if (atomic_read(&rport->act_ctrl_cnt) == 0)
888 		rport->lport->ops->remoteport_delete(portptr);
889 
890 	/*
891 	 * release the reference, which will allow, if all controllers
892 	 * go away, which should only occur after dev_loss_tmo occurs,
893 	 * for the rport to be torn down.
894 	 */
895 	nvme_fc_rport_put(rport);
896 
897 	return 0;
898 }
899 EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport);
900 
901 /**
902  * nvme_fc_rescan_remoteport - transport entry point called by an
903  *                              LLDD to request a nvme device rescan.
904  * @remoteport: pointer to the (registered) remote port that is to be
905  *              rescanned.
906  *
907  * Returns: N/A
908  */
909 void
910 nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport)
911 {
912 	struct nvme_fc_rport *rport = remoteport_to_rport(remoteport);
913 
914 	nvme_fc_signal_discovery_scan(rport->lport, rport);
915 }
916 EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport);
917 
918 int
919 nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr,
920 			u32 dev_loss_tmo)
921 {
922 	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
923 	unsigned long flags;
924 
925 	spin_lock_irqsave(&rport->lock, flags);
926 
927 	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
928 		spin_unlock_irqrestore(&rport->lock, flags);
929 		return -EINVAL;
930 	}
931 
932 	/* a dev_loss_tmo of 0 (immediate) is allowed to be set */
933 	rport->remoteport.dev_loss_tmo = dev_loss_tmo;
934 
935 	spin_unlock_irqrestore(&rport->lock, flags);
936 
937 	return 0;
938 }
939 EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss);
940 
941 
942 /* *********************** FC-NVME DMA Handling **************************** */
943 
944 /*
945  * The fcloop device passes in a NULL device pointer. Real LLD's will
946  * pass in a valid device pointer. If NULL is passed to the dma mapping
947  * routines, depending on the platform, it may or may not succeed, and
948  * may crash.
949  *
950  * As such:
951  * Wrapper all the dma routines and check the dev pointer.
952  *
953  * If simple mappings (return just a dma address, we'll noop them,
954  * returning a dma address of 0.
955  *
956  * On more complex mappings (dma_map_sg), a pseudo routine fills
957  * in the scatter list, setting all dma addresses to 0.
958  */
959 
960 static inline dma_addr_t
961 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
962 		enum dma_data_direction dir)
963 {
964 	return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
965 }
966 
967 static inline int
968 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
969 {
970 	return dev ? dma_mapping_error(dev, dma_addr) : 0;
971 }
972 
973 static inline void
974 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
975 	enum dma_data_direction dir)
976 {
977 	if (dev)
978 		dma_unmap_single(dev, addr, size, dir);
979 }
980 
981 static inline void
982 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
983 		enum dma_data_direction dir)
984 {
985 	if (dev)
986 		dma_sync_single_for_cpu(dev, addr, size, dir);
987 }
988 
989 static inline void
990 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
991 		enum dma_data_direction dir)
992 {
993 	if (dev)
994 		dma_sync_single_for_device(dev, addr, size, dir);
995 }
996 
997 /* pseudo dma_map_sg call */
998 static int
999 fc_map_sg(struct scatterlist *sg, int nents)
1000 {
1001 	struct scatterlist *s;
1002 	int i;
1003 
1004 	WARN_ON(nents == 0 || sg[0].length == 0);
1005 
1006 	for_each_sg(sg, s, nents, i) {
1007 		s->dma_address = 0L;
1008 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1009 		s->dma_length = s->length;
1010 #endif
1011 	}
1012 	return nents;
1013 }
1014 
1015 static inline int
1016 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1017 		enum dma_data_direction dir)
1018 {
1019 	return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
1020 }
1021 
1022 static inline void
1023 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1024 		enum dma_data_direction dir)
1025 {
1026 	if (dev)
1027 		dma_unmap_sg(dev, sg, nents, dir);
1028 }
1029 
1030 /* *********************** FC-NVME LS Handling **************************** */
1031 
1032 static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
1033 static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
1034 
1035 static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
1036 
1037 static void
1038 __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
1039 {
1040 	struct nvme_fc_rport *rport = lsop->rport;
1041 	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1042 	unsigned long flags;
1043 
1044 	spin_lock_irqsave(&rport->lock, flags);
1045 
1046 	if (!lsop->req_queued) {
1047 		spin_unlock_irqrestore(&rport->lock, flags);
1048 		return;
1049 	}
1050 
1051 	list_del(&lsop->lsreq_list);
1052 
1053 	lsop->req_queued = false;
1054 
1055 	spin_unlock_irqrestore(&rport->lock, flags);
1056 
1057 	fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1058 				  (lsreq->rqstlen + lsreq->rsplen),
1059 				  DMA_BIDIRECTIONAL);
1060 
1061 	nvme_fc_rport_put(rport);
1062 }
1063 
1064 static int
1065 __nvme_fc_send_ls_req(struct nvme_fc_rport *rport,
1066 		struct nvmefc_ls_req_op *lsop,
1067 		void (*done)(struct nvmefc_ls_req *req, int status))
1068 {
1069 	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1070 	unsigned long flags;
1071 	int ret = 0;
1072 
1073 	if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
1074 		return -ECONNREFUSED;
1075 
1076 	if (!nvme_fc_rport_get(rport))
1077 		return -ESHUTDOWN;
1078 
1079 	lsreq->done = done;
1080 	lsop->rport = rport;
1081 	lsop->req_queued = false;
1082 	INIT_LIST_HEAD(&lsop->lsreq_list);
1083 	init_completion(&lsop->ls_done);
1084 
1085 	lsreq->rqstdma = fc_dma_map_single(rport->dev, lsreq->rqstaddr,
1086 				  lsreq->rqstlen + lsreq->rsplen,
1087 				  DMA_BIDIRECTIONAL);
1088 	if (fc_dma_mapping_error(rport->dev, lsreq->rqstdma)) {
1089 		ret = -EFAULT;
1090 		goto out_putrport;
1091 	}
1092 	lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
1093 
1094 	spin_lock_irqsave(&rport->lock, flags);
1095 
1096 	list_add_tail(&lsop->lsreq_list, &rport->ls_req_list);
1097 
1098 	lsop->req_queued = true;
1099 
1100 	spin_unlock_irqrestore(&rport->lock, flags);
1101 
1102 	ret = rport->lport->ops->ls_req(&rport->lport->localport,
1103 					&rport->remoteport, lsreq);
1104 	if (ret)
1105 		goto out_unlink;
1106 
1107 	return 0;
1108 
1109 out_unlink:
1110 	lsop->ls_error = ret;
1111 	spin_lock_irqsave(&rport->lock, flags);
1112 	lsop->req_queued = false;
1113 	list_del(&lsop->lsreq_list);
1114 	spin_unlock_irqrestore(&rport->lock, flags);
1115 	fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1116 				  (lsreq->rqstlen + lsreq->rsplen),
1117 				  DMA_BIDIRECTIONAL);
1118 out_putrport:
1119 	nvme_fc_rport_put(rport);
1120 
1121 	return ret;
1122 }
1123 
1124 static void
1125 nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
1126 {
1127 	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1128 
1129 	lsop->ls_error = status;
1130 	complete(&lsop->ls_done);
1131 }
1132 
1133 static int
1134 nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop)
1135 {
1136 	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1137 	struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
1138 	int ret;
1139 
1140 	ret = __nvme_fc_send_ls_req(rport, lsop, nvme_fc_send_ls_req_done);
1141 
1142 	if (!ret) {
1143 		/*
1144 		 * No timeout/not interruptible as we need the struct
1145 		 * to exist until the lldd calls us back. Thus mandate
1146 		 * wait until driver calls back. lldd responsible for
1147 		 * the timeout action
1148 		 */
1149 		wait_for_completion(&lsop->ls_done);
1150 
1151 		__nvme_fc_finish_ls_req(lsop);
1152 
1153 		ret = lsop->ls_error;
1154 	}
1155 
1156 	if (ret)
1157 		return ret;
1158 
1159 	/* ACC or RJT payload ? */
1160 	if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
1161 		return -ENXIO;
1162 
1163 	return 0;
1164 }
1165 
1166 static int
1167 nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport,
1168 		struct nvmefc_ls_req_op *lsop,
1169 		void (*done)(struct nvmefc_ls_req *req, int status))
1170 {
1171 	/* don't wait for completion */
1172 
1173 	return __nvme_fc_send_ls_req(rport, lsop, done);
1174 }
1175 
1176 static int
1177 nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
1178 	struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio)
1179 {
1180 	struct nvmefc_ls_req_op *lsop;
1181 	struct nvmefc_ls_req *lsreq;
1182 	struct fcnvme_ls_cr_assoc_rqst *assoc_rqst;
1183 	struct fcnvme_ls_cr_assoc_acc *assoc_acc;
1184 	unsigned long flags;
1185 	int ret, fcret = 0;
1186 
1187 	lsop = kzalloc((sizeof(*lsop) +
1188 			 sizeof(*assoc_rqst) + sizeof(*assoc_acc) +
1189 			 ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1190 	if (!lsop) {
1191 		dev_info(ctrl->ctrl.device,
1192 			"NVME-FC{%d}: send Create Association failed: ENOMEM\n",
1193 			ctrl->cnum);
1194 		ret = -ENOMEM;
1195 		goto out_no_memory;
1196 	}
1197 
1198 	assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)&lsop[1];
1199 	assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1];
1200 	lsreq = &lsop->ls_req;
1201 	if (ctrl->lport->ops->lsrqst_priv_sz)
1202 		lsreq->private = &assoc_acc[1];
1203 	else
1204 		lsreq->private = NULL;
1205 
1206 	assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION;
1207 	assoc_rqst->desc_list_len =
1208 			cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1209 
1210 	assoc_rqst->assoc_cmd.desc_tag =
1211 			cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD);
1212 	assoc_rqst->assoc_cmd.desc_len =
1213 			fcnvme_lsdesc_len(
1214 				sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1215 
1216 	assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1217 	assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1);
1218 	/* Linux supports only Dynamic controllers */
1219 	assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
1220 	uuid_copy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id);
1221 	strscpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn,
1222 		sizeof(assoc_rqst->assoc_cmd.hostnqn));
1223 	strscpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn,
1224 		sizeof(assoc_rqst->assoc_cmd.subnqn));
1225 
1226 	lsop->queue = queue;
1227 	lsreq->rqstaddr = assoc_rqst;
1228 	lsreq->rqstlen = sizeof(*assoc_rqst);
1229 	lsreq->rspaddr = assoc_acc;
1230 	lsreq->rsplen = sizeof(*assoc_acc);
1231 	lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1232 
1233 	ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1234 	if (ret)
1235 		goto out_free_buffer;
1236 
1237 	/* process connect LS completion */
1238 
1239 	/* validate the ACC response */
1240 	if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1241 		fcret = VERR_LSACC;
1242 	else if (assoc_acc->hdr.desc_list_len !=
1243 			fcnvme_lsdesc_len(
1244 				sizeof(struct fcnvme_ls_cr_assoc_acc)))
1245 		fcret = VERR_CR_ASSOC_ACC_LEN;
1246 	else if (assoc_acc->hdr.rqst.desc_tag !=
1247 			cpu_to_be32(FCNVME_LSDESC_RQST))
1248 		fcret = VERR_LSDESC_RQST;
1249 	else if (assoc_acc->hdr.rqst.desc_len !=
1250 			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1251 		fcret = VERR_LSDESC_RQST_LEN;
1252 	else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION)
1253 		fcret = VERR_CR_ASSOC;
1254 	else if (assoc_acc->associd.desc_tag !=
1255 			cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1256 		fcret = VERR_ASSOC_ID;
1257 	else if (assoc_acc->associd.desc_len !=
1258 			fcnvme_lsdesc_len(
1259 				sizeof(struct fcnvme_lsdesc_assoc_id)))
1260 		fcret = VERR_ASSOC_ID_LEN;
1261 	else if (assoc_acc->connectid.desc_tag !=
1262 			cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1263 		fcret = VERR_CONN_ID;
1264 	else if (assoc_acc->connectid.desc_len !=
1265 			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1266 		fcret = VERR_CONN_ID_LEN;
1267 
1268 	if (fcret) {
1269 		ret = -EBADF;
1270 		dev_err(ctrl->dev,
1271 			"q %d Create Association LS failed: %s\n",
1272 			queue->qnum, validation_errors[fcret]);
1273 	} else {
1274 		spin_lock_irqsave(&ctrl->lock, flags);
1275 		ctrl->association_id =
1276 			be64_to_cpu(assoc_acc->associd.association_id);
1277 		queue->connection_id =
1278 			be64_to_cpu(assoc_acc->connectid.connection_id);
1279 		set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1280 		spin_unlock_irqrestore(&ctrl->lock, flags);
1281 	}
1282 
1283 out_free_buffer:
1284 	kfree(lsop);
1285 out_no_memory:
1286 	if (ret)
1287 		dev_err(ctrl->dev,
1288 			"queue %d connect admin queue failed (%d).\n",
1289 			queue->qnum, ret);
1290 	return ret;
1291 }
1292 
1293 static int
1294 nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
1295 			u16 qsize, u16 ersp_ratio)
1296 {
1297 	struct nvmefc_ls_req_op *lsop;
1298 	struct nvmefc_ls_req *lsreq;
1299 	struct fcnvme_ls_cr_conn_rqst *conn_rqst;
1300 	struct fcnvme_ls_cr_conn_acc *conn_acc;
1301 	int ret, fcret = 0;
1302 
1303 	lsop = kzalloc((sizeof(*lsop) +
1304 			 sizeof(*conn_rqst) + sizeof(*conn_acc) +
1305 			 ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1306 	if (!lsop) {
1307 		dev_info(ctrl->ctrl.device,
1308 			"NVME-FC{%d}: send Create Connection failed: ENOMEM\n",
1309 			ctrl->cnum);
1310 		ret = -ENOMEM;
1311 		goto out_no_memory;
1312 	}
1313 
1314 	conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)&lsop[1];
1315 	conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1];
1316 	lsreq = &lsop->ls_req;
1317 	if (ctrl->lport->ops->lsrqst_priv_sz)
1318 		lsreq->private = (void *)&conn_acc[1];
1319 	else
1320 		lsreq->private = NULL;
1321 
1322 	conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION;
1323 	conn_rqst->desc_list_len = cpu_to_be32(
1324 				sizeof(struct fcnvme_lsdesc_assoc_id) +
1325 				sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1326 
1327 	conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1328 	conn_rqst->associd.desc_len =
1329 			fcnvme_lsdesc_len(
1330 				sizeof(struct fcnvme_lsdesc_assoc_id));
1331 	conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
1332 	conn_rqst->connect_cmd.desc_tag =
1333 			cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD);
1334 	conn_rqst->connect_cmd.desc_len =
1335 			fcnvme_lsdesc_len(
1336 				sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1337 	conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1338 	conn_rqst->connect_cmd.qid  = cpu_to_be16(queue->qnum);
1339 	conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1);
1340 
1341 	lsop->queue = queue;
1342 	lsreq->rqstaddr = conn_rqst;
1343 	lsreq->rqstlen = sizeof(*conn_rqst);
1344 	lsreq->rspaddr = conn_acc;
1345 	lsreq->rsplen = sizeof(*conn_acc);
1346 	lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1347 
1348 	ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1349 	if (ret)
1350 		goto out_free_buffer;
1351 
1352 	/* process connect LS completion */
1353 
1354 	/* validate the ACC response */
1355 	if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1356 		fcret = VERR_LSACC;
1357 	else if (conn_acc->hdr.desc_list_len !=
1358 			fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)))
1359 		fcret = VERR_CR_CONN_ACC_LEN;
1360 	else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
1361 		fcret = VERR_LSDESC_RQST;
1362 	else if (conn_acc->hdr.rqst.desc_len !=
1363 			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1364 		fcret = VERR_LSDESC_RQST_LEN;
1365 	else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION)
1366 		fcret = VERR_CR_CONN;
1367 	else if (conn_acc->connectid.desc_tag !=
1368 			cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1369 		fcret = VERR_CONN_ID;
1370 	else if (conn_acc->connectid.desc_len !=
1371 			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1372 		fcret = VERR_CONN_ID_LEN;
1373 
1374 	if (fcret) {
1375 		ret = -EBADF;
1376 		dev_err(ctrl->dev,
1377 			"q %d Create I/O Connection LS failed: %s\n",
1378 			queue->qnum, validation_errors[fcret]);
1379 	} else {
1380 		queue->connection_id =
1381 			be64_to_cpu(conn_acc->connectid.connection_id);
1382 		set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1383 	}
1384 
1385 out_free_buffer:
1386 	kfree(lsop);
1387 out_no_memory:
1388 	if (ret)
1389 		dev_err(ctrl->dev,
1390 			"queue %d connect I/O queue failed (%d).\n",
1391 			queue->qnum, ret);
1392 	return ret;
1393 }
1394 
1395 static void
1396 nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
1397 {
1398 	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1399 
1400 	__nvme_fc_finish_ls_req(lsop);
1401 
1402 	/* fc-nvme initiator doesn't care about success or failure of cmd */
1403 
1404 	kfree(lsop);
1405 }
1406 
1407 /*
1408  * This routine sends a FC-NVME LS to disconnect (aka terminate)
1409  * the FC-NVME Association.  Terminating the association also
1410  * terminates the FC-NVME connections (per queue, both admin and io
1411  * queues) that are part of the association. E.g. things are torn
1412  * down, and the related FC-NVME Association ID and Connection IDs
1413  * become invalid.
1414  *
1415  * The behavior of the fc-nvme initiator is such that it's
1416  * understanding of the association and connections will implicitly
1417  * be torn down. The action is implicit as it may be due to a loss of
1418  * connectivity with the fc-nvme target, so you may never get a
1419  * response even if you tried.  As such, the action of this routine
1420  * is to asynchronously send the LS, ignore any results of the LS, and
1421  * continue on with terminating the association. If the fc-nvme target
1422  * is present and receives the LS, it too can tear down.
1423  */
1424 static void
1425 nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
1426 {
1427 	struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
1428 	struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
1429 	struct nvmefc_ls_req_op *lsop;
1430 	struct nvmefc_ls_req *lsreq;
1431 	int ret;
1432 
1433 	lsop = kzalloc((sizeof(*lsop) +
1434 			sizeof(*discon_rqst) + sizeof(*discon_acc) +
1435 			ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1436 	if (!lsop) {
1437 		dev_info(ctrl->ctrl.device,
1438 			"NVME-FC{%d}: send Disconnect Association "
1439 			"failed: ENOMEM\n",
1440 			ctrl->cnum);
1441 		return;
1442 	}
1443 
1444 	discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
1445 	discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
1446 	lsreq = &lsop->ls_req;
1447 	if (ctrl->lport->ops->lsrqst_priv_sz)
1448 		lsreq->private = (void *)&discon_acc[1];
1449 	else
1450 		lsreq->private = NULL;
1451 
1452 	nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
1453 				ctrl->association_id);
1454 
1455 	ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop,
1456 				nvme_fc_disconnect_assoc_done);
1457 	if (ret)
1458 		kfree(lsop);
1459 }
1460 
1461 static void
1462 nvme_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1463 {
1464 	struct nvmefc_ls_rcv_op *lsop = lsrsp->nvme_fc_private;
1465 	struct nvme_fc_rport *rport = lsop->rport;
1466 	struct nvme_fc_lport *lport = rport->lport;
1467 	unsigned long flags;
1468 
1469 	spin_lock_irqsave(&rport->lock, flags);
1470 	list_del(&lsop->lsrcv_list);
1471 	spin_unlock_irqrestore(&rport->lock, flags);
1472 
1473 	fc_dma_sync_single_for_cpu(lport->dev, lsop->rspdma,
1474 				sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1475 	fc_dma_unmap_single(lport->dev, lsop->rspdma,
1476 			sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1477 
1478 	kfree(lsop->rspbuf);
1479 	kfree(lsop->rqstbuf);
1480 	kfree(lsop);
1481 
1482 	nvme_fc_rport_put(rport);
1483 }
1484 
1485 static void
1486 nvme_fc_xmt_ls_rsp(struct nvmefc_ls_rcv_op *lsop)
1487 {
1488 	struct nvme_fc_rport *rport = lsop->rport;
1489 	struct nvme_fc_lport *lport = rport->lport;
1490 	struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1491 	int ret;
1492 
1493 	fc_dma_sync_single_for_device(lport->dev, lsop->rspdma,
1494 				  sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1495 
1496 	ret = lport->ops->xmt_ls_rsp(&lport->localport, &rport->remoteport,
1497 				     lsop->lsrsp);
1498 	if (ret) {
1499 		dev_warn(lport->dev,
1500 			"LLDD rejected LS RSP xmt: LS %d status %d\n",
1501 			w0->ls_cmd, ret);
1502 		nvme_fc_xmt_ls_rsp_done(lsop->lsrsp);
1503 		return;
1504 	}
1505 }
1506 
1507 static struct nvme_fc_ctrl *
1508 nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport,
1509 		      struct nvmefc_ls_rcv_op *lsop)
1510 {
1511 	struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1512 					&lsop->rqstbuf->rq_dis_assoc;
1513 	struct nvme_fc_ctrl *ctrl, *ret = NULL;
1514 	struct nvmefc_ls_rcv_op *oldls = NULL;
1515 	u64 association_id = be64_to_cpu(rqst->associd.association_id);
1516 	unsigned long flags;
1517 
1518 	spin_lock_irqsave(&rport->lock, flags);
1519 
1520 	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
1521 		if (!nvme_fc_ctrl_get(ctrl))
1522 			continue;
1523 		spin_lock(&ctrl->lock);
1524 		if (association_id == ctrl->association_id) {
1525 			oldls = ctrl->rcv_disconn;
1526 			ctrl->rcv_disconn = lsop;
1527 			ret = ctrl;
1528 		}
1529 		spin_unlock(&ctrl->lock);
1530 		if (ret)
1531 			/* leave the ctrl get reference */
1532 			break;
1533 		nvme_fc_ctrl_put(ctrl);
1534 	}
1535 
1536 	spin_unlock_irqrestore(&rport->lock, flags);
1537 
1538 	/* transmit a response for anything that was pending */
1539 	if (oldls) {
1540 		dev_info(rport->lport->dev,
1541 			"NVME-FC{%d}: Multiple Disconnect Association "
1542 			"LS's received\n", ctrl->cnum);
1543 		/* overwrite good response with bogus failure */
1544 		oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf,
1545 						sizeof(*oldls->rspbuf),
1546 						rqst->w0.ls_cmd,
1547 						FCNVME_RJT_RC_UNAB,
1548 						FCNVME_RJT_EXP_NONE, 0);
1549 		nvme_fc_xmt_ls_rsp(oldls);
1550 	}
1551 
1552 	return ret;
1553 }
1554 
1555 /*
1556  * returns true to mean LS handled and ls_rsp can be sent
1557  * returns false to defer ls_rsp xmt (will be done as part of
1558  *     association termination)
1559  */
1560 static bool
1561 nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op *lsop)
1562 {
1563 	struct nvme_fc_rport *rport = lsop->rport;
1564 	struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1565 					&lsop->rqstbuf->rq_dis_assoc;
1566 	struct fcnvme_ls_disconnect_assoc_acc *acc =
1567 					&lsop->rspbuf->rsp_dis_assoc;
1568 	struct nvme_fc_ctrl *ctrl = NULL;
1569 	int ret = 0;
1570 
1571 	memset(acc, 0, sizeof(*acc));
1572 
1573 	ret = nvmefc_vldt_lsreq_discon_assoc(lsop->rqstdatalen, rqst);
1574 	if (!ret) {
1575 		/* match an active association */
1576 		ctrl = nvme_fc_match_disconn_ls(rport, lsop);
1577 		if (!ctrl)
1578 			ret = VERR_NO_ASSOC;
1579 	}
1580 
1581 	if (ret) {
1582 		dev_info(rport->lport->dev,
1583 			"Disconnect LS failed: %s\n",
1584 			validation_errors[ret]);
1585 		lsop->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1586 					sizeof(*acc), rqst->w0.ls_cmd,
1587 					(ret == VERR_NO_ASSOC) ?
1588 						FCNVME_RJT_RC_INV_ASSOC :
1589 						FCNVME_RJT_RC_LOGIC,
1590 					FCNVME_RJT_EXP_NONE, 0);
1591 		return true;
1592 	}
1593 
1594 	/* format an ACCept response */
1595 
1596 	lsop->lsrsp->rsplen = sizeof(*acc);
1597 
1598 	nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1599 			fcnvme_lsdesc_len(
1600 				sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1601 			FCNVME_LS_DISCONNECT_ASSOC);
1602 
1603 	/*
1604 	 * the transmit of the response will occur after the exchanges
1605 	 * for the association have been ABTS'd by
1606 	 * nvme_fc_delete_association().
1607 	 */
1608 
1609 	/* fail the association */
1610 	nvme_fc_error_recovery(ctrl, "Disconnect Association LS received");
1611 
1612 	/* release the reference taken by nvme_fc_match_disconn_ls() */
1613 	nvme_fc_ctrl_put(ctrl);
1614 
1615 	return false;
1616 }
1617 
1618 /*
1619  * Actual Processing routine for received FC-NVME LS Requests from the LLD
1620  * returns true if a response should be sent afterward, false if rsp will
1621  * be sent asynchronously.
1622  */
1623 static bool
1624 nvme_fc_handle_ls_rqst(struct nvmefc_ls_rcv_op *lsop)
1625 {
1626 	struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1627 	bool ret = true;
1628 
1629 	lsop->lsrsp->nvme_fc_private = lsop;
1630 	lsop->lsrsp->rspbuf = lsop->rspbuf;
1631 	lsop->lsrsp->rspdma = lsop->rspdma;
1632 	lsop->lsrsp->done = nvme_fc_xmt_ls_rsp_done;
1633 	/* Be preventative. handlers will later set to valid length */
1634 	lsop->lsrsp->rsplen = 0;
1635 
1636 	/*
1637 	 * handlers:
1638 	 *   parse request input, execute the request, and format the
1639 	 *   LS response
1640 	 */
1641 	switch (w0->ls_cmd) {
1642 	case FCNVME_LS_DISCONNECT_ASSOC:
1643 		ret = nvme_fc_ls_disconnect_assoc(lsop);
1644 		break;
1645 	case FCNVME_LS_DISCONNECT_CONN:
1646 		lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1647 				sizeof(*lsop->rspbuf), w0->ls_cmd,
1648 				FCNVME_RJT_RC_UNSUP, FCNVME_RJT_EXP_NONE, 0);
1649 		break;
1650 	case FCNVME_LS_CREATE_ASSOCIATION:
1651 	case FCNVME_LS_CREATE_CONNECTION:
1652 		lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1653 				sizeof(*lsop->rspbuf), w0->ls_cmd,
1654 				FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0);
1655 		break;
1656 	default:
1657 		lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1658 				sizeof(*lsop->rspbuf), w0->ls_cmd,
1659 				FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1660 		break;
1661 	}
1662 
1663 	return(ret);
1664 }
1665 
1666 static void
1667 nvme_fc_handle_ls_rqst_work(struct work_struct *work)
1668 {
1669 	struct nvme_fc_rport *rport =
1670 		container_of(work, struct nvme_fc_rport, lsrcv_work);
1671 	struct fcnvme_ls_rqst_w0 *w0;
1672 	struct nvmefc_ls_rcv_op *lsop;
1673 	unsigned long flags;
1674 	bool sendrsp;
1675 
1676 restart:
1677 	sendrsp = true;
1678 	spin_lock_irqsave(&rport->lock, flags);
1679 	list_for_each_entry(lsop, &rport->ls_rcv_list, lsrcv_list) {
1680 		if (lsop->handled)
1681 			continue;
1682 
1683 		lsop->handled = true;
1684 		if (rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
1685 			spin_unlock_irqrestore(&rport->lock, flags);
1686 			sendrsp = nvme_fc_handle_ls_rqst(lsop);
1687 		} else {
1688 			spin_unlock_irqrestore(&rport->lock, flags);
1689 			w0 = &lsop->rqstbuf->w0;
1690 			lsop->lsrsp->rsplen = nvme_fc_format_rjt(
1691 						lsop->rspbuf,
1692 						sizeof(*lsop->rspbuf),
1693 						w0->ls_cmd,
1694 						FCNVME_RJT_RC_UNAB,
1695 						FCNVME_RJT_EXP_NONE, 0);
1696 		}
1697 		if (sendrsp)
1698 			nvme_fc_xmt_ls_rsp(lsop);
1699 		goto restart;
1700 	}
1701 	spin_unlock_irqrestore(&rport->lock, flags);
1702 }
1703 
1704 static
1705 void nvme_fc_rcv_ls_req_err_msg(struct nvme_fc_lport *lport,
1706 				struct fcnvme_ls_rqst_w0 *w0)
1707 {
1708 	dev_info(lport->dev, "RCV %s LS failed: No memory\n",
1709 		(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1710 			nvmefc_ls_names[w0->ls_cmd] : "");
1711 }
1712 
1713 /**
1714  * nvme_fc_rcv_ls_req - transport entry point called by an LLDD
1715  *                       upon the reception of a NVME LS request.
1716  *
1717  * The nvme-fc layer will copy payload to an internal structure for
1718  * processing.  As such, upon completion of the routine, the LLDD may
1719  * immediately free/reuse the LS request buffer passed in the call.
1720  *
1721  * If this routine returns error, the LLDD should abort the exchange.
1722  *
1723  * @portptr:    pointer to the (registered) remote port that the LS
1724  *              was received from. The remoteport is associated with
1725  *              a specific localport.
1726  * @lsrsp:      pointer to a nvmefc_ls_rsp response structure to be
1727  *              used to reference the exchange corresponding to the LS
1728  *              when issuing an ls response.
1729  * @lsreqbuf:   pointer to the buffer containing the LS Request
1730  * @lsreqbuf_len: length, in bytes, of the received LS request
1731  */
1732 int
1733 nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr,
1734 			struct nvmefc_ls_rsp *lsrsp,
1735 			void *lsreqbuf, u32 lsreqbuf_len)
1736 {
1737 	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
1738 	struct nvme_fc_lport *lport = rport->lport;
1739 	struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
1740 	struct nvmefc_ls_rcv_op *lsop;
1741 	unsigned long flags;
1742 	int ret;
1743 
1744 	nvme_fc_rport_get(rport);
1745 
1746 	/* validate there's a routine to transmit a response */
1747 	if (!lport->ops->xmt_ls_rsp) {
1748 		dev_info(lport->dev,
1749 			"RCV %s LS failed: no LLDD xmt_ls_rsp\n",
1750 			(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1751 				nvmefc_ls_names[w0->ls_cmd] : "");
1752 		ret = -EINVAL;
1753 		goto out_put;
1754 	}
1755 
1756 	if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
1757 		dev_info(lport->dev,
1758 			"RCV %s LS failed: payload too large\n",
1759 			(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1760 				nvmefc_ls_names[w0->ls_cmd] : "");
1761 		ret = -E2BIG;
1762 		goto out_put;
1763 	}
1764 
1765 	lsop = kzalloc(sizeof(*lsop), GFP_KERNEL);
1766 	if (!lsop) {
1767 		nvme_fc_rcv_ls_req_err_msg(lport, w0);
1768 		ret = -ENOMEM;
1769 		goto out_put;
1770 	}
1771 
1772 	lsop->rqstbuf = kzalloc(sizeof(*lsop->rqstbuf), GFP_KERNEL);
1773 	lsop->rspbuf = kzalloc(sizeof(*lsop->rspbuf), GFP_KERNEL);
1774 	if (!lsop->rqstbuf || !lsop->rspbuf) {
1775 		nvme_fc_rcv_ls_req_err_msg(lport, w0);
1776 		ret = -ENOMEM;
1777 		goto out_free;
1778 	}
1779 
1780 	lsop->rspdma = fc_dma_map_single(lport->dev, lsop->rspbuf,
1781 					sizeof(*lsop->rspbuf),
1782 					DMA_TO_DEVICE);
1783 	if (fc_dma_mapping_error(lport->dev, lsop->rspdma)) {
1784 		dev_info(lport->dev,
1785 			"RCV %s LS failed: DMA mapping failure\n",
1786 			(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1787 				nvmefc_ls_names[w0->ls_cmd] : "");
1788 		ret = -EFAULT;
1789 		goto out_free;
1790 	}
1791 
1792 	lsop->rport = rport;
1793 	lsop->lsrsp = lsrsp;
1794 
1795 	memcpy(lsop->rqstbuf, lsreqbuf, lsreqbuf_len);
1796 	lsop->rqstdatalen = lsreqbuf_len;
1797 
1798 	spin_lock_irqsave(&rport->lock, flags);
1799 	if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) {
1800 		spin_unlock_irqrestore(&rport->lock, flags);
1801 		ret = -ENOTCONN;
1802 		goto out_unmap;
1803 	}
1804 	list_add_tail(&lsop->lsrcv_list, &rport->ls_rcv_list);
1805 	spin_unlock_irqrestore(&rport->lock, flags);
1806 
1807 	schedule_work(&rport->lsrcv_work);
1808 
1809 	return 0;
1810 
1811 out_unmap:
1812 	fc_dma_unmap_single(lport->dev, lsop->rspdma,
1813 			sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1814 out_free:
1815 	kfree(lsop->rspbuf);
1816 	kfree(lsop->rqstbuf);
1817 	kfree(lsop);
1818 out_put:
1819 	nvme_fc_rport_put(rport);
1820 	return ret;
1821 }
1822 EXPORT_SYMBOL_GPL(nvme_fc_rcv_ls_req);
1823 
1824 
1825 /* *********************** NVME Ctrl Routines **************************** */
1826 
1827 static void
1828 __nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl,
1829 		struct nvme_fc_fcp_op *op)
1830 {
1831 	fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.rspdma,
1832 				sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1833 	fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma,
1834 				sizeof(op->cmd_iu), DMA_TO_DEVICE);
1835 
1836 	atomic_set(&op->state, FCPOP_STATE_UNINIT);
1837 }
1838 
1839 static void
1840 nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq,
1841 		unsigned int hctx_idx)
1842 {
1843 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
1844 
1845 	return __nvme_fc_exit_request(to_fc_ctrl(set->driver_data), op);
1846 }
1847 
1848 static int
1849 __nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
1850 {
1851 	unsigned long flags;
1852 	int opstate;
1853 
1854 	spin_lock_irqsave(&ctrl->lock, flags);
1855 	opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED);
1856 	if (opstate != FCPOP_STATE_ACTIVE)
1857 		atomic_set(&op->state, opstate);
1858 	else if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) {
1859 		op->flags |= FCOP_FLAGS_TERMIO;
1860 		ctrl->iocnt++;
1861 	}
1862 	spin_unlock_irqrestore(&ctrl->lock, flags);
1863 
1864 	if (opstate != FCPOP_STATE_ACTIVE)
1865 		return -ECANCELED;
1866 
1867 	ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
1868 					&ctrl->rport->remoteport,
1869 					op->queue->lldd_handle,
1870 					&op->fcp_req);
1871 
1872 	return 0;
1873 }
1874 
1875 static void
1876 nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl)
1877 {
1878 	struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
1879 	int i;
1880 
1881 	/* ensure we've initialized the ops once */
1882 	if (!(aen_op->flags & FCOP_FLAGS_AEN))
1883 		return;
1884 
1885 	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++)
1886 		__nvme_fc_abort_op(ctrl, aen_op);
1887 }
1888 
1889 static inline void
1890 __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
1891 		struct nvme_fc_fcp_op *op, int opstate)
1892 {
1893 	unsigned long flags;
1894 
1895 	if (opstate == FCPOP_STATE_ABORTED) {
1896 		spin_lock_irqsave(&ctrl->lock, flags);
1897 		if (test_bit(FCCTRL_TERMIO, &ctrl->flags) &&
1898 		    op->flags & FCOP_FLAGS_TERMIO) {
1899 			if (!--ctrl->iocnt)
1900 				wake_up(&ctrl->ioabort_wait);
1901 		}
1902 		spin_unlock_irqrestore(&ctrl->lock, flags);
1903 	}
1904 }
1905 
1906 static void
1907 nvme_fc_ctrl_ioerr_work(struct work_struct *work)
1908 {
1909 	struct nvme_fc_ctrl *ctrl =
1910 			container_of(work, struct nvme_fc_ctrl, ioerr_work);
1911 
1912 	nvme_fc_error_recovery(ctrl, "transport detected io error");
1913 }
1914 
1915 /*
1916  * nvme_fc_io_getuuid - Routine called to get the appid field
1917  * associated with request by the lldd
1918  * @req:IO request from nvme fc to driver
1919  * Returns: UUID if there is an appid associated with VM or
1920  * NULL if the user/libvirt has not set the appid to VM
1921  */
1922 char *nvme_fc_io_getuuid(struct nvmefc_fcp_req *req)
1923 {
1924 	struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1925 	struct request *rq = op->rq;
1926 
1927 	if (!IS_ENABLED(CONFIG_BLK_CGROUP_FC_APPID) || !rq || !rq->bio)
1928 		return NULL;
1929 	return blkcg_get_fc_appid(rq->bio);
1930 }
1931 EXPORT_SYMBOL_GPL(nvme_fc_io_getuuid);
1932 
1933 static void
1934 nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
1935 {
1936 	struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1937 	struct request *rq = op->rq;
1938 	struct nvmefc_fcp_req *freq = &op->fcp_req;
1939 	struct nvme_fc_ctrl *ctrl = op->ctrl;
1940 	struct nvme_fc_queue *queue = op->queue;
1941 	struct nvme_completion *cqe = &op->rsp_iu.cqe;
1942 	struct nvme_command *sqe = &op->cmd_iu.sqe;
1943 	__le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1);
1944 	union nvme_result result;
1945 	bool terminate_assoc = true;
1946 	int opstate;
1947 
1948 	/*
1949 	 * WARNING:
1950 	 * The current linux implementation of a nvme controller
1951 	 * allocates a single tag set for all io queues and sizes
1952 	 * the io queues to fully hold all possible tags. Thus, the
1953 	 * implementation does not reference or care about the sqhd
1954 	 * value as it never needs to use the sqhd/sqtail pointers
1955 	 * for submission pacing.
1956 	 *
1957 	 * This affects the FC-NVME implementation in two ways:
1958 	 * 1) As the value doesn't matter, we don't need to waste
1959 	 *    cycles extracting it from ERSPs and stamping it in the
1960 	 *    cases where the transport fabricates CQEs on successful
1961 	 *    completions.
1962 	 * 2) The FC-NVME implementation requires that delivery of
1963 	 *    ERSP completions are to go back to the nvme layer in order
1964 	 *    relative to the rsn, such that the sqhd value will always
1965 	 *    be "in order" for the nvme layer. As the nvme layer in
1966 	 *    linux doesn't care about sqhd, there's no need to return
1967 	 *    them in order.
1968 	 *
1969 	 * Additionally:
1970 	 * As the core nvme layer in linux currently does not look at
1971 	 * every field in the cqe - in cases where the FC transport must
1972 	 * fabricate a CQE, the following fields will not be set as they
1973 	 * are not referenced:
1974 	 *      cqe.sqid,  cqe.sqhd,  cqe.command_id
1975 	 *
1976 	 * Failure or error of an individual i/o, in a transport
1977 	 * detected fashion unrelated to the nvme completion status,
1978 	 * potentially cause the initiator and target sides to get out
1979 	 * of sync on SQ head/tail (aka outstanding io count allowed).
1980 	 * Per FC-NVME spec, failure of an individual command requires
1981 	 * the connection to be terminated, which in turn requires the
1982 	 * association to be terminated.
1983 	 */
1984 
1985 	opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
1986 
1987 	fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma,
1988 				sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1989 
1990 	if (opstate == FCPOP_STATE_ABORTED)
1991 		status = cpu_to_le16(NVME_SC_HOST_ABORTED_CMD << 1);
1992 	else if (freq->status) {
1993 		status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
1994 		dev_info(ctrl->ctrl.device,
1995 			"NVME-FC{%d}: io failed due to lldd error %d\n",
1996 			ctrl->cnum, freq->status);
1997 	}
1998 
1999 	/*
2000 	 * For the linux implementation, if we have an unsuccesful
2001 	 * status, they blk-mq layer can typically be called with the
2002 	 * non-zero status and the content of the cqe isn't important.
2003 	 */
2004 	if (status)
2005 		goto done;
2006 
2007 	/*
2008 	 * command completed successfully relative to the wire
2009 	 * protocol. However, validate anything received and
2010 	 * extract the status and result from the cqe (create it
2011 	 * where necessary).
2012 	 */
2013 
2014 	switch (freq->rcv_rsplen) {
2015 
2016 	case 0:
2017 	case NVME_FC_SIZEOF_ZEROS_RSP:
2018 		/*
2019 		 * No response payload or 12 bytes of payload (which
2020 		 * should all be zeros) are considered successful and
2021 		 * no payload in the CQE by the transport.
2022 		 */
2023 		if (freq->transferred_length !=
2024 		    be32_to_cpu(op->cmd_iu.data_len)) {
2025 			status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2026 			dev_info(ctrl->ctrl.device,
2027 				"NVME-FC{%d}: io failed due to bad transfer "
2028 				"length: %d vs expected %d\n",
2029 				ctrl->cnum, freq->transferred_length,
2030 				be32_to_cpu(op->cmd_iu.data_len));
2031 			goto done;
2032 		}
2033 		result.u64 = 0;
2034 		break;
2035 
2036 	case sizeof(struct nvme_fc_ersp_iu):
2037 		/*
2038 		 * The ERSP IU contains a full completion with CQE.
2039 		 * Validate ERSP IU and look at cqe.
2040 		 */
2041 		if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) !=
2042 					(freq->rcv_rsplen / 4) ||
2043 			     be32_to_cpu(op->rsp_iu.xfrd_len) !=
2044 					freq->transferred_length ||
2045 			     op->rsp_iu.ersp_result ||
2046 			     sqe->common.command_id != cqe->command_id)) {
2047 			status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2048 			dev_info(ctrl->ctrl.device,
2049 				"NVME-FC{%d}: io failed due to bad NVMe_ERSP: "
2050 				"iu len %d, xfr len %d vs %d, status code "
2051 				"%d, cmdid %d vs %d\n",
2052 				ctrl->cnum, be16_to_cpu(op->rsp_iu.iu_len),
2053 				be32_to_cpu(op->rsp_iu.xfrd_len),
2054 				freq->transferred_length,
2055 				op->rsp_iu.ersp_result,
2056 				sqe->common.command_id,
2057 				cqe->command_id);
2058 			goto done;
2059 		}
2060 		result = cqe->result;
2061 		status = cqe->status;
2062 		break;
2063 
2064 	default:
2065 		status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2066 		dev_info(ctrl->ctrl.device,
2067 			"NVME-FC{%d}: io failed due to odd NVMe_xRSP iu "
2068 			"len %d\n",
2069 			ctrl->cnum, freq->rcv_rsplen);
2070 		goto done;
2071 	}
2072 
2073 	terminate_assoc = false;
2074 
2075 done:
2076 	if (op->flags & FCOP_FLAGS_AEN) {
2077 		nvme_complete_async_event(&queue->ctrl->ctrl, status, &result);
2078 		__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2079 		atomic_set(&op->state, FCPOP_STATE_IDLE);
2080 		op->flags = FCOP_FLAGS_AEN;	/* clear other flags */
2081 		nvme_fc_ctrl_put(ctrl);
2082 		goto check_error;
2083 	}
2084 
2085 	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2086 	if (!nvme_try_complete_req(rq, status, result))
2087 		nvme_fc_complete_rq(rq);
2088 
2089 check_error:
2090 	if (terminate_assoc && ctrl->ctrl.state != NVME_CTRL_RESETTING)
2091 		queue_work(nvme_reset_wq, &ctrl->ioerr_work);
2092 }
2093 
2094 static int
2095 __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl,
2096 		struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op,
2097 		struct request *rq, u32 rqno)
2098 {
2099 	struct nvme_fcp_op_w_sgl *op_w_sgl =
2100 		container_of(op, typeof(*op_w_sgl), op);
2101 	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2102 	int ret = 0;
2103 
2104 	memset(op, 0, sizeof(*op));
2105 	op->fcp_req.cmdaddr = &op->cmd_iu;
2106 	op->fcp_req.cmdlen = sizeof(op->cmd_iu);
2107 	op->fcp_req.rspaddr = &op->rsp_iu;
2108 	op->fcp_req.rsplen = sizeof(op->rsp_iu);
2109 	op->fcp_req.done = nvme_fc_fcpio_done;
2110 	op->ctrl = ctrl;
2111 	op->queue = queue;
2112 	op->rq = rq;
2113 	op->rqno = rqno;
2114 
2115 	cmdiu->format_id = NVME_CMD_FORMAT_ID;
2116 	cmdiu->fc_id = NVME_CMD_FC_ID;
2117 	cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
2118 	if (queue->qnum)
2119 		cmdiu->rsv_cat = fccmnd_set_cat_css(0,
2120 					(NVME_CC_CSS_NVM >> NVME_CC_CSS_SHIFT));
2121 	else
2122 		cmdiu->rsv_cat = fccmnd_set_cat_admin(0);
2123 
2124 	op->fcp_req.cmddma = fc_dma_map_single(ctrl->lport->dev,
2125 				&op->cmd_iu, sizeof(op->cmd_iu), DMA_TO_DEVICE);
2126 	if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.cmddma)) {
2127 		dev_err(ctrl->dev,
2128 			"FCP Op failed - cmdiu dma mapping failed.\n");
2129 		ret = -EFAULT;
2130 		goto out_on_error;
2131 	}
2132 
2133 	op->fcp_req.rspdma = fc_dma_map_single(ctrl->lport->dev,
2134 				&op->rsp_iu, sizeof(op->rsp_iu),
2135 				DMA_FROM_DEVICE);
2136 	if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.rspdma)) {
2137 		dev_err(ctrl->dev,
2138 			"FCP Op failed - rspiu dma mapping failed.\n");
2139 		ret = -EFAULT;
2140 	}
2141 
2142 	atomic_set(&op->state, FCPOP_STATE_IDLE);
2143 out_on_error:
2144 	return ret;
2145 }
2146 
2147 static int
2148 nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq,
2149 		unsigned int hctx_idx, unsigned int numa_node)
2150 {
2151 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data);
2152 	struct nvme_fcp_op_w_sgl *op = blk_mq_rq_to_pdu(rq);
2153 	int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
2154 	struct nvme_fc_queue *queue = &ctrl->queues[queue_idx];
2155 	int res;
2156 
2157 	res = __nvme_fc_init_request(ctrl, queue, &op->op, rq, queue->rqcnt++);
2158 	if (res)
2159 		return res;
2160 	op->op.fcp_req.first_sgl = op->sgl;
2161 	op->op.fcp_req.private = &op->priv[0];
2162 	nvme_req(rq)->ctrl = &ctrl->ctrl;
2163 	nvme_req(rq)->cmd = &op->op.cmd_iu.sqe;
2164 	return res;
2165 }
2166 
2167 static int
2168 nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl)
2169 {
2170 	struct nvme_fc_fcp_op *aen_op;
2171 	struct nvme_fc_cmd_iu *cmdiu;
2172 	struct nvme_command *sqe;
2173 	void *private = NULL;
2174 	int i, ret;
2175 
2176 	aen_op = ctrl->aen_ops;
2177 	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2178 		if (ctrl->lport->ops->fcprqst_priv_sz) {
2179 			private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz,
2180 						GFP_KERNEL);
2181 			if (!private)
2182 				return -ENOMEM;
2183 		}
2184 
2185 		cmdiu = &aen_op->cmd_iu;
2186 		sqe = &cmdiu->sqe;
2187 		ret = __nvme_fc_init_request(ctrl, &ctrl->queues[0],
2188 				aen_op, (struct request *)NULL,
2189 				(NVME_AQ_BLK_MQ_DEPTH + i));
2190 		if (ret) {
2191 			kfree(private);
2192 			return ret;
2193 		}
2194 
2195 		aen_op->flags = FCOP_FLAGS_AEN;
2196 		aen_op->fcp_req.private = private;
2197 
2198 		memset(sqe, 0, sizeof(*sqe));
2199 		sqe->common.opcode = nvme_admin_async_event;
2200 		/* Note: core layer may overwrite the sqe.command_id value */
2201 		sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i;
2202 	}
2203 	return 0;
2204 }
2205 
2206 static void
2207 nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl)
2208 {
2209 	struct nvme_fc_fcp_op *aen_op;
2210 	int i;
2211 
2212 	cancel_work_sync(&ctrl->ctrl.async_event_work);
2213 	aen_op = ctrl->aen_ops;
2214 	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2215 		__nvme_fc_exit_request(ctrl, aen_op);
2216 
2217 		kfree(aen_op->fcp_req.private);
2218 		aen_op->fcp_req.private = NULL;
2219 	}
2220 }
2221 
2222 static inline int
2223 __nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int qidx)
2224 {
2225 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(data);
2226 	struct nvme_fc_queue *queue = &ctrl->queues[qidx];
2227 
2228 	hctx->driver_data = queue;
2229 	queue->hctx = hctx;
2230 	return 0;
2231 }
2232 
2233 static int
2234 nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx)
2235 {
2236 	return __nvme_fc_init_hctx(hctx, data, hctx_idx + 1);
2237 }
2238 
2239 static int
2240 nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
2241 		unsigned int hctx_idx)
2242 {
2243 	return __nvme_fc_init_hctx(hctx, data, hctx_idx);
2244 }
2245 
2246 static void
2247 nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx)
2248 {
2249 	struct nvme_fc_queue *queue;
2250 
2251 	queue = &ctrl->queues[idx];
2252 	memset(queue, 0, sizeof(*queue));
2253 	queue->ctrl = ctrl;
2254 	queue->qnum = idx;
2255 	atomic_set(&queue->csn, 0);
2256 	queue->dev = ctrl->dev;
2257 
2258 	if (idx > 0)
2259 		queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
2260 	else
2261 		queue->cmnd_capsule_len = sizeof(struct nvme_command);
2262 
2263 	/*
2264 	 * Considered whether we should allocate buffers for all SQEs
2265 	 * and CQEs and dma map them - mapping their respective entries
2266 	 * into the request structures (kernel vm addr and dma address)
2267 	 * thus the driver could use the buffers/mappings directly.
2268 	 * It only makes sense if the LLDD would use them for its
2269 	 * messaging api. It's very unlikely most adapter api's would use
2270 	 * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
2271 	 * structures were used instead.
2272 	 */
2273 }
2274 
2275 /*
2276  * This routine terminates a queue at the transport level.
2277  * The transport has already ensured that all outstanding ios on
2278  * the queue have been terminated.
2279  * The transport will send a Disconnect LS request to terminate
2280  * the queue's connection. Termination of the admin queue will also
2281  * terminate the association at the target.
2282  */
2283 static void
2284 nvme_fc_free_queue(struct nvme_fc_queue *queue)
2285 {
2286 	if (!test_and_clear_bit(NVME_FC_Q_CONNECTED, &queue->flags))
2287 		return;
2288 
2289 	clear_bit(NVME_FC_Q_LIVE, &queue->flags);
2290 	/*
2291 	 * Current implementation never disconnects a single queue.
2292 	 * It always terminates a whole association. So there is never
2293 	 * a disconnect(queue) LS sent to the target.
2294 	 */
2295 
2296 	queue->connection_id = 0;
2297 	atomic_set(&queue->csn, 0);
2298 }
2299 
2300 static void
2301 __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl,
2302 	struct nvme_fc_queue *queue, unsigned int qidx)
2303 {
2304 	if (ctrl->lport->ops->delete_queue)
2305 		ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx,
2306 				queue->lldd_handle);
2307 	queue->lldd_handle = NULL;
2308 }
2309 
2310 static void
2311 nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl)
2312 {
2313 	int i;
2314 
2315 	for (i = 1; i < ctrl->ctrl.queue_count; i++)
2316 		nvme_fc_free_queue(&ctrl->queues[i]);
2317 }
2318 
2319 static int
2320 __nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl,
2321 	struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize)
2322 {
2323 	int ret = 0;
2324 
2325 	queue->lldd_handle = NULL;
2326 	if (ctrl->lport->ops->create_queue)
2327 		ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport,
2328 				qidx, qsize, &queue->lldd_handle);
2329 
2330 	return ret;
2331 }
2332 
2333 static void
2334 nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)
2335 {
2336 	struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1];
2337 	int i;
2338 
2339 	for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--)
2340 		__nvme_fc_delete_hw_queue(ctrl, queue, i);
2341 }
2342 
2343 static int
2344 nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2345 {
2346 	struct nvme_fc_queue *queue = &ctrl->queues[1];
2347 	int i, ret;
2348 
2349 	for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) {
2350 		ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize);
2351 		if (ret)
2352 			goto delete_queues;
2353 	}
2354 
2355 	return 0;
2356 
2357 delete_queues:
2358 	for (; i > 0; i--)
2359 		__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i);
2360 	return ret;
2361 }
2362 
2363 static int
2364 nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2365 {
2366 	int i, ret = 0;
2367 
2368 	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
2369 		ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize,
2370 					(qsize / 5));
2371 		if (ret)
2372 			break;
2373 		ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
2374 		if (ret)
2375 			break;
2376 
2377 		set_bit(NVME_FC_Q_LIVE, &ctrl->queues[i].flags);
2378 	}
2379 
2380 	return ret;
2381 }
2382 
2383 static void
2384 nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl)
2385 {
2386 	int i;
2387 
2388 	for (i = 1; i < ctrl->ctrl.queue_count; i++)
2389 		nvme_fc_init_queue(ctrl, i);
2390 }
2391 
2392 static void
2393 nvme_fc_ctrl_free(struct kref *ref)
2394 {
2395 	struct nvme_fc_ctrl *ctrl =
2396 		container_of(ref, struct nvme_fc_ctrl, ref);
2397 	unsigned long flags;
2398 
2399 	if (ctrl->ctrl.tagset)
2400 		nvme_remove_io_tag_set(&ctrl->ctrl);
2401 
2402 	/* remove from rport list */
2403 	spin_lock_irqsave(&ctrl->rport->lock, flags);
2404 	list_del(&ctrl->ctrl_list);
2405 	spin_unlock_irqrestore(&ctrl->rport->lock, flags);
2406 
2407 	nvme_unquiesce_admin_queue(&ctrl->ctrl);
2408 	nvme_remove_admin_tag_set(&ctrl->ctrl);
2409 
2410 	kfree(ctrl->queues);
2411 
2412 	put_device(ctrl->dev);
2413 	nvme_fc_rport_put(ctrl->rport);
2414 
2415 	ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
2416 	if (ctrl->ctrl.opts)
2417 		nvmf_free_options(ctrl->ctrl.opts);
2418 	kfree(ctrl);
2419 }
2420 
2421 static void
2422 nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl)
2423 {
2424 	kref_put(&ctrl->ref, nvme_fc_ctrl_free);
2425 }
2426 
2427 static int
2428 nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl)
2429 {
2430 	return kref_get_unless_zero(&ctrl->ref);
2431 }
2432 
2433 /*
2434  * All accesses from nvme core layer done - can now free the
2435  * controller. Called after last nvme_put_ctrl() call
2436  */
2437 static void
2438 nvme_fc_free_ctrl(struct nvme_ctrl *nctrl)
2439 {
2440 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2441 
2442 	WARN_ON(nctrl != &ctrl->ctrl);
2443 
2444 	nvme_fc_ctrl_put(ctrl);
2445 }
2446 
2447 /*
2448  * This routine is used by the transport when it needs to find active
2449  * io on a queue that is to be terminated. The transport uses
2450  * blk_mq_tagset_busy_itr() to find the busy requests, which then invoke
2451  * this routine to kill them on a 1 by 1 basis.
2452  *
2453  * As FC allocates FC exchange for each io, the transport must contact
2454  * the LLDD to terminate the exchange, thus releasing the FC exchange.
2455  * After terminating the exchange the LLDD will call the transport's
2456  * normal io done path for the request, but it will have an aborted
2457  * status. The done path will return the io request back to the block
2458  * layer with an error status.
2459  */
2460 static bool nvme_fc_terminate_exchange(struct request *req, void *data)
2461 {
2462 	struct nvme_ctrl *nctrl = data;
2463 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2464 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req);
2465 
2466 	op->nreq.flags |= NVME_REQ_CANCELLED;
2467 	__nvme_fc_abort_op(ctrl, op);
2468 	return true;
2469 }
2470 
2471 /*
2472  * This routine runs through all outstanding commands on the association
2473  * and aborts them.  This routine is typically be called by the
2474  * delete_association routine. It is also called due to an error during
2475  * reconnect. In that scenario, it is most likely a command that initializes
2476  * the controller, including fabric Connect commands on io queues, that
2477  * may have timed out or failed thus the io must be killed for the connect
2478  * thread to see the error.
2479  */
2480 static void
2481 __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)
2482 {
2483 	int q;
2484 
2485 	/*
2486 	 * if aborting io, the queues are no longer good, mark them
2487 	 * all as not live.
2488 	 */
2489 	if (ctrl->ctrl.queue_count > 1) {
2490 		for (q = 1; q < ctrl->ctrl.queue_count; q++)
2491 			clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[q].flags);
2492 	}
2493 	clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
2494 
2495 	/*
2496 	 * If io queues are present, stop them and terminate all outstanding
2497 	 * ios on them. As FC allocates FC exchange for each io, the
2498 	 * transport must contact the LLDD to terminate the exchange,
2499 	 * thus releasing the FC exchange. We use blk_mq_tagset_busy_itr()
2500 	 * to tell us what io's are busy and invoke a transport routine
2501 	 * to kill them with the LLDD.  After terminating the exchange
2502 	 * the LLDD will call the transport's normal io done path, but it
2503 	 * will have an aborted status. The done path will return the
2504 	 * io requests back to the block layer as part of normal completions
2505 	 * (but with error status).
2506 	 */
2507 	if (ctrl->ctrl.queue_count > 1) {
2508 		nvme_quiesce_io_queues(&ctrl->ctrl);
2509 		nvme_sync_io_queues(&ctrl->ctrl);
2510 		blk_mq_tagset_busy_iter(&ctrl->tag_set,
2511 				nvme_fc_terminate_exchange, &ctrl->ctrl);
2512 		blk_mq_tagset_wait_completed_request(&ctrl->tag_set);
2513 		if (start_queues)
2514 			nvme_unquiesce_io_queues(&ctrl->ctrl);
2515 	}
2516 
2517 	/*
2518 	 * Other transports, which don't have link-level contexts bound
2519 	 * to sqe's, would try to gracefully shutdown the controller by
2520 	 * writing the registers for shutdown and polling (call
2521 	 * nvme_disable_ctrl()). Given a bunch of i/o was potentially
2522 	 * just aborted and we will wait on those contexts, and given
2523 	 * there was no indication of how live the controlelr is on the
2524 	 * link, don't send more io to create more contexts for the
2525 	 * shutdown. Let the controller fail via keepalive failure if
2526 	 * its still present.
2527 	 */
2528 
2529 	/*
2530 	 * clean up the admin queue. Same thing as above.
2531 	 */
2532 	nvme_quiesce_admin_queue(&ctrl->ctrl);
2533 	blk_sync_queue(ctrl->ctrl.admin_q);
2534 	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
2535 				nvme_fc_terminate_exchange, &ctrl->ctrl);
2536 	blk_mq_tagset_wait_completed_request(&ctrl->admin_tag_set);
2537 	if (start_queues)
2538 		nvme_unquiesce_admin_queue(&ctrl->ctrl);
2539 }
2540 
2541 static void
2542 nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg)
2543 {
2544 	/*
2545 	 * if an error (io timeout, etc) while (re)connecting, the remote
2546 	 * port requested terminating of the association (disconnect_ls)
2547 	 * or an error (timeout or abort) occurred on an io while creating
2548 	 * the controller.  Abort any ios on the association and let the
2549 	 * create_association error path resolve things.
2550 	 */
2551 	if (ctrl->ctrl.state == NVME_CTRL_CONNECTING) {
2552 		__nvme_fc_abort_outstanding_ios(ctrl, true);
2553 		dev_warn(ctrl->ctrl.device,
2554 			"NVME-FC{%d}: transport error during (re)connect\n",
2555 			ctrl->cnum);
2556 		return;
2557 	}
2558 
2559 	/* Otherwise, only proceed if in LIVE state - e.g. on first error */
2560 	if (ctrl->ctrl.state != NVME_CTRL_LIVE)
2561 		return;
2562 
2563 	dev_warn(ctrl->ctrl.device,
2564 		"NVME-FC{%d}: transport association event: %s\n",
2565 		ctrl->cnum, errmsg);
2566 	dev_warn(ctrl->ctrl.device,
2567 		"NVME-FC{%d}: resetting controller\n", ctrl->cnum);
2568 
2569 	nvme_reset_ctrl(&ctrl->ctrl);
2570 }
2571 
2572 static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq)
2573 {
2574 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2575 	struct nvme_fc_ctrl *ctrl = op->ctrl;
2576 	u16 qnum = op->queue->qnum;
2577 	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2578 	struct nvme_command *sqe = &cmdiu->sqe;
2579 
2580 	/*
2581 	 * Attempt to abort the offending command. Command completion
2582 	 * will detect the aborted io and will fail the connection.
2583 	 */
2584 	dev_info(ctrl->ctrl.device,
2585 		"NVME-FC{%d.%d}: io timeout: opcode %d fctype %d (%s) w10/11: "
2586 		"x%08x/x%08x\n",
2587 		ctrl->cnum, qnum, sqe->common.opcode, sqe->fabrics.fctype,
2588 		nvme_fabrics_opcode_str(qnum, sqe),
2589 		sqe->common.cdw10, sqe->common.cdw11);
2590 	if (__nvme_fc_abort_op(ctrl, op))
2591 		nvme_fc_error_recovery(ctrl, "io timeout abort failed");
2592 
2593 	/*
2594 	 * the io abort has been initiated. Have the reset timer
2595 	 * restarted and the abort completion will complete the io
2596 	 * shortly. Avoids a synchronous wait while the abort finishes.
2597 	 */
2598 	return BLK_EH_RESET_TIMER;
2599 }
2600 
2601 static int
2602 nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2603 		struct nvme_fc_fcp_op *op)
2604 {
2605 	struct nvmefc_fcp_req *freq = &op->fcp_req;
2606 	int ret;
2607 
2608 	freq->sg_cnt = 0;
2609 
2610 	if (!blk_rq_nr_phys_segments(rq))
2611 		return 0;
2612 
2613 	freq->sg_table.sgl = freq->first_sgl;
2614 	ret = sg_alloc_table_chained(&freq->sg_table,
2615 			blk_rq_nr_phys_segments(rq), freq->sg_table.sgl,
2616 			NVME_INLINE_SG_CNT);
2617 	if (ret)
2618 		return -ENOMEM;
2619 
2620 	op->nents = blk_rq_map_sg(rq->q, rq, freq->sg_table.sgl);
2621 	WARN_ON(op->nents > blk_rq_nr_phys_segments(rq));
2622 	freq->sg_cnt = fc_dma_map_sg(ctrl->lport->dev, freq->sg_table.sgl,
2623 				op->nents, rq_dma_dir(rq));
2624 	if (unlikely(freq->sg_cnt <= 0)) {
2625 		sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2626 		freq->sg_cnt = 0;
2627 		return -EFAULT;
2628 	}
2629 
2630 	/*
2631 	 * TODO: blk_integrity_rq(rq)  for DIF
2632 	 */
2633 	return 0;
2634 }
2635 
2636 static void
2637 nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2638 		struct nvme_fc_fcp_op *op)
2639 {
2640 	struct nvmefc_fcp_req *freq = &op->fcp_req;
2641 
2642 	if (!freq->sg_cnt)
2643 		return;
2644 
2645 	fc_dma_unmap_sg(ctrl->lport->dev, freq->sg_table.sgl, op->nents,
2646 			rq_dma_dir(rq));
2647 
2648 	sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2649 
2650 	freq->sg_cnt = 0;
2651 }
2652 
2653 /*
2654  * In FC, the queue is a logical thing. At transport connect, the target
2655  * creates its "queue" and returns a handle that is to be given to the
2656  * target whenever it posts something to the corresponding SQ.  When an
2657  * SQE is sent on a SQ, FC effectively considers the SQE, or rather the
2658  * command contained within the SQE, an io, and assigns a FC exchange
2659  * to it. The SQE and the associated SQ handle are sent in the initial
2660  * CMD IU sents on the exchange. All transfers relative to the io occur
2661  * as part of the exchange.  The CQE is the last thing for the io,
2662  * which is transferred (explicitly or implicitly) with the RSP IU
2663  * sent on the exchange. After the CQE is received, the FC exchange is
2664  * terminaed and the Exchange may be used on a different io.
2665  *
2666  * The transport to LLDD api has the transport making a request for a
2667  * new fcp io request to the LLDD. The LLDD then allocates a FC exchange
2668  * resource and transfers the command. The LLDD will then process all
2669  * steps to complete the io. Upon completion, the transport done routine
2670  * is called.
2671  *
2672  * So - while the operation is outstanding to the LLDD, there is a link
2673  * level FC exchange resource that is also outstanding. This must be
2674  * considered in all cleanup operations.
2675  */
2676 static blk_status_t
2677 nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
2678 	struct nvme_fc_fcp_op *op, u32 data_len,
2679 	enum nvmefc_fcp_datadir	io_dir)
2680 {
2681 	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2682 	struct nvme_command *sqe = &cmdiu->sqe;
2683 	int ret, opstate;
2684 
2685 	/*
2686 	 * before attempting to send the io, check to see if we believe
2687 	 * the target device is present
2688 	 */
2689 	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
2690 		return BLK_STS_RESOURCE;
2691 
2692 	if (!nvme_fc_ctrl_get(ctrl))
2693 		return BLK_STS_IOERR;
2694 
2695 	/* format the FC-NVME CMD IU and fcp_req */
2696 	cmdiu->connection_id = cpu_to_be64(queue->connection_id);
2697 	cmdiu->data_len = cpu_to_be32(data_len);
2698 	switch (io_dir) {
2699 	case NVMEFC_FCP_WRITE:
2700 		cmdiu->flags = FCNVME_CMD_FLAGS_WRITE;
2701 		break;
2702 	case NVMEFC_FCP_READ:
2703 		cmdiu->flags = FCNVME_CMD_FLAGS_READ;
2704 		break;
2705 	case NVMEFC_FCP_NODATA:
2706 		cmdiu->flags = 0;
2707 		break;
2708 	}
2709 	op->fcp_req.payload_length = data_len;
2710 	op->fcp_req.io_dir = io_dir;
2711 	op->fcp_req.transferred_length = 0;
2712 	op->fcp_req.rcv_rsplen = 0;
2713 	op->fcp_req.status = NVME_SC_SUCCESS;
2714 	op->fcp_req.sqid = cpu_to_le16(queue->qnum);
2715 
2716 	/*
2717 	 * validate per fabric rules, set fields mandated by fabric spec
2718 	 * as well as those by FC-NVME spec.
2719 	 */
2720 	WARN_ON_ONCE(sqe->common.metadata);
2721 	sqe->common.flags |= NVME_CMD_SGL_METABUF;
2722 
2723 	/*
2724 	 * format SQE DPTR field per FC-NVME rules:
2725 	 *    type=0x5     Transport SGL Data Block Descriptor
2726 	 *    subtype=0xA  Transport-specific value
2727 	 *    address=0
2728 	 *    length=length of the data series
2729 	 */
2730 	sqe->rw.dptr.sgl.type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2731 					NVME_SGL_FMT_TRANSPORT_A;
2732 	sqe->rw.dptr.sgl.length = cpu_to_le32(data_len);
2733 	sqe->rw.dptr.sgl.addr = 0;
2734 
2735 	if (!(op->flags & FCOP_FLAGS_AEN)) {
2736 		ret = nvme_fc_map_data(ctrl, op->rq, op);
2737 		if (ret < 0) {
2738 			nvme_cleanup_cmd(op->rq);
2739 			nvme_fc_ctrl_put(ctrl);
2740 			if (ret == -ENOMEM || ret == -EAGAIN)
2741 				return BLK_STS_RESOURCE;
2742 			return BLK_STS_IOERR;
2743 		}
2744 	}
2745 
2746 	fc_dma_sync_single_for_device(ctrl->lport->dev, op->fcp_req.cmddma,
2747 				  sizeof(op->cmd_iu), DMA_TO_DEVICE);
2748 
2749 	atomic_set(&op->state, FCPOP_STATE_ACTIVE);
2750 
2751 	if (!(op->flags & FCOP_FLAGS_AEN))
2752 		nvme_start_request(op->rq);
2753 
2754 	cmdiu->csn = cpu_to_be32(atomic_inc_return(&queue->csn));
2755 	ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport,
2756 					&ctrl->rport->remoteport,
2757 					queue->lldd_handle, &op->fcp_req);
2758 
2759 	if (ret) {
2760 		/*
2761 		 * If the lld fails to send the command is there an issue with
2762 		 * the csn value?  If the command that fails is the Connect,
2763 		 * no - as the connection won't be live.  If it is a command
2764 		 * post-connect, it's possible a gap in csn may be created.
2765 		 * Does this matter?  As Linux initiators don't send fused
2766 		 * commands, no.  The gap would exist, but as there's nothing
2767 		 * that depends on csn order to be delivered on the target
2768 		 * side, it shouldn't hurt.  It would be difficult for a
2769 		 * target to even detect the csn gap as it has no idea when the
2770 		 * cmd with the csn was supposed to arrive.
2771 		 */
2772 		opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
2773 		__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2774 
2775 		if (!(op->flags & FCOP_FLAGS_AEN)) {
2776 			nvme_fc_unmap_data(ctrl, op->rq, op);
2777 			nvme_cleanup_cmd(op->rq);
2778 		}
2779 
2780 		nvme_fc_ctrl_put(ctrl);
2781 
2782 		if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE &&
2783 				ret != -EBUSY)
2784 			return BLK_STS_IOERR;
2785 
2786 		return BLK_STS_RESOURCE;
2787 	}
2788 
2789 	return BLK_STS_OK;
2790 }
2791 
2792 static blk_status_t
2793 nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx,
2794 			const struct blk_mq_queue_data *bd)
2795 {
2796 	struct nvme_ns *ns = hctx->queue->queuedata;
2797 	struct nvme_fc_queue *queue = hctx->driver_data;
2798 	struct nvme_fc_ctrl *ctrl = queue->ctrl;
2799 	struct request *rq = bd->rq;
2800 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2801 	enum nvmefc_fcp_datadir	io_dir;
2802 	bool queue_ready = test_bit(NVME_FC_Q_LIVE, &queue->flags);
2803 	u32 data_len;
2804 	blk_status_t ret;
2805 
2806 	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE ||
2807 	    !nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2808 		return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
2809 
2810 	ret = nvme_setup_cmd(ns, rq);
2811 	if (ret)
2812 		return ret;
2813 
2814 	/*
2815 	 * nvme core doesn't quite treat the rq opaquely. Commands such
2816 	 * as WRITE ZEROES will return a non-zero rq payload_bytes yet
2817 	 * there is no actual payload to be transferred.
2818 	 * To get it right, key data transmission on there being 1 or
2819 	 * more physical segments in the sg list. If there is no
2820 	 * physical segments, there is no payload.
2821 	 */
2822 	if (blk_rq_nr_phys_segments(rq)) {
2823 		data_len = blk_rq_payload_bytes(rq);
2824 		io_dir = ((rq_data_dir(rq) == WRITE) ?
2825 					NVMEFC_FCP_WRITE : NVMEFC_FCP_READ);
2826 	} else {
2827 		data_len = 0;
2828 		io_dir = NVMEFC_FCP_NODATA;
2829 	}
2830 
2831 
2832 	return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir);
2833 }
2834 
2835 static void
2836 nvme_fc_submit_async_event(struct nvme_ctrl *arg)
2837 {
2838 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(arg);
2839 	struct nvme_fc_fcp_op *aen_op;
2840 	blk_status_t ret;
2841 
2842 	if (test_bit(FCCTRL_TERMIO, &ctrl->flags))
2843 		return;
2844 
2845 	aen_op = &ctrl->aen_ops[0];
2846 
2847 	ret = nvme_fc_start_fcp_op(ctrl, aen_op->queue, aen_op, 0,
2848 					NVMEFC_FCP_NODATA);
2849 	if (ret)
2850 		dev_err(ctrl->ctrl.device,
2851 			"failed async event work\n");
2852 }
2853 
2854 static void
2855 nvme_fc_complete_rq(struct request *rq)
2856 {
2857 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2858 	struct nvme_fc_ctrl *ctrl = op->ctrl;
2859 
2860 	atomic_set(&op->state, FCPOP_STATE_IDLE);
2861 	op->flags &= ~FCOP_FLAGS_TERMIO;
2862 
2863 	nvme_fc_unmap_data(ctrl, rq, op);
2864 	nvme_complete_rq(rq);
2865 	nvme_fc_ctrl_put(ctrl);
2866 }
2867 
2868 static void nvme_fc_map_queues(struct blk_mq_tag_set *set)
2869 {
2870 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data);
2871 	int i;
2872 
2873 	for (i = 0; i < set->nr_maps; i++) {
2874 		struct blk_mq_queue_map *map = &set->map[i];
2875 
2876 		if (!map->nr_queues) {
2877 			WARN_ON(i == HCTX_TYPE_DEFAULT);
2878 			continue;
2879 		}
2880 
2881 		/* Call LLDD map queue functionality if defined */
2882 		if (ctrl->lport->ops->map_queues)
2883 			ctrl->lport->ops->map_queues(&ctrl->lport->localport,
2884 						     map);
2885 		else
2886 			blk_mq_map_queues(map);
2887 	}
2888 }
2889 
2890 static const struct blk_mq_ops nvme_fc_mq_ops = {
2891 	.queue_rq	= nvme_fc_queue_rq,
2892 	.complete	= nvme_fc_complete_rq,
2893 	.init_request	= nvme_fc_init_request,
2894 	.exit_request	= nvme_fc_exit_request,
2895 	.init_hctx	= nvme_fc_init_hctx,
2896 	.timeout	= nvme_fc_timeout,
2897 	.map_queues	= nvme_fc_map_queues,
2898 };
2899 
2900 static int
2901 nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
2902 {
2903 	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2904 	unsigned int nr_io_queues;
2905 	int ret;
2906 
2907 	nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2908 				ctrl->lport->ops->max_hw_queues);
2909 	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2910 	if (ret) {
2911 		dev_info(ctrl->ctrl.device,
2912 			"set_queue_count failed: %d\n", ret);
2913 		return ret;
2914 	}
2915 
2916 	ctrl->ctrl.queue_count = nr_io_queues + 1;
2917 	if (!nr_io_queues)
2918 		return 0;
2919 
2920 	nvme_fc_init_io_queues(ctrl);
2921 
2922 	ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,
2923 			&nvme_fc_mq_ops, 1,
2924 			struct_size_t(struct nvme_fcp_op_w_sgl, priv,
2925 				      ctrl->lport->ops->fcprqst_priv_sz));
2926 	if (ret)
2927 		return ret;
2928 
2929 	ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2930 	if (ret)
2931 		goto out_cleanup_tagset;
2932 
2933 	ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2934 	if (ret)
2935 		goto out_delete_hw_queues;
2936 
2937 	ctrl->ioq_live = true;
2938 
2939 	return 0;
2940 
2941 out_delete_hw_queues:
2942 	nvme_fc_delete_hw_io_queues(ctrl);
2943 out_cleanup_tagset:
2944 	nvme_remove_io_tag_set(&ctrl->ctrl);
2945 	nvme_fc_free_io_queues(ctrl);
2946 
2947 	/* force put free routine to ignore io queues */
2948 	ctrl->ctrl.tagset = NULL;
2949 
2950 	return ret;
2951 }
2952 
2953 static int
2954 nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)
2955 {
2956 	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2957 	u32 prior_ioq_cnt = ctrl->ctrl.queue_count - 1;
2958 	unsigned int nr_io_queues;
2959 	int ret;
2960 
2961 	nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2962 				ctrl->lport->ops->max_hw_queues);
2963 	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2964 	if (ret) {
2965 		dev_info(ctrl->ctrl.device,
2966 			"set_queue_count failed: %d\n", ret);
2967 		return ret;
2968 	}
2969 
2970 	if (!nr_io_queues && prior_ioq_cnt) {
2971 		dev_info(ctrl->ctrl.device,
2972 			"Fail Reconnect: At least 1 io queue "
2973 			"required (was %d)\n", prior_ioq_cnt);
2974 		return -ENOSPC;
2975 	}
2976 
2977 	ctrl->ctrl.queue_count = nr_io_queues + 1;
2978 	/* check for io queues existing */
2979 	if (ctrl->ctrl.queue_count == 1)
2980 		return 0;
2981 
2982 	if (prior_ioq_cnt != nr_io_queues) {
2983 		dev_info(ctrl->ctrl.device,
2984 			"reconnect: revising io queue count from %d to %d\n",
2985 			prior_ioq_cnt, nr_io_queues);
2986 		blk_mq_update_nr_hw_queues(&ctrl->tag_set, nr_io_queues);
2987 	}
2988 
2989 	ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2990 	if (ret)
2991 		goto out_free_io_queues;
2992 
2993 	ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2994 	if (ret)
2995 		goto out_delete_hw_queues;
2996 
2997 	return 0;
2998 
2999 out_delete_hw_queues:
3000 	nvme_fc_delete_hw_io_queues(ctrl);
3001 out_free_io_queues:
3002 	nvme_fc_free_io_queues(ctrl);
3003 	return ret;
3004 }
3005 
3006 static void
3007 nvme_fc_rport_active_on_lport(struct nvme_fc_rport *rport)
3008 {
3009 	struct nvme_fc_lport *lport = rport->lport;
3010 
3011 	atomic_inc(&lport->act_rport_cnt);
3012 }
3013 
3014 static void
3015 nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport *rport)
3016 {
3017 	struct nvme_fc_lport *lport = rport->lport;
3018 	u32 cnt;
3019 
3020 	cnt = atomic_dec_return(&lport->act_rport_cnt);
3021 	if (cnt == 0 && lport->localport.port_state == FC_OBJSTATE_DELETED)
3022 		lport->ops->localport_delete(&lport->localport);
3023 }
3024 
3025 static int
3026 nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl)
3027 {
3028 	struct nvme_fc_rport *rport = ctrl->rport;
3029 	u32 cnt;
3030 
3031 	if (test_and_set_bit(ASSOC_ACTIVE, &ctrl->flags))
3032 		return 1;
3033 
3034 	cnt = atomic_inc_return(&rport->act_ctrl_cnt);
3035 	if (cnt == 1)
3036 		nvme_fc_rport_active_on_lport(rport);
3037 
3038 	return 0;
3039 }
3040 
3041 static int
3042 nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl)
3043 {
3044 	struct nvme_fc_rport *rport = ctrl->rport;
3045 	struct nvme_fc_lport *lport = rport->lport;
3046 	u32 cnt;
3047 
3048 	/* clearing of ctrl->flags ASSOC_ACTIVE bit is in association delete */
3049 
3050 	cnt = atomic_dec_return(&rport->act_ctrl_cnt);
3051 	if (cnt == 0) {
3052 		if (rport->remoteport.port_state == FC_OBJSTATE_DELETED)
3053 			lport->ops->remoteport_delete(&rport->remoteport);
3054 		nvme_fc_rport_inactive_on_lport(rport);
3055 	}
3056 
3057 	return 0;
3058 }
3059 
3060 /*
3061  * This routine restarts the controller on the host side, and
3062  * on the link side, recreates the controller association.
3063  */
3064 static int
3065 nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
3066 {
3067 	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
3068 	struct nvmefc_ls_rcv_op *disls = NULL;
3069 	unsigned long flags;
3070 	int ret;
3071 	bool changed;
3072 
3073 	++ctrl->ctrl.nr_reconnects;
3074 
3075 	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3076 		return -ENODEV;
3077 
3078 	if (nvme_fc_ctlr_active_on_rport(ctrl))
3079 		return -ENOTUNIQ;
3080 
3081 	dev_info(ctrl->ctrl.device,
3082 		"NVME-FC{%d}: create association : host wwpn 0x%016llx "
3083 		" rport wwpn 0x%016llx: NQN \"%s\"\n",
3084 		ctrl->cnum, ctrl->lport->localport.port_name,
3085 		ctrl->rport->remoteport.port_name, ctrl->ctrl.opts->subsysnqn);
3086 
3087 	clear_bit(ASSOC_FAILED, &ctrl->flags);
3088 
3089 	/*
3090 	 * Create the admin queue
3091 	 */
3092 
3093 	ret = __nvme_fc_create_hw_queue(ctrl, &ctrl->queues[0], 0,
3094 				NVME_AQ_DEPTH);
3095 	if (ret)
3096 		goto out_free_queue;
3097 
3098 	ret = nvme_fc_connect_admin_queue(ctrl, &ctrl->queues[0],
3099 				NVME_AQ_DEPTH, (NVME_AQ_DEPTH / 4));
3100 	if (ret)
3101 		goto out_delete_hw_queue;
3102 
3103 	ret = nvmf_connect_admin_queue(&ctrl->ctrl);
3104 	if (ret)
3105 		goto out_disconnect_admin_queue;
3106 
3107 	set_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
3108 
3109 	/*
3110 	 * Check controller capabilities
3111 	 *
3112 	 * todo:- add code to check if ctrl attributes changed from
3113 	 * prior connection values
3114 	 */
3115 
3116 	ret = nvme_enable_ctrl(&ctrl->ctrl);
3117 	if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3118 		ret = -EIO;
3119 	if (ret)
3120 		goto out_disconnect_admin_queue;
3121 
3122 	ctrl->ctrl.max_segments = ctrl->lport->ops->max_sgl_segments;
3123 	ctrl->ctrl.max_hw_sectors = ctrl->ctrl.max_segments <<
3124 						(ilog2(SZ_4K) - 9);
3125 
3126 	nvme_unquiesce_admin_queue(&ctrl->ctrl);
3127 
3128 	ret = nvme_init_ctrl_finish(&ctrl->ctrl, false);
3129 	if (ret)
3130 		goto out_disconnect_admin_queue;
3131 	if (test_bit(ASSOC_FAILED, &ctrl->flags)) {
3132 		ret = -EIO;
3133 		goto out_stop_keep_alive;
3134 	}
3135 	/* sanity checks */
3136 
3137 	/* FC-NVME does not have other data in the capsule */
3138 	if (ctrl->ctrl.icdoff) {
3139 		dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n",
3140 				ctrl->ctrl.icdoff);
3141 		ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
3142 		goto out_stop_keep_alive;
3143 	}
3144 
3145 	/* FC-NVME supports normal SGL Data Block Descriptors */
3146 	if (!nvme_ctrl_sgl_supported(&ctrl->ctrl)) {
3147 		dev_err(ctrl->ctrl.device,
3148 			"Mandatory sgls are not supported!\n");
3149 		ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
3150 		goto out_stop_keep_alive;
3151 	}
3152 
3153 	if (opts->queue_size > ctrl->ctrl.maxcmd) {
3154 		/* warn if maxcmd is lower than queue_size */
3155 		dev_warn(ctrl->ctrl.device,
3156 			"queue_size %zu > ctrl maxcmd %u, reducing "
3157 			"to maxcmd\n",
3158 			opts->queue_size, ctrl->ctrl.maxcmd);
3159 		opts->queue_size = ctrl->ctrl.maxcmd;
3160 		ctrl->ctrl.sqsize = opts->queue_size - 1;
3161 	}
3162 
3163 	ret = nvme_fc_init_aen_ops(ctrl);
3164 	if (ret)
3165 		goto out_term_aen_ops;
3166 
3167 	/*
3168 	 * Create the io queues
3169 	 */
3170 
3171 	if (ctrl->ctrl.queue_count > 1) {
3172 		if (!ctrl->ioq_live)
3173 			ret = nvme_fc_create_io_queues(ctrl);
3174 		else
3175 			ret = nvme_fc_recreate_io_queues(ctrl);
3176 	}
3177 	if (ret)
3178 		goto out_term_aen_ops;
3179 
3180 	spin_lock_irqsave(&ctrl->lock, flags);
3181 	if (!test_bit(ASSOC_FAILED, &ctrl->flags))
3182 		changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
3183 	else
3184 		ret = -EIO;
3185 	spin_unlock_irqrestore(&ctrl->lock, flags);
3186 
3187 	if (ret)
3188 		goto out_term_aen_ops;
3189 
3190 	ctrl->ctrl.nr_reconnects = 0;
3191 
3192 	if (changed)
3193 		nvme_start_ctrl(&ctrl->ctrl);
3194 
3195 	return 0;	/* Success */
3196 
3197 out_term_aen_ops:
3198 	nvme_fc_term_aen_ops(ctrl);
3199 out_stop_keep_alive:
3200 	nvme_stop_keep_alive(&ctrl->ctrl);
3201 out_disconnect_admin_queue:
3202 	dev_warn(ctrl->ctrl.device,
3203 		"NVME-FC{%d}: create_assoc failed, assoc_id %llx ret %d\n",
3204 		ctrl->cnum, ctrl->association_id, ret);
3205 	/* send a Disconnect(association) LS to fc-nvme target */
3206 	nvme_fc_xmt_disconnect_assoc(ctrl);
3207 	spin_lock_irqsave(&ctrl->lock, flags);
3208 	ctrl->association_id = 0;
3209 	disls = ctrl->rcv_disconn;
3210 	ctrl->rcv_disconn = NULL;
3211 	spin_unlock_irqrestore(&ctrl->lock, flags);
3212 	if (disls)
3213 		nvme_fc_xmt_ls_rsp(disls);
3214 out_delete_hw_queue:
3215 	__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3216 out_free_queue:
3217 	nvme_fc_free_queue(&ctrl->queues[0]);
3218 	clear_bit(ASSOC_ACTIVE, &ctrl->flags);
3219 	nvme_fc_ctlr_inactive_on_rport(ctrl);
3220 
3221 	return ret;
3222 }
3223 
3224 
3225 /*
3226  * This routine stops operation of the controller on the host side.
3227  * On the host os stack side: Admin and IO queues are stopped,
3228  *   outstanding ios on them terminated via FC ABTS.
3229  * On the link side: the association is terminated.
3230  */
3231 static void
3232 nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl)
3233 {
3234 	struct nvmefc_ls_rcv_op *disls = NULL;
3235 	unsigned long flags;
3236 
3237 	if (!test_and_clear_bit(ASSOC_ACTIVE, &ctrl->flags))
3238 		return;
3239 
3240 	spin_lock_irqsave(&ctrl->lock, flags);
3241 	set_bit(FCCTRL_TERMIO, &ctrl->flags);
3242 	ctrl->iocnt = 0;
3243 	spin_unlock_irqrestore(&ctrl->lock, flags);
3244 
3245 	__nvme_fc_abort_outstanding_ios(ctrl, false);
3246 
3247 	/* kill the aens as they are a separate path */
3248 	nvme_fc_abort_aen_ops(ctrl);
3249 
3250 	/* wait for all io that had to be aborted */
3251 	spin_lock_irq(&ctrl->lock);
3252 	wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock);
3253 	clear_bit(FCCTRL_TERMIO, &ctrl->flags);
3254 	spin_unlock_irq(&ctrl->lock);
3255 
3256 	nvme_fc_term_aen_ops(ctrl);
3257 
3258 	/*
3259 	 * send a Disconnect(association) LS to fc-nvme target
3260 	 * Note: could have been sent at top of process, but
3261 	 * cleaner on link traffic if after the aborts complete.
3262 	 * Note: if association doesn't exist, association_id will be 0
3263 	 */
3264 	if (ctrl->association_id)
3265 		nvme_fc_xmt_disconnect_assoc(ctrl);
3266 
3267 	spin_lock_irqsave(&ctrl->lock, flags);
3268 	ctrl->association_id = 0;
3269 	disls = ctrl->rcv_disconn;
3270 	ctrl->rcv_disconn = NULL;
3271 	spin_unlock_irqrestore(&ctrl->lock, flags);
3272 	if (disls)
3273 		/*
3274 		 * if a Disconnect Request was waiting for a response, send
3275 		 * now that all ABTS's have been issued (and are complete).
3276 		 */
3277 		nvme_fc_xmt_ls_rsp(disls);
3278 
3279 	if (ctrl->ctrl.tagset) {
3280 		nvme_fc_delete_hw_io_queues(ctrl);
3281 		nvme_fc_free_io_queues(ctrl);
3282 	}
3283 
3284 	__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3285 	nvme_fc_free_queue(&ctrl->queues[0]);
3286 
3287 	/* re-enable the admin_q so anything new can fast fail */
3288 	nvme_unquiesce_admin_queue(&ctrl->ctrl);
3289 
3290 	/* resume the io queues so that things will fast fail */
3291 	nvme_unquiesce_io_queues(&ctrl->ctrl);
3292 
3293 	nvme_fc_ctlr_inactive_on_rport(ctrl);
3294 }
3295 
3296 static void
3297 nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
3298 {
3299 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
3300 
3301 	cancel_work_sync(&ctrl->ioerr_work);
3302 	cancel_delayed_work_sync(&ctrl->connect_work);
3303 	/*
3304 	 * kill the association on the link side.  this will block
3305 	 * waiting for io to terminate
3306 	 */
3307 	nvme_fc_delete_association(ctrl);
3308 }
3309 
3310 static void
3311 nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
3312 {
3313 	struct nvme_fc_rport *rport = ctrl->rport;
3314 	struct nvme_fc_remote_port *portptr = &rport->remoteport;
3315 	unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ;
3316 	bool recon = true;
3317 
3318 	if (nvme_ctrl_state(&ctrl->ctrl) != NVME_CTRL_CONNECTING)
3319 		return;
3320 
3321 	if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3322 		dev_info(ctrl->ctrl.device,
3323 			"NVME-FC{%d}: reset: Reconnect attempt failed (%d)\n",
3324 			ctrl->cnum, status);
3325 	} else if (time_after_eq(jiffies, rport->dev_loss_end))
3326 		recon = false;
3327 
3328 	if (recon && nvmf_should_reconnect(&ctrl->ctrl, status)) {
3329 		if (portptr->port_state == FC_OBJSTATE_ONLINE)
3330 			dev_info(ctrl->ctrl.device,
3331 				"NVME-FC{%d}: Reconnect attempt in %ld "
3332 				"seconds\n",
3333 				ctrl->cnum, recon_delay / HZ);
3334 		else if (time_after(jiffies + recon_delay, rport->dev_loss_end))
3335 			recon_delay = rport->dev_loss_end - jiffies;
3336 
3337 		queue_delayed_work(nvme_wq, &ctrl->connect_work, recon_delay);
3338 	} else {
3339 		if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3340 			if (status > 0 && (status & NVME_STATUS_DNR))
3341 				dev_warn(ctrl->ctrl.device,
3342 					 "NVME-FC{%d}: reconnect failure\n",
3343 					 ctrl->cnum);
3344 			else
3345 				dev_warn(ctrl->ctrl.device,
3346 					 "NVME-FC{%d}: Max reconnect attempts "
3347 					 "(%d) reached.\n",
3348 					 ctrl->cnum, ctrl->ctrl.nr_reconnects);
3349 		} else
3350 			dev_warn(ctrl->ctrl.device,
3351 				"NVME-FC{%d}: dev_loss_tmo (%d) expired "
3352 				"while waiting for remoteport connectivity.\n",
3353 				ctrl->cnum, min_t(int, portptr->dev_loss_tmo,
3354 					(ctrl->ctrl.opts->max_reconnects *
3355 					 ctrl->ctrl.opts->reconnect_delay)));
3356 		WARN_ON(nvme_delete_ctrl(&ctrl->ctrl));
3357 	}
3358 }
3359 
3360 static void
3361 nvme_fc_reset_ctrl_work(struct work_struct *work)
3362 {
3363 	struct nvme_fc_ctrl *ctrl =
3364 		container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
3365 
3366 	nvme_stop_ctrl(&ctrl->ctrl);
3367 
3368 	/* will block will waiting for io to terminate */
3369 	nvme_fc_delete_association(ctrl);
3370 
3371 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
3372 		dev_err(ctrl->ctrl.device,
3373 			"NVME-FC{%d}: error_recovery: Couldn't change state "
3374 			"to CONNECTING\n", ctrl->cnum);
3375 
3376 	if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
3377 		if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3378 			dev_err(ctrl->ctrl.device,
3379 				"NVME-FC{%d}: failed to schedule connect "
3380 				"after reset\n", ctrl->cnum);
3381 		} else {
3382 			flush_delayed_work(&ctrl->connect_work);
3383 		}
3384 	} else {
3385 		nvme_fc_reconnect_or_delete(ctrl, -ENOTCONN);
3386 	}
3387 }
3388 
3389 
3390 static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
3391 	.name			= "fc",
3392 	.module			= THIS_MODULE,
3393 	.flags			= NVME_F_FABRICS,
3394 	.reg_read32		= nvmf_reg_read32,
3395 	.reg_read64		= nvmf_reg_read64,
3396 	.reg_write32		= nvmf_reg_write32,
3397 	.subsystem_reset	= nvmf_subsystem_reset,
3398 	.free_ctrl		= nvme_fc_free_ctrl,
3399 	.submit_async_event	= nvme_fc_submit_async_event,
3400 	.delete_ctrl		= nvme_fc_delete_ctrl,
3401 	.get_address		= nvmf_get_address,
3402 };
3403 
3404 static void
3405 nvme_fc_connect_ctrl_work(struct work_struct *work)
3406 {
3407 	int ret;
3408 
3409 	struct nvme_fc_ctrl *ctrl =
3410 			container_of(to_delayed_work(work),
3411 				struct nvme_fc_ctrl, connect_work);
3412 
3413 	ret = nvme_fc_create_association(ctrl);
3414 	if (ret)
3415 		nvme_fc_reconnect_or_delete(ctrl, ret);
3416 	else
3417 		dev_info(ctrl->ctrl.device,
3418 			"NVME-FC{%d}: controller connect complete\n",
3419 			ctrl->cnum);
3420 }
3421 
3422 
3423 static const struct blk_mq_ops nvme_fc_admin_mq_ops = {
3424 	.queue_rq	= nvme_fc_queue_rq,
3425 	.complete	= nvme_fc_complete_rq,
3426 	.init_request	= nvme_fc_init_request,
3427 	.exit_request	= nvme_fc_exit_request,
3428 	.init_hctx	= nvme_fc_init_admin_hctx,
3429 	.timeout	= nvme_fc_timeout,
3430 };
3431 
3432 
3433 /*
3434  * Fails a controller request if it matches an existing controller
3435  * (association) with the same tuple:
3436  * <Host NQN, Host ID, local FC port, remote FC port, SUBSYS NQN>
3437  *
3438  * The ports don't need to be compared as they are intrinsically
3439  * already matched by the port pointers supplied.
3440  */
3441 static bool
3442 nvme_fc_existing_controller(struct nvme_fc_rport *rport,
3443 		struct nvmf_ctrl_options *opts)
3444 {
3445 	struct nvme_fc_ctrl *ctrl;
3446 	unsigned long flags;
3447 	bool found = false;
3448 
3449 	spin_lock_irqsave(&rport->lock, flags);
3450 	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3451 		found = nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts);
3452 		if (found)
3453 			break;
3454 	}
3455 	spin_unlock_irqrestore(&rport->lock, flags);
3456 
3457 	return found;
3458 }
3459 
3460 static struct nvme_fc_ctrl *
3461 nvme_fc_alloc_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3462 	struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3463 {
3464 	struct nvme_fc_ctrl *ctrl;
3465 	int ret, idx, ctrl_loss_tmo;
3466 
3467 	if (!(rport->remoteport.port_role &
3468 	    (FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET))) {
3469 		ret = -EBADR;
3470 		goto out_fail;
3471 	}
3472 
3473 	if (!opts->duplicate_connect &&
3474 	    nvme_fc_existing_controller(rport, opts)) {
3475 		ret = -EALREADY;
3476 		goto out_fail;
3477 	}
3478 
3479 	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
3480 	if (!ctrl) {
3481 		ret = -ENOMEM;
3482 		goto out_fail;
3483 	}
3484 
3485 	idx = ida_alloc(&nvme_fc_ctrl_cnt, GFP_KERNEL);
3486 	if (idx < 0) {
3487 		ret = -ENOSPC;
3488 		goto out_free_ctrl;
3489 	}
3490 
3491 	/*
3492 	 * if ctrl_loss_tmo is being enforced and the default reconnect delay
3493 	 * is being used, change to a shorter reconnect delay for FC.
3494 	 */
3495 	if (opts->max_reconnects != -1 &&
3496 	    opts->reconnect_delay == NVMF_DEF_RECONNECT_DELAY &&
3497 	    opts->reconnect_delay > NVME_FC_DEFAULT_RECONNECT_TMO) {
3498 		ctrl_loss_tmo = opts->max_reconnects * opts->reconnect_delay;
3499 		opts->reconnect_delay = NVME_FC_DEFAULT_RECONNECT_TMO;
3500 		opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3501 						opts->reconnect_delay);
3502 	}
3503 
3504 	ctrl->ctrl.opts = opts;
3505 	ctrl->ctrl.nr_reconnects = 0;
3506 	INIT_LIST_HEAD(&ctrl->ctrl_list);
3507 	ctrl->lport = lport;
3508 	ctrl->rport = rport;
3509 	ctrl->dev = lport->dev;
3510 	ctrl->cnum = idx;
3511 	ctrl->ioq_live = false;
3512 	init_waitqueue_head(&ctrl->ioabort_wait);
3513 
3514 	get_device(ctrl->dev);
3515 	kref_init(&ctrl->ref);
3516 
3517 	INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
3518 	INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
3519 	INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work);
3520 	spin_lock_init(&ctrl->lock);
3521 
3522 	/* io queue count */
3523 	ctrl->ctrl.queue_count = min_t(unsigned int,
3524 				opts->nr_io_queues,
3525 				lport->ops->max_hw_queues);
3526 	ctrl->ctrl.queue_count++;	/* +1 for admin queue */
3527 
3528 	ctrl->ctrl.sqsize = opts->queue_size - 1;
3529 	ctrl->ctrl.kato = opts->kato;
3530 	ctrl->ctrl.cntlid = 0xffff;
3531 
3532 	ret = -ENOMEM;
3533 	ctrl->queues = kcalloc(ctrl->ctrl.queue_count,
3534 				sizeof(struct nvme_fc_queue), GFP_KERNEL);
3535 	if (!ctrl->queues)
3536 		goto out_free_ida;
3537 
3538 	nvme_fc_init_queue(ctrl, 0);
3539 
3540 	/*
3541 	 * Would have been nice to init io queues tag set as well.
3542 	 * However, we require interaction from the controller
3543 	 * for max io queue count before we can do so.
3544 	 * Defer this to the connect path.
3545 	 */
3546 
3547 	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0);
3548 	if (ret)
3549 		goto out_free_queues;
3550 	if (lport->dev)
3551 		ctrl->ctrl.numa_node = dev_to_node(lport->dev);
3552 
3553 	return ctrl;
3554 
3555 out_free_queues:
3556 	kfree(ctrl->queues);
3557 out_free_ida:
3558 	put_device(ctrl->dev);
3559 	ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
3560 out_free_ctrl:
3561 	kfree(ctrl);
3562 out_fail:
3563 	/* exit via here doesn't follow ctlr ref points */
3564 	return ERR_PTR(ret);
3565 }
3566 
3567 static struct nvme_ctrl *
3568 nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3569 	struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3570 {
3571 	struct nvme_fc_ctrl *ctrl;
3572 	unsigned long flags;
3573 	int ret;
3574 
3575 	ctrl = nvme_fc_alloc_ctrl(dev, opts, lport, rport);
3576 	if (IS_ERR(ctrl))
3577 		return ERR_CAST(ctrl);
3578 
3579 	ret = nvme_add_ctrl(&ctrl->ctrl);
3580 	if (ret)
3581 		goto out_put_ctrl;
3582 
3583 	ret = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,
3584 			&nvme_fc_admin_mq_ops,
3585 			struct_size_t(struct nvme_fcp_op_w_sgl, priv,
3586 				      ctrl->lport->ops->fcprqst_priv_sz));
3587 	if (ret)
3588 		goto fail_ctrl;
3589 
3590 	spin_lock_irqsave(&rport->lock, flags);
3591 	list_add_tail(&ctrl->ctrl_list, &rport->ctrl_list);
3592 	spin_unlock_irqrestore(&rport->lock, flags);
3593 
3594 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
3595 		dev_err(ctrl->ctrl.device,
3596 			"NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum);
3597 		goto fail_ctrl;
3598 	}
3599 
3600 	if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3601 		dev_err(ctrl->ctrl.device,
3602 			"NVME-FC{%d}: failed to schedule initial connect\n",
3603 			ctrl->cnum);
3604 		goto fail_ctrl;
3605 	}
3606 
3607 	flush_delayed_work(&ctrl->connect_work);
3608 
3609 	dev_info(ctrl->ctrl.device,
3610 		"NVME-FC{%d}: new ctrl: NQN \"%s\", hostnqn: %s\n",
3611 		ctrl->cnum, nvmf_ctrl_subsysnqn(&ctrl->ctrl), opts->host->nqn);
3612 
3613 	return &ctrl->ctrl;
3614 
3615 fail_ctrl:
3616 	nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING);
3617 	cancel_work_sync(&ctrl->ioerr_work);
3618 	cancel_work_sync(&ctrl->ctrl.reset_work);
3619 	cancel_delayed_work_sync(&ctrl->connect_work);
3620 
3621 	ctrl->ctrl.opts = NULL;
3622 
3623 	/* initiate nvme ctrl ref counting teardown */
3624 	nvme_uninit_ctrl(&ctrl->ctrl);
3625 
3626 out_put_ctrl:
3627 	/* Remove core ctrl ref. */
3628 	nvme_put_ctrl(&ctrl->ctrl);
3629 
3630 	/* as we're past the point where we transition to the ref
3631 	 * counting teardown path, if we return a bad pointer here,
3632 	 * the calling routine, thinking it's prior to the
3633 	 * transition, will do an rport put. Since the teardown
3634 	 * path also does a rport put, we do an extra get here to
3635 	 * so proper order/teardown happens.
3636 	 */
3637 	nvme_fc_rport_get(rport);
3638 
3639 	return ERR_PTR(-EIO);
3640 }
3641 
3642 struct nvmet_fc_traddr {
3643 	u64	nn;
3644 	u64	pn;
3645 };
3646 
3647 static int
3648 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
3649 {
3650 	u64 token64;
3651 
3652 	if (match_u64(sstr, &token64))
3653 		return -EINVAL;
3654 	*val = token64;
3655 
3656 	return 0;
3657 }
3658 
3659 /*
3660  * This routine validates and extracts the WWN's from the TRADDR string.
3661  * As kernel parsers need the 0x to determine number base, universally
3662  * build string to parse with 0x prefix before parsing name strings.
3663  */
3664 static int
3665 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
3666 {
3667 	char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
3668 	substring_t wwn = { name, &name[sizeof(name)-1] };
3669 	int nnoffset, pnoffset;
3670 
3671 	/* validate if string is one of the 2 allowed formats */
3672 	if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
3673 			!strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
3674 			!strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
3675 				"pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
3676 		nnoffset = NVME_FC_TRADDR_OXNNLEN;
3677 		pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
3678 						NVME_FC_TRADDR_OXNNLEN;
3679 	} else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
3680 			!strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
3681 			!strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
3682 				"pn-", NVME_FC_TRADDR_NNLEN))) {
3683 		nnoffset = NVME_FC_TRADDR_NNLEN;
3684 		pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
3685 	} else
3686 		goto out_einval;
3687 
3688 	name[0] = '0';
3689 	name[1] = 'x';
3690 	name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
3691 
3692 	memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3693 	if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
3694 		goto out_einval;
3695 
3696 	memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3697 	if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
3698 		goto out_einval;
3699 
3700 	return 0;
3701 
3702 out_einval:
3703 	pr_warn("%s: bad traddr string\n", __func__);
3704 	return -EINVAL;
3705 }
3706 
3707 static struct nvme_ctrl *
3708 nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
3709 {
3710 	struct nvme_fc_lport *lport;
3711 	struct nvme_fc_rport *rport;
3712 	struct nvme_ctrl *ctrl;
3713 	struct nvmet_fc_traddr laddr = { 0L, 0L };
3714 	struct nvmet_fc_traddr raddr = { 0L, 0L };
3715 	unsigned long flags;
3716 	int ret;
3717 
3718 	ret = nvme_fc_parse_traddr(&raddr, opts->traddr, NVMF_TRADDR_SIZE);
3719 	if (ret || !raddr.nn || !raddr.pn)
3720 		return ERR_PTR(-EINVAL);
3721 
3722 	ret = nvme_fc_parse_traddr(&laddr, opts->host_traddr, NVMF_TRADDR_SIZE);
3723 	if (ret || !laddr.nn || !laddr.pn)
3724 		return ERR_PTR(-EINVAL);
3725 
3726 	/* find the host and remote ports to connect together */
3727 	spin_lock_irqsave(&nvme_fc_lock, flags);
3728 	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3729 		if (lport->localport.node_name != laddr.nn ||
3730 		    lport->localport.port_name != laddr.pn ||
3731 		    lport->localport.port_state != FC_OBJSTATE_ONLINE)
3732 			continue;
3733 
3734 		list_for_each_entry(rport, &lport->endp_list, endp_list) {
3735 			if (rport->remoteport.node_name != raddr.nn ||
3736 			    rport->remoteport.port_name != raddr.pn ||
3737 			    rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3738 				continue;
3739 
3740 			/* if fail to get reference fall through. Will error */
3741 			if (!nvme_fc_rport_get(rport))
3742 				break;
3743 
3744 			spin_unlock_irqrestore(&nvme_fc_lock, flags);
3745 
3746 			ctrl = nvme_fc_init_ctrl(dev, opts, lport, rport);
3747 			if (IS_ERR(ctrl))
3748 				nvme_fc_rport_put(rport);
3749 			return ctrl;
3750 		}
3751 	}
3752 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
3753 
3754 	pr_warn("%s: %s - %s combination not found\n",
3755 		__func__, opts->traddr, opts->host_traddr);
3756 	return ERR_PTR(-ENOENT);
3757 }
3758 
3759 
3760 static struct nvmf_transport_ops nvme_fc_transport = {
3761 	.name		= "fc",
3762 	.module		= THIS_MODULE,
3763 	.required_opts	= NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR,
3764 	.allowed_opts	= NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO,
3765 	.create_ctrl	= nvme_fc_create_ctrl,
3766 };
3767 
3768 /* Arbitrary successive failures max. With lots of subsystems could be high */
3769 #define DISCOVERY_MAX_FAIL	20
3770 
3771 static ssize_t nvme_fc_nvme_discovery_store(struct device *dev,
3772 		struct device_attribute *attr, const char *buf, size_t count)
3773 {
3774 	unsigned long flags;
3775 	LIST_HEAD(local_disc_list);
3776 	struct nvme_fc_lport *lport;
3777 	struct nvme_fc_rport *rport;
3778 	int failcnt = 0;
3779 
3780 	spin_lock_irqsave(&nvme_fc_lock, flags);
3781 restart:
3782 	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3783 		list_for_each_entry(rport, &lport->endp_list, endp_list) {
3784 			if (!nvme_fc_lport_get(lport))
3785 				continue;
3786 			if (!nvme_fc_rport_get(rport)) {
3787 				/*
3788 				 * This is a temporary condition. Upon restart
3789 				 * this rport will be gone from the list.
3790 				 *
3791 				 * Revert the lport put and retry.  Anything
3792 				 * added to the list already will be skipped (as
3793 				 * they are no longer list_empty).  Loops should
3794 				 * resume at rports that were not yet seen.
3795 				 */
3796 				nvme_fc_lport_put(lport);
3797 
3798 				if (failcnt++ < DISCOVERY_MAX_FAIL)
3799 					goto restart;
3800 
3801 				pr_err("nvme_discovery: too many reference "
3802 				       "failures\n");
3803 				goto process_local_list;
3804 			}
3805 			if (list_empty(&rport->disc_list))
3806 				list_add_tail(&rport->disc_list,
3807 					      &local_disc_list);
3808 		}
3809 	}
3810 
3811 process_local_list:
3812 	while (!list_empty(&local_disc_list)) {
3813 		rport = list_first_entry(&local_disc_list,
3814 					 struct nvme_fc_rport, disc_list);
3815 		list_del_init(&rport->disc_list);
3816 		spin_unlock_irqrestore(&nvme_fc_lock, flags);
3817 
3818 		lport = rport->lport;
3819 		/* signal discovery. Won't hurt if it repeats */
3820 		nvme_fc_signal_discovery_scan(lport, rport);
3821 		nvme_fc_rport_put(rport);
3822 		nvme_fc_lport_put(lport);
3823 
3824 		spin_lock_irqsave(&nvme_fc_lock, flags);
3825 	}
3826 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
3827 
3828 	return count;
3829 }
3830 
3831 static DEVICE_ATTR(nvme_discovery, 0200, NULL, nvme_fc_nvme_discovery_store);
3832 
3833 #ifdef CONFIG_BLK_CGROUP_FC_APPID
3834 /* Parse the cgroup id from a buf and return the length of cgrpid */
3835 static int fc_parse_cgrpid(const char *buf, u64 *id)
3836 {
3837 	char cgrp_id[16+1];
3838 	int cgrpid_len, j;
3839 
3840 	memset(cgrp_id, 0x0, sizeof(cgrp_id));
3841 	for (cgrpid_len = 0, j = 0; cgrpid_len < 17; cgrpid_len++) {
3842 		if (buf[cgrpid_len] != ':')
3843 			cgrp_id[cgrpid_len] = buf[cgrpid_len];
3844 		else {
3845 			j = 1;
3846 			break;
3847 		}
3848 	}
3849 	if (!j)
3850 		return -EINVAL;
3851 	if (kstrtou64(cgrp_id, 16, id) < 0)
3852 		return -EINVAL;
3853 	return cgrpid_len;
3854 }
3855 
3856 /*
3857  * Parse and update the appid in the blkcg associated with the cgroupid.
3858  */
3859 static ssize_t fc_appid_store(struct device *dev,
3860 		struct device_attribute *attr, const char *buf, size_t count)
3861 {
3862 	size_t orig_count = count;
3863 	u64 cgrp_id;
3864 	int appid_len = 0;
3865 	int cgrpid_len = 0;
3866 	char app_id[FC_APPID_LEN];
3867 	int ret = 0;
3868 
3869 	if (buf[count-1] == '\n')
3870 		count--;
3871 
3872 	if ((count > (16+1+FC_APPID_LEN)) || (!strchr(buf, ':')))
3873 		return -EINVAL;
3874 
3875 	cgrpid_len = fc_parse_cgrpid(buf, &cgrp_id);
3876 	if (cgrpid_len < 0)
3877 		return -EINVAL;
3878 	appid_len = count - cgrpid_len - 1;
3879 	if (appid_len > FC_APPID_LEN)
3880 		return -EINVAL;
3881 
3882 	memset(app_id, 0x0, sizeof(app_id));
3883 	memcpy(app_id, &buf[cgrpid_len+1], appid_len);
3884 	ret = blkcg_set_fc_appid(app_id, cgrp_id, sizeof(app_id));
3885 	if (ret < 0)
3886 		return ret;
3887 	return orig_count;
3888 }
3889 static DEVICE_ATTR(appid_store, 0200, NULL, fc_appid_store);
3890 #endif /* CONFIG_BLK_CGROUP_FC_APPID */
3891 
3892 static struct attribute *nvme_fc_attrs[] = {
3893 	&dev_attr_nvme_discovery.attr,
3894 #ifdef CONFIG_BLK_CGROUP_FC_APPID
3895 	&dev_attr_appid_store.attr,
3896 #endif
3897 	NULL
3898 };
3899 
3900 static const struct attribute_group nvme_fc_attr_group = {
3901 	.attrs = nvme_fc_attrs,
3902 };
3903 
3904 static const struct attribute_group *nvme_fc_attr_groups[] = {
3905 	&nvme_fc_attr_group,
3906 	NULL
3907 };
3908 
3909 static struct class fc_class = {
3910 	.name = "fc",
3911 	.dev_groups = nvme_fc_attr_groups,
3912 };
3913 
3914 static int __init nvme_fc_init_module(void)
3915 {
3916 	int ret;
3917 
3918 	/*
3919 	 * NOTE:
3920 	 * It is expected that in the future the kernel will combine
3921 	 * the FC-isms that are currently under scsi and now being
3922 	 * added to by NVME into a new standalone FC class. The SCSI
3923 	 * and NVME protocols and their devices would be under this
3924 	 * new FC class.
3925 	 *
3926 	 * As we need something to post FC-specific udev events to,
3927 	 * specifically for nvme probe events, start by creating the
3928 	 * new device class.  When the new standalone FC class is
3929 	 * put in place, this code will move to a more generic
3930 	 * location for the class.
3931 	 */
3932 	ret = class_register(&fc_class);
3933 	if (ret) {
3934 		pr_err("couldn't register class fc\n");
3935 		return ret;
3936 	}
3937 
3938 	/*
3939 	 * Create a device for the FC-centric udev events
3940 	 */
3941 	fc_udev_device = device_create(&fc_class, NULL, MKDEV(0, 0), NULL,
3942 				"fc_udev_device");
3943 	if (IS_ERR(fc_udev_device)) {
3944 		pr_err("couldn't create fc_udev device!\n");
3945 		ret = PTR_ERR(fc_udev_device);
3946 		goto out_destroy_class;
3947 	}
3948 
3949 	ret = nvmf_register_transport(&nvme_fc_transport);
3950 	if (ret)
3951 		goto out_destroy_device;
3952 
3953 	return 0;
3954 
3955 out_destroy_device:
3956 	device_destroy(&fc_class, MKDEV(0, 0));
3957 out_destroy_class:
3958 	class_unregister(&fc_class);
3959 
3960 	return ret;
3961 }
3962 
3963 static void
3964 nvme_fc_delete_controllers(struct nvme_fc_rport *rport)
3965 {
3966 	struct nvme_fc_ctrl *ctrl;
3967 
3968 	spin_lock(&rport->lock);
3969 	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3970 		dev_warn(ctrl->ctrl.device,
3971 			"NVME-FC{%d}: transport unloading: deleting ctrl\n",
3972 			ctrl->cnum);
3973 		nvme_delete_ctrl(&ctrl->ctrl);
3974 	}
3975 	spin_unlock(&rport->lock);
3976 }
3977 
3978 static void __exit nvme_fc_exit_module(void)
3979 {
3980 	struct nvme_fc_lport *lport;
3981 	struct nvme_fc_rport *rport;
3982 	unsigned long flags;
3983 
3984 	spin_lock_irqsave(&nvme_fc_lock, flags);
3985 	list_for_each_entry(lport, &nvme_fc_lport_list, port_list)
3986 		list_for_each_entry(rport, &lport->endp_list, endp_list)
3987 			nvme_fc_delete_controllers(rport);
3988 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
3989 	flush_workqueue(nvme_delete_wq);
3990 
3991 	nvmf_unregister_transport(&nvme_fc_transport);
3992 
3993 	device_destroy(&fc_class, MKDEV(0, 0));
3994 	class_unregister(&fc_class);
3995 }
3996 
3997 module_init(nvme_fc_init_module);
3998 module_exit(nvme_fc_exit_module);
3999 
4000 MODULE_DESCRIPTION("NVMe host FC transport driver");
4001 MODULE_LICENSE("GPL v2");
4002