1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016 Avago Technologies. All rights reserved. 4 */ 5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 6 #include <linux/module.h> 7 #include <linux/parser.h> 8 #include <uapi/scsi/fc/fc_fs.h> 9 #include <uapi/scsi/fc/fc_els.h> 10 #include <linux/delay.h> 11 #include <linux/overflow.h> 12 #include <linux/blk-cgroup.h> 13 #include "nvme.h" 14 #include "fabrics.h" 15 #include <linux/nvme-fc-driver.h> 16 #include <linux/nvme-fc.h> 17 #include "fc.h" 18 #include <scsi/scsi_transport_fc.h> 19 20 /* *************************** Data Structures/Defines ****************** */ 21 22 23 enum nvme_fc_queue_flags { 24 NVME_FC_Q_CONNECTED = 0, 25 NVME_FC_Q_LIVE, 26 }; 27 28 #define NVME_FC_DEFAULT_DEV_LOSS_TMO 60 /* seconds */ 29 #define NVME_FC_DEFAULT_RECONNECT_TMO 2 /* delay between reconnects 30 * when connected and a 31 * connection failure. 32 */ 33 34 struct nvme_fc_queue { 35 struct nvme_fc_ctrl *ctrl; 36 struct device *dev; 37 struct blk_mq_hw_ctx *hctx; 38 void *lldd_handle; 39 size_t cmnd_capsule_len; 40 u32 qnum; 41 u32 rqcnt; 42 u32 seqno; 43 44 u64 connection_id; 45 atomic_t csn; 46 47 unsigned long flags; 48 } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ 49 50 enum nvme_fcop_flags { 51 FCOP_FLAGS_TERMIO = (1 << 0), 52 FCOP_FLAGS_AEN = (1 << 1), 53 }; 54 55 struct nvmefc_ls_req_op { 56 struct nvmefc_ls_req ls_req; 57 58 struct nvme_fc_rport *rport; 59 struct nvme_fc_queue *queue; 60 struct request *rq; 61 u32 flags; 62 63 int ls_error; 64 struct completion ls_done; 65 struct list_head lsreq_list; /* rport->ls_req_list */ 66 bool req_queued; 67 }; 68 69 struct nvmefc_ls_rcv_op { 70 struct nvme_fc_rport *rport; 71 struct nvmefc_ls_rsp *lsrsp; 72 union nvmefc_ls_requests *rqstbuf; 73 union nvmefc_ls_responses *rspbuf; 74 u16 rqstdatalen; 75 bool handled; 76 dma_addr_t rspdma; 77 struct list_head lsrcv_list; /* rport->ls_rcv_list */ 78 } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ 79 80 enum nvme_fcpop_state { 81 FCPOP_STATE_UNINIT = 0, 82 FCPOP_STATE_IDLE = 1, 83 FCPOP_STATE_ACTIVE = 2, 84 FCPOP_STATE_ABORTED = 3, 85 FCPOP_STATE_COMPLETE = 4, 86 }; 87 88 struct nvme_fc_fcp_op { 89 struct nvme_request nreq; /* 90 * nvme/host/core.c 91 * requires this to be 92 * the 1st element in the 93 * private structure 94 * associated with the 95 * request. 96 */ 97 struct nvmefc_fcp_req fcp_req; 98 99 struct nvme_fc_ctrl *ctrl; 100 struct nvme_fc_queue *queue; 101 struct request *rq; 102 103 atomic_t state; 104 u32 flags; 105 u32 rqno; 106 u32 nents; 107 108 struct nvme_fc_cmd_iu cmd_iu; 109 struct nvme_fc_ersp_iu rsp_iu; 110 }; 111 112 struct nvme_fcp_op_w_sgl { 113 struct nvme_fc_fcp_op op; 114 struct scatterlist sgl[NVME_INLINE_SG_CNT]; 115 uint8_t priv[]; 116 }; 117 118 struct nvme_fc_lport { 119 struct nvme_fc_local_port localport; 120 121 struct ida endp_cnt; 122 struct list_head port_list; /* nvme_fc_port_list */ 123 struct list_head endp_list; 124 struct device *dev; /* physical device for dma */ 125 struct nvme_fc_port_template *ops; 126 struct kref ref; 127 atomic_t act_rport_cnt; 128 } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ 129 130 struct nvme_fc_rport { 131 struct nvme_fc_remote_port remoteport; 132 133 struct list_head endp_list; /* for lport->endp_list */ 134 struct list_head ctrl_list; 135 struct list_head ls_req_list; 136 struct list_head ls_rcv_list; 137 struct list_head disc_list; 138 struct device *dev; /* physical device for dma */ 139 struct nvme_fc_lport *lport; 140 spinlock_t lock; 141 struct kref ref; 142 atomic_t act_ctrl_cnt; 143 unsigned long dev_loss_end; 144 struct work_struct lsrcv_work; 145 } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ 146 147 /* fc_ctrl flags values - specified as bit positions */ 148 #define ASSOC_ACTIVE 0 149 #define ASSOC_FAILED 1 150 #define FCCTRL_TERMIO 2 151 152 struct nvme_fc_ctrl { 153 spinlock_t lock; 154 struct nvme_fc_queue *queues; 155 struct device *dev; 156 struct nvme_fc_lport *lport; 157 struct nvme_fc_rport *rport; 158 u32 cnum; 159 160 bool ioq_live; 161 u64 association_id; 162 struct nvmefc_ls_rcv_op *rcv_disconn; 163 164 struct list_head ctrl_list; /* rport->ctrl_list */ 165 166 struct blk_mq_tag_set admin_tag_set; 167 struct blk_mq_tag_set tag_set; 168 169 struct work_struct ioerr_work; 170 struct delayed_work connect_work; 171 172 struct kref ref; 173 unsigned long flags; 174 u32 iocnt; 175 wait_queue_head_t ioabort_wait; 176 177 struct nvme_fc_fcp_op aen_ops[NVME_NR_AEN_COMMANDS]; 178 179 struct nvme_ctrl ctrl; 180 }; 181 182 static inline struct nvme_fc_ctrl * 183 to_fc_ctrl(struct nvme_ctrl *ctrl) 184 { 185 return container_of(ctrl, struct nvme_fc_ctrl, ctrl); 186 } 187 188 static inline struct nvme_fc_lport * 189 localport_to_lport(struct nvme_fc_local_port *portptr) 190 { 191 return container_of(portptr, struct nvme_fc_lport, localport); 192 } 193 194 static inline struct nvme_fc_rport * 195 remoteport_to_rport(struct nvme_fc_remote_port *portptr) 196 { 197 return container_of(portptr, struct nvme_fc_rport, remoteport); 198 } 199 200 static inline struct nvmefc_ls_req_op * 201 ls_req_to_lsop(struct nvmefc_ls_req *lsreq) 202 { 203 return container_of(lsreq, struct nvmefc_ls_req_op, ls_req); 204 } 205 206 static inline struct nvme_fc_fcp_op * 207 fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq) 208 { 209 return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req); 210 } 211 212 213 214 /* *************************** Globals **************************** */ 215 216 217 static DEFINE_SPINLOCK(nvme_fc_lock); 218 219 static LIST_HEAD(nvme_fc_lport_list); 220 static DEFINE_IDA(nvme_fc_local_port_cnt); 221 static DEFINE_IDA(nvme_fc_ctrl_cnt); 222 223 /* 224 * These items are short-term. They will eventually be moved into 225 * a generic FC class. See comments in module init. 226 */ 227 static struct device *fc_udev_device; 228 229 static void nvme_fc_complete_rq(struct request *rq); 230 231 /* *********************** FC-NVME Port Management ************************ */ 232 233 static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *, 234 struct nvme_fc_queue *, unsigned int); 235 236 static void nvme_fc_handle_ls_rqst_work(struct work_struct *work); 237 238 239 static void 240 nvme_fc_free_lport(struct kref *ref) 241 { 242 struct nvme_fc_lport *lport = 243 container_of(ref, struct nvme_fc_lport, ref); 244 unsigned long flags; 245 246 WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED); 247 WARN_ON(!list_empty(&lport->endp_list)); 248 249 /* remove from transport list */ 250 spin_lock_irqsave(&nvme_fc_lock, flags); 251 list_del(&lport->port_list); 252 spin_unlock_irqrestore(&nvme_fc_lock, flags); 253 254 ida_free(&nvme_fc_local_port_cnt, lport->localport.port_num); 255 ida_destroy(&lport->endp_cnt); 256 257 put_device(lport->dev); 258 259 kfree(lport); 260 } 261 262 static void 263 nvme_fc_lport_put(struct nvme_fc_lport *lport) 264 { 265 kref_put(&lport->ref, nvme_fc_free_lport); 266 } 267 268 static int 269 nvme_fc_lport_get(struct nvme_fc_lport *lport) 270 { 271 return kref_get_unless_zero(&lport->ref); 272 } 273 274 275 static struct nvme_fc_lport * 276 nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo, 277 struct nvme_fc_port_template *ops, 278 struct device *dev) 279 { 280 struct nvme_fc_lport *lport; 281 unsigned long flags; 282 283 spin_lock_irqsave(&nvme_fc_lock, flags); 284 285 list_for_each_entry(lport, &nvme_fc_lport_list, port_list) { 286 if (lport->localport.node_name != pinfo->node_name || 287 lport->localport.port_name != pinfo->port_name) 288 continue; 289 290 if (lport->dev != dev) { 291 lport = ERR_PTR(-EXDEV); 292 goto out_done; 293 } 294 295 if (lport->localport.port_state != FC_OBJSTATE_DELETED) { 296 lport = ERR_PTR(-EEXIST); 297 goto out_done; 298 } 299 300 if (!nvme_fc_lport_get(lport)) { 301 /* 302 * fails if ref cnt already 0. If so, 303 * act as if lport already deleted 304 */ 305 lport = NULL; 306 goto out_done; 307 } 308 309 /* resume the lport */ 310 311 lport->ops = ops; 312 lport->localport.port_role = pinfo->port_role; 313 lport->localport.port_id = pinfo->port_id; 314 lport->localport.port_state = FC_OBJSTATE_ONLINE; 315 316 spin_unlock_irqrestore(&nvme_fc_lock, flags); 317 318 return lport; 319 } 320 321 lport = NULL; 322 323 out_done: 324 spin_unlock_irqrestore(&nvme_fc_lock, flags); 325 326 return lport; 327 } 328 329 /** 330 * nvme_fc_register_localport - transport entry point called by an 331 * LLDD to register the existence of a NVME 332 * host FC port. 333 * @pinfo: pointer to information about the port to be registered 334 * @template: LLDD entrypoints and operational parameters for the port 335 * @dev: physical hardware device node port corresponds to. Will be 336 * used for DMA mappings 337 * @portptr: pointer to a local port pointer. Upon success, the routine 338 * will allocate a nvme_fc_local_port structure and place its 339 * address in the local port pointer. Upon failure, local port 340 * pointer will be set to 0. 341 * 342 * Returns: 343 * a completion status. Must be 0 upon success; a negative errno 344 * (ex: -ENXIO) upon failure. 345 */ 346 int 347 nvme_fc_register_localport(struct nvme_fc_port_info *pinfo, 348 struct nvme_fc_port_template *template, 349 struct device *dev, 350 struct nvme_fc_local_port **portptr) 351 { 352 struct nvme_fc_lport *newrec; 353 unsigned long flags; 354 int ret, idx; 355 356 if (!template->localport_delete || !template->remoteport_delete || 357 !template->ls_req || !template->fcp_io || 358 !template->ls_abort || !template->fcp_abort || 359 !template->max_hw_queues || !template->max_sgl_segments || 360 !template->max_dif_sgl_segments || !template->dma_boundary) { 361 ret = -EINVAL; 362 goto out_reghost_failed; 363 } 364 365 /* 366 * look to see if there is already a localport that had been 367 * deregistered and in the process of waiting for all the 368 * references to fully be removed. If the references haven't 369 * expired, we can simply re-enable the localport. Remoteports 370 * and controller reconnections should resume naturally. 371 */ 372 newrec = nvme_fc_attach_to_unreg_lport(pinfo, template, dev); 373 374 /* found an lport, but something about its state is bad */ 375 if (IS_ERR(newrec)) { 376 ret = PTR_ERR(newrec); 377 goto out_reghost_failed; 378 379 /* found existing lport, which was resumed */ 380 } else if (newrec) { 381 *portptr = &newrec->localport; 382 return 0; 383 } 384 385 /* nothing found - allocate a new localport struct */ 386 387 newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz), 388 GFP_KERNEL); 389 if (!newrec) { 390 ret = -ENOMEM; 391 goto out_reghost_failed; 392 } 393 394 idx = ida_alloc(&nvme_fc_local_port_cnt, GFP_KERNEL); 395 if (idx < 0) { 396 ret = -ENOSPC; 397 goto out_fail_kfree; 398 } 399 400 if (!get_device(dev) && dev) { 401 ret = -ENODEV; 402 goto out_ida_put; 403 } 404 405 INIT_LIST_HEAD(&newrec->port_list); 406 INIT_LIST_HEAD(&newrec->endp_list); 407 kref_init(&newrec->ref); 408 atomic_set(&newrec->act_rport_cnt, 0); 409 newrec->ops = template; 410 newrec->dev = dev; 411 ida_init(&newrec->endp_cnt); 412 if (template->local_priv_sz) 413 newrec->localport.private = &newrec[1]; 414 else 415 newrec->localport.private = NULL; 416 newrec->localport.node_name = pinfo->node_name; 417 newrec->localport.port_name = pinfo->port_name; 418 newrec->localport.port_role = pinfo->port_role; 419 newrec->localport.port_id = pinfo->port_id; 420 newrec->localport.port_state = FC_OBJSTATE_ONLINE; 421 newrec->localport.port_num = idx; 422 423 spin_lock_irqsave(&nvme_fc_lock, flags); 424 list_add_tail(&newrec->port_list, &nvme_fc_lport_list); 425 spin_unlock_irqrestore(&nvme_fc_lock, flags); 426 427 if (dev) 428 dma_set_seg_boundary(dev, template->dma_boundary); 429 430 *portptr = &newrec->localport; 431 return 0; 432 433 out_ida_put: 434 ida_free(&nvme_fc_local_port_cnt, idx); 435 out_fail_kfree: 436 kfree(newrec); 437 out_reghost_failed: 438 *portptr = NULL; 439 440 return ret; 441 } 442 EXPORT_SYMBOL_GPL(nvme_fc_register_localport); 443 444 /** 445 * nvme_fc_unregister_localport - transport entry point called by an 446 * LLDD to deregister/remove a previously 447 * registered a NVME host FC port. 448 * @portptr: pointer to the (registered) local port that is to be deregistered. 449 * 450 * Returns: 451 * a completion status. Must be 0 upon success; a negative errno 452 * (ex: -ENXIO) upon failure. 453 */ 454 int 455 nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr) 456 { 457 struct nvme_fc_lport *lport = localport_to_lport(portptr); 458 unsigned long flags; 459 460 if (!portptr) 461 return -EINVAL; 462 463 spin_lock_irqsave(&nvme_fc_lock, flags); 464 465 if (portptr->port_state != FC_OBJSTATE_ONLINE) { 466 spin_unlock_irqrestore(&nvme_fc_lock, flags); 467 return -EINVAL; 468 } 469 portptr->port_state = FC_OBJSTATE_DELETED; 470 471 spin_unlock_irqrestore(&nvme_fc_lock, flags); 472 473 if (atomic_read(&lport->act_rport_cnt) == 0) 474 lport->ops->localport_delete(&lport->localport); 475 476 nvme_fc_lport_put(lport); 477 478 return 0; 479 } 480 EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport); 481 482 /* 483 * TRADDR strings, per FC-NVME are fixed format: 484 * "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters 485 * udev event will only differ by prefix of what field is 486 * being specified: 487 * "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters 488 * 19 + 43 + null_fudge = 64 characters 489 */ 490 #define FCNVME_TRADDR_LENGTH 64 491 492 static void 493 nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport, 494 struct nvme_fc_rport *rport) 495 { 496 char hostaddr[FCNVME_TRADDR_LENGTH]; /* NVMEFC_HOST_TRADDR=...*/ 497 char tgtaddr[FCNVME_TRADDR_LENGTH]; /* NVMEFC_TRADDR=...*/ 498 char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL }; 499 500 if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY)) 501 return; 502 503 snprintf(hostaddr, sizeof(hostaddr), 504 "NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx", 505 lport->localport.node_name, lport->localport.port_name); 506 snprintf(tgtaddr, sizeof(tgtaddr), 507 "NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx", 508 rport->remoteport.node_name, rport->remoteport.port_name); 509 kobject_uevent_env(&fc_udev_device->kobj, KOBJ_CHANGE, envp); 510 } 511 512 static void 513 nvme_fc_free_rport(struct kref *ref) 514 { 515 struct nvme_fc_rport *rport = 516 container_of(ref, struct nvme_fc_rport, ref); 517 struct nvme_fc_lport *lport = 518 localport_to_lport(rport->remoteport.localport); 519 unsigned long flags; 520 521 WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED); 522 WARN_ON(!list_empty(&rport->ctrl_list)); 523 524 /* remove from lport list */ 525 spin_lock_irqsave(&nvme_fc_lock, flags); 526 list_del(&rport->endp_list); 527 spin_unlock_irqrestore(&nvme_fc_lock, flags); 528 529 WARN_ON(!list_empty(&rport->disc_list)); 530 ida_free(&lport->endp_cnt, rport->remoteport.port_num); 531 532 kfree(rport); 533 534 nvme_fc_lport_put(lport); 535 } 536 537 static void 538 nvme_fc_rport_put(struct nvme_fc_rport *rport) 539 { 540 kref_put(&rport->ref, nvme_fc_free_rport); 541 } 542 543 static int 544 nvme_fc_rport_get(struct nvme_fc_rport *rport) 545 { 546 return kref_get_unless_zero(&rport->ref); 547 } 548 549 static void 550 nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl) 551 { 552 switch (nvme_ctrl_state(&ctrl->ctrl)) { 553 case NVME_CTRL_NEW: 554 case NVME_CTRL_CONNECTING: 555 /* 556 * As all reconnects were suppressed, schedule a 557 * connect. 558 */ 559 dev_info(ctrl->ctrl.device, 560 "NVME-FC{%d}: connectivity re-established. " 561 "Attempting reconnect\n", ctrl->cnum); 562 563 queue_delayed_work(nvme_wq, &ctrl->connect_work, 0); 564 break; 565 566 case NVME_CTRL_RESETTING: 567 /* 568 * Controller is already in the process of terminating the 569 * association. No need to do anything further. The reconnect 570 * step will naturally occur after the reset completes. 571 */ 572 break; 573 574 default: 575 /* no action to take - let it delete */ 576 break; 577 } 578 } 579 580 static struct nvme_fc_rport * 581 nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport, 582 struct nvme_fc_port_info *pinfo) 583 { 584 struct nvme_fc_rport *rport; 585 struct nvme_fc_ctrl *ctrl; 586 unsigned long flags; 587 588 spin_lock_irqsave(&nvme_fc_lock, flags); 589 590 list_for_each_entry(rport, &lport->endp_list, endp_list) { 591 if (rport->remoteport.node_name != pinfo->node_name || 592 rport->remoteport.port_name != pinfo->port_name) 593 continue; 594 595 if (!nvme_fc_rport_get(rport)) { 596 rport = ERR_PTR(-ENOLCK); 597 goto out_done; 598 } 599 600 spin_unlock_irqrestore(&nvme_fc_lock, flags); 601 602 spin_lock_irqsave(&rport->lock, flags); 603 604 /* has it been unregistered */ 605 if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) { 606 /* means lldd called us twice */ 607 spin_unlock_irqrestore(&rport->lock, flags); 608 nvme_fc_rport_put(rport); 609 return ERR_PTR(-ESTALE); 610 } 611 612 rport->remoteport.port_role = pinfo->port_role; 613 rport->remoteport.port_id = pinfo->port_id; 614 rport->remoteport.port_state = FC_OBJSTATE_ONLINE; 615 rport->dev_loss_end = 0; 616 617 /* 618 * kick off a reconnect attempt on all associations to the 619 * remote port. A successful reconnects will resume i/o. 620 */ 621 list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) 622 nvme_fc_resume_controller(ctrl); 623 624 spin_unlock_irqrestore(&rport->lock, flags); 625 626 return rport; 627 } 628 629 rport = NULL; 630 631 out_done: 632 spin_unlock_irqrestore(&nvme_fc_lock, flags); 633 634 return rport; 635 } 636 637 static inline void 638 __nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport, 639 struct nvme_fc_port_info *pinfo) 640 { 641 if (pinfo->dev_loss_tmo) 642 rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo; 643 else 644 rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO; 645 } 646 647 /** 648 * nvme_fc_register_remoteport - transport entry point called by an 649 * LLDD to register the existence of a NVME 650 * subsystem FC port on its fabric. 651 * @localport: pointer to the (registered) local port that the remote 652 * subsystem port is connected to. 653 * @pinfo: pointer to information about the port to be registered 654 * @portptr: pointer to a remote port pointer. Upon success, the routine 655 * will allocate a nvme_fc_remote_port structure and place its 656 * address in the remote port pointer. Upon failure, remote port 657 * pointer will be set to 0. 658 * 659 * Returns: 660 * a completion status. Must be 0 upon success; a negative errno 661 * (ex: -ENXIO) upon failure. 662 */ 663 int 664 nvme_fc_register_remoteport(struct nvme_fc_local_port *localport, 665 struct nvme_fc_port_info *pinfo, 666 struct nvme_fc_remote_port **portptr) 667 { 668 struct nvme_fc_lport *lport = localport_to_lport(localport); 669 struct nvme_fc_rport *newrec; 670 unsigned long flags; 671 int ret, idx; 672 673 if (!nvme_fc_lport_get(lport)) { 674 ret = -ESHUTDOWN; 675 goto out_reghost_failed; 676 } 677 678 /* 679 * look to see if there is already a remoteport that is waiting 680 * for a reconnect (within dev_loss_tmo) with the same WWN's. 681 * If so, transition to it and reconnect. 682 */ 683 newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo); 684 685 /* found an rport, but something about its state is bad */ 686 if (IS_ERR(newrec)) { 687 ret = PTR_ERR(newrec); 688 goto out_lport_put; 689 690 /* found existing rport, which was resumed */ 691 } else if (newrec) { 692 nvme_fc_lport_put(lport); 693 __nvme_fc_set_dev_loss_tmo(newrec, pinfo); 694 nvme_fc_signal_discovery_scan(lport, newrec); 695 *portptr = &newrec->remoteport; 696 return 0; 697 } 698 699 /* nothing found - allocate a new remoteport struct */ 700 701 newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz), 702 GFP_KERNEL); 703 if (!newrec) { 704 ret = -ENOMEM; 705 goto out_lport_put; 706 } 707 708 idx = ida_alloc(&lport->endp_cnt, GFP_KERNEL); 709 if (idx < 0) { 710 ret = -ENOSPC; 711 goto out_kfree_rport; 712 } 713 714 INIT_LIST_HEAD(&newrec->endp_list); 715 INIT_LIST_HEAD(&newrec->ctrl_list); 716 INIT_LIST_HEAD(&newrec->ls_req_list); 717 INIT_LIST_HEAD(&newrec->disc_list); 718 kref_init(&newrec->ref); 719 atomic_set(&newrec->act_ctrl_cnt, 0); 720 spin_lock_init(&newrec->lock); 721 newrec->remoteport.localport = &lport->localport; 722 INIT_LIST_HEAD(&newrec->ls_rcv_list); 723 newrec->dev = lport->dev; 724 newrec->lport = lport; 725 if (lport->ops->remote_priv_sz) 726 newrec->remoteport.private = &newrec[1]; 727 else 728 newrec->remoteport.private = NULL; 729 newrec->remoteport.port_role = pinfo->port_role; 730 newrec->remoteport.node_name = pinfo->node_name; 731 newrec->remoteport.port_name = pinfo->port_name; 732 newrec->remoteport.port_id = pinfo->port_id; 733 newrec->remoteport.port_state = FC_OBJSTATE_ONLINE; 734 newrec->remoteport.port_num = idx; 735 __nvme_fc_set_dev_loss_tmo(newrec, pinfo); 736 INIT_WORK(&newrec->lsrcv_work, nvme_fc_handle_ls_rqst_work); 737 738 spin_lock_irqsave(&nvme_fc_lock, flags); 739 list_add_tail(&newrec->endp_list, &lport->endp_list); 740 spin_unlock_irqrestore(&nvme_fc_lock, flags); 741 742 nvme_fc_signal_discovery_scan(lport, newrec); 743 744 *portptr = &newrec->remoteport; 745 return 0; 746 747 out_kfree_rport: 748 kfree(newrec); 749 out_lport_put: 750 nvme_fc_lport_put(lport); 751 out_reghost_failed: 752 *portptr = NULL; 753 return ret; 754 } 755 EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport); 756 757 static int 758 nvme_fc_abort_lsops(struct nvme_fc_rport *rport) 759 { 760 struct nvmefc_ls_req_op *lsop; 761 unsigned long flags; 762 763 restart: 764 spin_lock_irqsave(&rport->lock, flags); 765 766 list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) { 767 if (!(lsop->flags & FCOP_FLAGS_TERMIO)) { 768 lsop->flags |= FCOP_FLAGS_TERMIO; 769 spin_unlock_irqrestore(&rport->lock, flags); 770 rport->lport->ops->ls_abort(&rport->lport->localport, 771 &rport->remoteport, 772 &lsop->ls_req); 773 goto restart; 774 } 775 } 776 spin_unlock_irqrestore(&rport->lock, flags); 777 778 return 0; 779 } 780 781 static void 782 nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl) 783 { 784 enum nvme_ctrl_state state; 785 unsigned long flags; 786 787 dev_info(ctrl->ctrl.device, 788 "NVME-FC{%d}: controller connectivity lost. Awaiting " 789 "Reconnect", ctrl->cnum); 790 791 spin_lock_irqsave(&ctrl->lock, flags); 792 set_bit(ASSOC_FAILED, &ctrl->flags); 793 state = nvme_ctrl_state(&ctrl->ctrl); 794 spin_unlock_irqrestore(&ctrl->lock, flags); 795 796 switch (state) { 797 case NVME_CTRL_NEW: 798 case NVME_CTRL_LIVE: 799 /* 800 * Schedule a controller reset. The reset will terminate the 801 * association and schedule the reconnect timer. Reconnects 802 * will be attempted until either the ctlr_loss_tmo 803 * (max_retries * connect_delay) expires or the remoteport's 804 * dev_loss_tmo expires. 805 */ 806 if (nvme_reset_ctrl(&ctrl->ctrl)) { 807 dev_warn(ctrl->ctrl.device, 808 "NVME-FC{%d}: Couldn't schedule reset.\n", 809 ctrl->cnum); 810 nvme_delete_ctrl(&ctrl->ctrl); 811 } 812 break; 813 814 case NVME_CTRL_CONNECTING: 815 /* 816 * The association has already been terminated and the 817 * controller is attempting reconnects. No need to do anything 818 * futher. Reconnects will be attempted until either the 819 * ctlr_loss_tmo (max_retries * connect_delay) expires or the 820 * remoteport's dev_loss_tmo expires. 821 */ 822 break; 823 824 case NVME_CTRL_RESETTING: 825 /* 826 * Controller is already in the process of terminating the 827 * association. No need to do anything further. The reconnect 828 * step will kick in naturally after the association is 829 * terminated. 830 */ 831 break; 832 833 case NVME_CTRL_DELETING: 834 case NVME_CTRL_DELETING_NOIO: 835 default: 836 /* no action to take - let it delete */ 837 break; 838 } 839 } 840 841 /** 842 * nvme_fc_unregister_remoteport - transport entry point called by an 843 * LLDD to deregister/remove a previously 844 * registered a NVME subsystem FC port. 845 * @portptr: pointer to the (registered) remote port that is to be 846 * deregistered. 847 * 848 * Returns: 849 * a completion status. Must be 0 upon success; a negative errno 850 * (ex: -ENXIO) upon failure. 851 */ 852 int 853 nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr) 854 { 855 struct nvme_fc_rport *rport = remoteport_to_rport(portptr); 856 struct nvme_fc_ctrl *ctrl; 857 unsigned long flags; 858 859 if (!portptr) 860 return -EINVAL; 861 862 spin_lock_irqsave(&rport->lock, flags); 863 864 if (portptr->port_state != FC_OBJSTATE_ONLINE) { 865 spin_unlock_irqrestore(&rport->lock, flags); 866 return -EINVAL; 867 } 868 portptr->port_state = FC_OBJSTATE_DELETED; 869 870 rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ); 871 872 list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) { 873 /* if dev_loss_tmo==0, dev loss is immediate */ 874 if (!portptr->dev_loss_tmo) { 875 dev_warn(ctrl->ctrl.device, 876 "NVME-FC{%d}: controller connectivity lost.\n", 877 ctrl->cnum); 878 nvme_delete_ctrl(&ctrl->ctrl); 879 } else 880 nvme_fc_ctrl_connectivity_loss(ctrl); 881 } 882 883 spin_unlock_irqrestore(&rport->lock, flags); 884 885 nvme_fc_abort_lsops(rport); 886 887 if (atomic_read(&rport->act_ctrl_cnt) == 0) 888 rport->lport->ops->remoteport_delete(portptr); 889 890 /* 891 * release the reference, which will allow, if all controllers 892 * go away, which should only occur after dev_loss_tmo occurs, 893 * for the rport to be torn down. 894 */ 895 nvme_fc_rport_put(rport); 896 897 return 0; 898 } 899 EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport); 900 901 /** 902 * nvme_fc_rescan_remoteport - transport entry point called by an 903 * LLDD to request a nvme device rescan. 904 * @remoteport: pointer to the (registered) remote port that is to be 905 * rescanned. 906 * 907 * Returns: N/A 908 */ 909 void 910 nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport) 911 { 912 struct nvme_fc_rport *rport = remoteport_to_rport(remoteport); 913 914 nvme_fc_signal_discovery_scan(rport->lport, rport); 915 } 916 EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport); 917 918 int 919 nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr, 920 u32 dev_loss_tmo) 921 { 922 struct nvme_fc_rport *rport = remoteport_to_rport(portptr); 923 unsigned long flags; 924 925 spin_lock_irqsave(&rport->lock, flags); 926 927 if (portptr->port_state != FC_OBJSTATE_ONLINE) { 928 spin_unlock_irqrestore(&rport->lock, flags); 929 return -EINVAL; 930 } 931 932 /* a dev_loss_tmo of 0 (immediate) is allowed to be set */ 933 rport->remoteport.dev_loss_tmo = dev_loss_tmo; 934 935 spin_unlock_irqrestore(&rport->lock, flags); 936 937 return 0; 938 } 939 EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss); 940 941 942 /* *********************** FC-NVME DMA Handling **************************** */ 943 944 /* 945 * The fcloop device passes in a NULL device pointer. Real LLD's will 946 * pass in a valid device pointer. If NULL is passed to the dma mapping 947 * routines, depending on the platform, it may or may not succeed, and 948 * may crash. 949 * 950 * As such: 951 * Wrapper all the dma routines and check the dev pointer. 952 * 953 * If simple mappings (return just a dma address, we'll noop them, 954 * returning a dma address of 0. 955 * 956 * On more complex mappings (dma_map_sg), a pseudo routine fills 957 * in the scatter list, setting all dma addresses to 0. 958 */ 959 960 static inline dma_addr_t 961 fc_dma_map_single(struct device *dev, void *ptr, size_t size, 962 enum dma_data_direction dir) 963 { 964 return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L; 965 } 966 967 static inline int 968 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 969 { 970 return dev ? dma_mapping_error(dev, dma_addr) : 0; 971 } 972 973 static inline void 974 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size, 975 enum dma_data_direction dir) 976 { 977 if (dev) 978 dma_unmap_single(dev, addr, size, dir); 979 } 980 981 static inline void 982 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, 983 enum dma_data_direction dir) 984 { 985 if (dev) 986 dma_sync_single_for_cpu(dev, addr, size, dir); 987 } 988 989 static inline void 990 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size, 991 enum dma_data_direction dir) 992 { 993 if (dev) 994 dma_sync_single_for_device(dev, addr, size, dir); 995 } 996 997 /* pseudo dma_map_sg call */ 998 static int 999 fc_map_sg(struct scatterlist *sg, int nents) 1000 { 1001 struct scatterlist *s; 1002 int i; 1003 1004 WARN_ON(nents == 0 || sg[0].length == 0); 1005 1006 for_each_sg(sg, s, nents, i) { 1007 s->dma_address = 0L; 1008 #ifdef CONFIG_NEED_SG_DMA_LENGTH 1009 s->dma_length = s->length; 1010 #endif 1011 } 1012 return nents; 1013 } 1014 1015 static inline int 1016 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 1017 enum dma_data_direction dir) 1018 { 1019 return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents); 1020 } 1021 1022 static inline void 1023 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 1024 enum dma_data_direction dir) 1025 { 1026 if (dev) 1027 dma_unmap_sg(dev, sg, nents, dir); 1028 } 1029 1030 /* *********************** FC-NVME LS Handling **************************** */ 1031 1032 static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *); 1033 static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *); 1034 1035 static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg); 1036 1037 static void 1038 __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop) 1039 { 1040 struct nvme_fc_rport *rport = lsop->rport; 1041 struct nvmefc_ls_req *lsreq = &lsop->ls_req; 1042 unsigned long flags; 1043 1044 spin_lock_irqsave(&rport->lock, flags); 1045 1046 if (!lsop->req_queued) { 1047 spin_unlock_irqrestore(&rport->lock, flags); 1048 return; 1049 } 1050 1051 list_del(&lsop->lsreq_list); 1052 1053 lsop->req_queued = false; 1054 1055 spin_unlock_irqrestore(&rport->lock, flags); 1056 1057 fc_dma_unmap_single(rport->dev, lsreq->rqstdma, 1058 (lsreq->rqstlen + lsreq->rsplen), 1059 DMA_BIDIRECTIONAL); 1060 1061 nvme_fc_rport_put(rport); 1062 } 1063 1064 static int 1065 __nvme_fc_send_ls_req(struct nvme_fc_rport *rport, 1066 struct nvmefc_ls_req_op *lsop, 1067 void (*done)(struct nvmefc_ls_req *req, int status)) 1068 { 1069 struct nvmefc_ls_req *lsreq = &lsop->ls_req; 1070 unsigned long flags; 1071 int ret = 0; 1072 1073 if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) 1074 return -ECONNREFUSED; 1075 1076 if (!nvme_fc_rport_get(rport)) 1077 return -ESHUTDOWN; 1078 1079 lsreq->done = done; 1080 lsop->rport = rport; 1081 lsop->req_queued = false; 1082 INIT_LIST_HEAD(&lsop->lsreq_list); 1083 init_completion(&lsop->ls_done); 1084 1085 lsreq->rqstdma = fc_dma_map_single(rport->dev, lsreq->rqstaddr, 1086 lsreq->rqstlen + lsreq->rsplen, 1087 DMA_BIDIRECTIONAL); 1088 if (fc_dma_mapping_error(rport->dev, lsreq->rqstdma)) { 1089 ret = -EFAULT; 1090 goto out_putrport; 1091 } 1092 lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen; 1093 1094 spin_lock_irqsave(&rport->lock, flags); 1095 1096 list_add_tail(&lsop->lsreq_list, &rport->ls_req_list); 1097 1098 lsop->req_queued = true; 1099 1100 spin_unlock_irqrestore(&rport->lock, flags); 1101 1102 ret = rport->lport->ops->ls_req(&rport->lport->localport, 1103 &rport->remoteport, lsreq); 1104 if (ret) 1105 goto out_unlink; 1106 1107 return 0; 1108 1109 out_unlink: 1110 lsop->ls_error = ret; 1111 spin_lock_irqsave(&rport->lock, flags); 1112 lsop->req_queued = false; 1113 list_del(&lsop->lsreq_list); 1114 spin_unlock_irqrestore(&rport->lock, flags); 1115 fc_dma_unmap_single(rport->dev, lsreq->rqstdma, 1116 (lsreq->rqstlen + lsreq->rsplen), 1117 DMA_BIDIRECTIONAL); 1118 out_putrport: 1119 nvme_fc_rport_put(rport); 1120 1121 return ret; 1122 } 1123 1124 static void 1125 nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status) 1126 { 1127 struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq); 1128 1129 lsop->ls_error = status; 1130 complete(&lsop->ls_done); 1131 } 1132 1133 static int 1134 nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop) 1135 { 1136 struct nvmefc_ls_req *lsreq = &lsop->ls_req; 1137 struct fcnvme_ls_rjt *rjt = lsreq->rspaddr; 1138 int ret; 1139 1140 ret = __nvme_fc_send_ls_req(rport, lsop, nvme_fc_send_ls_req_done); 1141 1142 if (!ret) { 1143 /* 1144 * No timeout/not interruptible as we need the struct 1145 * to exist until the lldd calls us back. Thus mandate 1146 * wait until driver calls back. lldd responsible for 1147 * the timeout action 1148 */ 1149 wait_for_completion(&lsop->ls_done); 1150 1151 __nvme_fc_finish_ls_req(lsop); 1152 1153 ret = lsop->ls_error; 1154 } 1155 1156 if (ret) 1157 return ret; 1158 1159 /* ACC or RJT payload ? */ 1160 if (rjt->w0.ls_cmd == FCNVME_LS_RJT) 1161 return -ENXIO; 1162 1163 return 0; 1164 } 1165 1166 static int 1167 nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport, 1168 struct nvmefc_ls_req_op *lsop, 1169 void (*done)(struct nvmefc_ls_req *req, int status)) 1170 { 1171 /* don't wait for completion */ 1172 1173 return __nvme_fc_send_ls_req(rport, lsop, done); 1174 } 1175 1176 static int 1177 nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl, 1178 struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio) 1179 { 1180 struct nvmefc_ls_req_op *lsop; 1181 struct nvmefc_ls_req *lsreq; 1182 struct fcnvme_ls_cr_assoc_rqst *assoc_rqst; 1183 struct fcnvme_ls_cr_assoc_acc *assoc_acc; 1184 unsigned long flags; 1185 int ret, fcret = 0; 1186 1187 lsop = kzalloc((sizeof(*lsop) + 1188 sizeof(*assoc_rqst) + sizeof(*assoc_acc) + 1189 ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL); 1190 if (!lsop) { 1191 dev_info(ctrl->ctrl.device, 1192 "NVME-FC{%d}: send Create Association failed: ENOMEM\n", 1193 ctrl->cnum); 1194 ret = -ENOMEM; 1195 goto out_no_memory; 1196 } 1197 1198 assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)&lsop[1]; 1199 assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1]; 1200 lsreq = &lsop->ls_req; 1201 if (ctrl->lport->ops->lsrqst_priv_sz) 1202 lsreq->private = &assoc_acc[1]; 1203 else 1204 lsreq->private = NULL; 1205 1206 assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION; 1207 assoc_rqst->desc_list_len = 1208 cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd)); 1209 1210 assoc_rqst->assoc_cmd.desc_tag = 1211 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD); 1212 assoc_rqst->assoc_cmd.desc_len = 1213 fcnvme_lsdesc_len( 1214 sizeof(struct fcnvme_lsdesc_cr_assoc_cmd)); 1215 1216 assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio); 1217 assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1); 1218 /* Linux supports only Dynamic controllers */ 1219 assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff); 1220 uuid_copy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id); 1221 strscpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn, 1222 sizeof(assoc_rqst->assoc_cmd.hostnqn)); 1223 strscpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn, 1224 sizeof(assoc_rqst->assoc_cmd.subnqn)); 1225 1226 lsop->queue = queue; 1227 lsreq->rqstaddr = assoc_rqst; 1228 lsreq->rqstlen = sizeof(*assoc_rqst); 1229 lsreq->rspaddr = assoc_acc; 1230 lsreq->rsplen = sizeof(*assoc_acc); 1231 lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC; 1232 1233 ret = nvme_fc_send_ls_req(ctrl->rport, lsop); 1234 if (ret) 1235 goto out_free_buffer; 1236 1237 /* process connect LS completion */ 1238 1239 /* validate the ACC response */ 1240 if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC) 1241 fcret = VERR_LSACC; 1242 else if (assoc_acc->hdr.desc_list_len != 1243 fcnvme_lsdesc_len( 1244 sizeof(struct fcnvme_ls_cr_assoc_acc))) 1245 fcret = VERR_CR_ASSOC_ACC_LEN; 1246 else if (assoc_acc->hdr.rqst.desc_tag != 1247 cpu_to_be32(FCNVME_LSDESC_RQST)) 1248 fcret = VERR_LSDESC_RQST; 1249 else if (assoc_acc->hdr.rqst.desc_len != 1250 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst))) 1251 fcret = VERR_LSDESC_RQST_LEN; 1252 else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION) 1253 fcret = VERR_CR_ASSOC; 1254 else if (assoc_acc->associd.desc_tag != 1255 cpu_to_be32(FCNVME_LSDESC_ASSOC_ID)) 1256 fcret = VERR_ASSOC_ID; 1257 else if (assoc_acc->associd.desc_len != 1258 fcnvme_lsdesc_len( 1259 sizeof(struct fcnvme_lsdesc_assoc_id))) 1260 fcret = VERR_ASSOC_ID_LEN; 1261 else if (assoc_acc->connectid.desc_tag != 1262 cpu_to_be32(FCNVME_LSDESC_CONN_ID)) 1263 fcret = VERR_CONN_ID; 1264 else if (assoc_acc->connectid.desc_len != 1265 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id))) 1266 fcret = VERR_CONN_ID_LEN; 1267 1268 if (fcret) { 1269 ret = -EBADF; 1270 dev_err(ctrl->dev, 1271 "q %d Create Association LS failed: %s\n", 1272 queue->qnum, validation_errors[fcret]); 1273 } else { 1274 spin_lock_irqsave(&ctrl->lock, flags); 1275 ctrl->association_id = 1276 be64_to_cpu(assoc_acc->associd.association_id); 1277 queue->connection_id = 1278 be64_to_cpu(assoc_acc->connectid.connection_id); 1279 set_bit(NVME_FC_Q_CONNECTED, &queue->flags); 1280 spin_unlock_irqrestore(&ctrl->lock, flags); 1281 } 1282 1283 out_free_buffer: 1284 kfree(lsop); 1285 out_no_memory: 1286 if (ret) 1287 dev_err(ctrl->dev, 1288 "queue %d connect admin queue failed (%d).\n", 1289 queue->qnum, ret); 1290 return ret; 1291 } 1292 1293 static int 1294 nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, 1295 u16 qsize, u16 ersp_ratio) 1296 { 1297 struct nvmefc_ls_req_op *lsop; 1298 struct nvmefc_ls_req *lsreq; 1299 struct fcnvme_ls_cr_conn_rqst *conn_rqst; 1300 struct fcnvme_ls_cr_conn_acc *conn_acc; 1301 int ret, fcret = 0; 1302 1303 lsop = kzalloc((sizeof(*lsop) + 1304 sizeof(*conn_rqst) + sizeof(*conn_acc) + 1305 ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL); 1306 if (!lsop) { 1307 dev_info(ctrl->ctrl.device, 1308 "NVME-FC{%d}: send Create Connection failed: ENOMEM\n", 1309 ctrl->cnum); 1310 ret = -ENOMEM; 1311 goto out_no_memory; 1312 } 1313 1314 conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)&lsop[1]; 1315 conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1]; 1316 lsreq = &lsop->ls_req; 1317 if (ctrl->lport->ops->lsrqst_priv_sz) 1318 lsreq->private = (void *)&conn_acc[1]; 1319 else 1320 lsreq->private = NULL; 1321 1322 conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION; 1323 conn_rqst->desc_list_len = cpu_to_be32( 1324 sizeof(struct fcnvme_lsdesc_assoc_id) + 1325 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)); 1326 1327 conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID); 1328 conn_rqst->associd.desc_len = 1329 fcnvme_lsdesc_len( 1330 sizeof(struct fcnvme_lsdesc_assoc_id)); 1331 conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id); 1332 conn_rqst->connect_cmd.desc_tag = 1333 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD); 1334 conn_rqst->connect_cmd.desc_len = 1335 fcnvme_lsdesc_len( 1336 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)); 1337 conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio); 1338 conn_rqst->connect_cmd.qid = cpu_to_be16(queue->qnum); 1339 conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1); 1340 1341 lsop->queue = queue; 1342 lsreq->rqstaddr = conn_rqst; 1343 lsreq->rqstlen = sizeof(*conn_rqst); 1344 lsreq->rspaddr = conn_acc; 1345 lsreq->rsplen = sizeof(*conn_acc); 1346 lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC; 1347 1348 ret = nvme_fc_send_ls_req(ctrl->rport, lsop); 1349 if (ret) 1350 goto out_free_buffer; 1351 1352 /* process connect LS completion */ 1353 1354 /* validate the ACC response */ 1355 if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC) 1356 fcret = VERR_LSACC; 1357 else if (conn_acc->hdr.desc_list_len != 1358 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc))) 1359 fcret = VERR_CR_CONN_ACC_LEN; 1360 else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST)) 1361 fcret = VERR_LSDESC_RQST; 1362 else if (conn_acc->hdr.rqst.desc_len != 1363 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst))) 1364 fcret = VERR_LSDESC_RQST_LEN; 1365 else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION) 1366 fcret = VERR_CR_CONN; 1367 else if (conn_acc->connectid.desc_tag != 1368 cpu_to_be32(FCNVME_LSDESC_CONN_ID)) 1369 fcret = VERR_CONN_ID; 1370 else if (conn_acc->connectid.desc_len != 1371 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id))) 1372 fcret = VERR_CONN_ID_LEN; 1373 1374 if (fcret) { 1375 ret = -EBADF; 1376 dev_err(ctrl->dev, 1377 "q %d Create I/O Connection LS failed: %s\n", 1378 queue->qnum, validation_errors[fcret]); 1379 } else { 1380 queue->connection_id = 1381 be64_to_cpu(conn_acc->connectid.connection_id); 1382 set_bit(NVME_FC_Q_CONNECTED, &queue->flags); 1383 } 1384 1385 out_free_buffer: 1386 kfree(lsop); 1387 out_no_memory: 1388 if (ret) 1389 dev_err(ctrl->dev, 1390 "queue %d connect I/O queue failed (%d).\n", 1391 queue->qnum, ret); 1392 return ret; 1393 } 1394 1395 static void 1396 nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status) 1397 { 1398 struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq); 1399 1400 __nvme_fc_finish_ls_req(lsop); 1401 1402 /* fc-nvme initiator doesn't care about success or failure of cmd */ 1403 1404 kfree(lsop); 1405 } 1406 1407 /* 1408 * This routine sends a FC-NVME LS to disconnect (aka terminate) 1409 * the FC-NVME Association. Terminating the association also 1410 * terminates the FC-NVME connections (per queue, both admin and io 1411 * queues) that are part of the association. E.g. things are torn 1412 * down, and the related FC-NVME Association ID and Connection IDs 1413 * become invalid. 1414 * 1415 * The behavior of the fc-nvme initiator is such that it's 1416 * understanding of the association and connections will implicitly 1417 * be torn down. The action is implicit as it may be due to a loss of 1418 * connectivity with the fc-nvme target, so you may never get a 1419 * response even if you tried. As such, the action of this routine 1420 * is to asynchronously send the LS, ignore any results of the LS, and 1421 * continue on with terminating the association. If the fc-nvme target 1422 * is present and receives the LS, it too can tear down. 1423 */ 1424 static void 1425 nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl) 1426 { 1427 struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst; 1428 struct fcnvme_ls_disconnect_assoc_acc *discon_acc; 1429 struct nvmefc_ls_req_op *lsop; 1430 struct nvmefc_ls_req *lsreq; 1431 int ret; 1432 1433 lsop = kzalloc((sizeof(*lsop) + 1434 sizeof(*discon_rqst) + sizeof(*discon_acc) + 1435 ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL); 1436 if (!lsop) { 1437 dev_info(ctrl->ctrl.device, 1438 "NVME-FC{%d}: send Disconnect Association " 1439 "failed: ENOMEM\n", 1440 ctrl->cnum); 1441 return; 1442 } 1443 1444 discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1]; 1445 discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1]; 1446 lsreq = &lsop->ls_req; 1447 if (ctrl->lport->ops->lsrqst_priv_sz) 1448 lsreq->private = (void *)&discon_acc[1]; 1449 else 1450 lsreq->private = NULL; 1451 1452 nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc, 1453 ctrl->association_id); 1454 1455 ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop, 1456 nvme_fc_disconnect_assoc_done); 1457 if (ret) 1458 kfree(lsop); 1459 } 1460 1461 static void 1462 nvme_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp) 1463 { 1464 struct nvmefc_ls_rcv_op *lsop = lsrsp->nvme_fc_private; 1465 struct nvme_fc_rport *rport = lsop->rport; 1466 struct nvme_fc_lport *lport = rport->lport; 1467 unsigned long flags; 1468 1469 spin_lock_irqsave(&rport->lock, flags); 1470 list_del(&lsop->lsrcv_list); 1471 spin_unlock_irqrestore(&rport->lock, flags); 1472 1473 fc_dma_sync_single_for_cpu(lport->dev, lsop->rspdma, 1474 sizeof(*lsop->rspbuf), DMA_TO_DEVICE); 1475 fc_dma_unmap_single(lport->dev, lsop->rspdma, 1476 sizeof(*lsop->rspbuf), DMA_TO_DEVICE); 1477 1478 kfree(lsop->rspbuf); 1479 kfree(lsop->rqstbuf); 1480 kfree(lsop); 1481 1482 nvme_fc_rport_put(rport); 1483 } 1484 1485 static void 1486 nvme_fc_xmt_ls_rsp(struct nvmefc_ls_rcv_op *lsop) 1487 { 1488 struct nvme_fc_rport *rport = lsop->rport; 1489 struct nvme_fc_lport *lport = rport->lport; 1490 struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0; 1491 int ret; 1492 1493 fc_dma_sync_single_for_device(lport->dev, lsop->rspdma, 1494 sizeof(*lsop->rspbuf), DMA_TO_DEVICE); 1495 1496 ret = lport->ops->xmt_ls_rsp(&lport->localport, &rport->remoteport, 1497 lsop->lsrsp); 1498 if (ret) { 1499 dev_warn(lport->dev, 1500 "LLDD rejected LS RSP xmt: LS %d status %d\n", 1501 w0->ls_cmd, ret); 1502 nvme_fc_xmt_ls_rsp_done(lsop->lsrsp); 1503 return; 1504 } 1505 } 1506 1507 static struct nvme_fc_ctrl * 1508 nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport, 1509 struct nvmefc_ls_rcv_op *lsop) 1510 { 1511 struct fcnvme_ls_disconnect_assoc_rqst *rqst = 1512 &lsop->rqstbuf->rq_dis_assoc; 1513 struct nvme_fc_ctrl *ctrl, *ret = NULL; 1514 struct nvmefc_ls_rcv_op *oldls = NULL; 1515 u64 association_id = be64_to_cpu(rqst->associd.association_id); 1516 unsigned long flags; 1517 1518 spin_lock_irqsave(&rport->lock, flags); 1519 1520 list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) { 1521 if (!nvme_fc_ctrl_get(ctrl)) 1522 continue; 1523 spin_lock(&ctrl->lock); 1524 if (association_id == ctrl->association_id) { 1525 oldls = ctrl->rcv_disconn; 1526 ctrl->rcv_disconn = lsop; 1527 ret = ctrl; 1528 } 1529 spin_unlock(&ctrl->lock); 1530 if (ret) 1531 /* leave the ctrl get reference */ 1532 break; 1533 nvme_fc_ctrl_put(ctrl); 1534 } 1535 1536 spin_unlock_irqrestore(&rport->lock, flags); 1537 1538 /* transmit a response for anything that was pending */ 1539 if (oldls) { 1540 dev_info(rport->lport->dev, 1541 "NVME-FC{%d}: Multiple Disconnect Association " 1542 "LS's received\n", ctrl->cnum); 1543 /* overwrite good response with bogus failure */ 1544 oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf, 1545 sizeof(*oldls->rspbuf), 1546 rqst->w0.ls_cmd, 1547 FCNVME_RJT_RC_UNAB, 1548 FCNVME_RJT_EXP_NONE, 0); 1549 nvme_fc_xmt_ls_rsp(oldls); 1550 } 1551 1552 return ret; 1553 } 1554 1555 /* 1556 * returns true to mean LS handled and ls_rsp can be sent 1557 * returns false to defer ls_rsp xmt (will be done as part of 1558 * association termination) 1559 */ 1560 static bool 1561 nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op *lsop) 1562 { 1563 struct nvme_fc_rport *rport = lsop->rport; 1564 struct fcnvme_ls_disconnect_assoc_rqst *rqst = 1565 &lsop->rqstbuf->rq_dis_assoc; 1566 struct fcnvme_ls_disconnect_assoc_acc *acc = 1567 &lsop->rspbuf->rsp_dis_assoc; 1568 struct nvme_fc_ctrl *ctrl = NULL; 1569 int ret = 0; 1570 1571 memset(acc, 0, sizeof(*acc)); 1572 1573 ret = nvmefc_vldt_lsreq_discon_assoc(lsop->rqstdatalen, rqst); 1574 if (!ret) { 1575 /* match an active association */ 1576 ctrl = nvme_fc_match_disconn_ls(rport, lsop); 1577 if (!ctrl) 1578 ret = VERR_NO_ASSOC; 1579 } 1580 1581 if (ret) { 1582 dev_info(rport->lport->dev, 1583 "Disconnect LS failed: %s\n", 1584 validation_errors[ret]); 1585 lsop->lsrsp->rsplen = nvme_fc_format_rjt(acc, 1586 sizeof(*acc), rqst->w0.ls_cmd, 1587 (ret == VERR_NO_ASSOC) ? 1588 FCNVME_RJT_RC_INV_ASSOC : 1589 FCNVME_RJT_RC_LOGIC, 1590 FCNVME_RJT_EXP_NONE, 0); 1591 return true; 1592 } 1593 1594 /* format an ACCept response */ 1595 1596 lsop->lsrsp->rsplen = sizeof(*acc); 1597 1598 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, 1599 fcnvme_lsdesc_len( 1600 sizeof(struct fcnvme_ls_disconnect_assoc_acc)), 1601 FCNVME_LS_DISCONNECT_ASSOC); 1602 1603 /* 1604 * the transmit of the response will occur after the exchanges 1605 * for the association have been ABTS'd by 1606 * nvme_fc_delete_association(). 1607 */ 1608 1609 /* fail the association */ 1610 nvme_fc_error_recovery(ctrl, "Disconnect Association LS received"); 1611 1612 /* release the reference taken by nvme_fc_match_disconn_ls() */ 1613 nvme_fc_ctrl_put(ctrl); 1614 1615 return false; 1616 } 1617 1618 /* 1619 * Actual Processing routine for received FC-NVME LS Requests from the LLD 1620 * returns true if a response should be sent afterward, false if rsp will 1621 * be sent asynchronously. 1622 */ 1623 static bool 1624 nvme_fc_handle_ls_rqst(struct nvmefc_ls_rcv_op *lsop) 1625 { 1626 struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0; 1627 bool ret = true; 1628 1629 lsop->lsrsp->nvme_fc_private = lsop; 1630 lsop->lsrsp->rspbuf = lsop->rspbuf; 1631 lsop->lsrsp->rspdma = lsop->rspdma; 1632 lsop->lsrsp->done = nvme_fc_xmt_ls_rsp_done; 1633 /* Be preventative. handlers will later set to valid length */ 1634 lsop->lsrsp->rsplen = 0; 1635 1636 /* 1637 * handlers: 1638 * parse request input, execute the request, and format the 1639 * LS response 1640 */ 1641 switch (w0->ls_cmd) { 1642 case FCNVME_LS_DISCONNECT_ASSOC: 1643 ret = nvme_fc_ls_disconnect_assoc(lsop); 1644 break; 1645 case FCNVME_LS_DISCONNECT_CONN: 1646 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf, 1647 sizeof(*lsop->rspbuf), w0->ls_cmd, 1648 FCNVME_RJT_RC_UNSUP, FCNVME_RJT_EXP_NONE, 0); 1649 break; 1650 case FCNVME_LS_CREATE_ASSOCIATION: 1651 case FCNVME_LS_CREATE_CONNECTION: 1652 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf, 1653 sizeof(*lsop->rspbuf), w0->ls_cmd, 1654 FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0); 1655 break; 1656 default: 1657 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf, 1658 sizeof(*lsop->rspbuf), w0->ls_cmd, 1659 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0); 1660 break; 1661 } 1662 1663 return(ret); 1664 } 1665 1666 static void 1667 nvme_fc_handle_ls_rqst_work(struct work_struct *work) 1668 { 1669 struct nvme_fc_rport *rport = 1670 container_of(work, struct nvme_fc_rport, lsrcv_work); 1671 struct fcnvme_ls_rqst_w0 *w0; 1672 struct nvmefc_ls_rcv_op *lsop; 1673 unsigned long flags; 1674 bool sendrsp; 1675 1676 restart: 1677 sendrsp = true; 1678 spin_lock_irqsave(&rport->lock, flags); 1679 list_for_each_entry(lsop, &rport->ls_rcv_list, lsrcv_list) { 1680 if (lsop->handled) 1681 continue; 1682 1683 lsop->handled = true; 1684 if (rport->remoteport.port_state == FC_OBJSTATE_ONLINE) { 1685 spin_unlock_irqrestore(&rport->lock, flags); 1686 sendrsp = nvme_fc_handle_ls_rqst(lsop); 1687 } else { 1688 spin_unlock_irqrestore(&rport->lock, flags); 1689 w0 = &lsop->rqstbuf->w0; 1690 lsop->lsrsp->rsplen = nvme_fc_format_rjt( 1691 lsop->rspbuf, 1692 sizeof(*lsop->rspbuf), 1693 w0->ls_cmd, 1694 FCNVME_RJT_RC_UNAB, 1695 FCNVME_RJT_EXP_NONE, 0); 1696 } 1697 if (sendrsp) 1698 nvme_fc_xmt_ls_rsp(lsop); 1699 goto restart; 1700 } 1701 spin_unlock_irqrestore(&rport->lock, flags); 1702 } 1703 1704 static 1705 void nvme_fc_rcv_ls_req_err_msg(struct nvme_fc_lport *lport, 1706 struct fcnvme_ls_rqst_w0 *w0) 1707 { 1708 dev_info(lport->dev, "RCV %s LS failed: No memory\n", 1709 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? 1710 nvmefc_ls_names[w0->ls_cmd] : ""); 1711 } 1712 1713 /** 1714 * nvme_fc_rcv_ls_req - transport entry point called by an LLDD 1715 * upon the reception of a NVME LS request. 1716 * 1717 * The nvme-fc layer will copy payload to an internal structure for 1718 * processing. As such, upon completion of the routine, the LLDD may 1719 * immediately free/reuse the LS request buffer passed in the call. 1720 * 1721 * If this routine returns error, the LLDD should abort the exchange. 1722 * 1723 * @portptr: pointer to the (registered) remote port that the LS 1724 * was received from. The remoteport is associated with 1725 * a specific localport. 1726 * @lsrsp: pointer to a nvmefc_ls_rsp response structure to be 1727 * used to reference the exchange corresponding to the LS 1728 * when issuing an ls response. 1729 * @lsreqbuf: pointer to the buffer containing the LS Request 1730 * @lsreqbuf_len: length, in bytes, of the received LS request 1731 */ 1732 int 1733 nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr, 1734 struct nvmefc_ls_rsp *lsrsp, 1735 void *lsreqbuf, u32 lsreqbuf_len) 1736 { 1737 struct nvme_fc_rport *rport = remoteport_to_rport(portptr); 1738 struct nvme_fc_lport *lport = rport->lport; 1739 struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf; 1740 struct nvmefc_ls_rcv_op *lsop; 1741 unsigned long flags; 1742 int ret; 1743 1744 nvme_fc_rport_get(rport); 1745 1746 /* validate there's a routine to transmit a response */ 1747 if (!lport->ops->xmt_ls_rsp) { 1748 dev_info(lport->dev, 1749 "RCV %s LS failed: no LLDD xmt_ls_rsp\n", 1750 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? 1751 nvmefc_ls_names[w0->ls_cmd] : ""); 1752 ret = -EINVAL; 1753 goto out_put; 1754 } 1755 1756 if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) { 1757 dev_info(lport->dev, 1758 "RCV %s LS failed: payload too large\n", 1759 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? 1760 nvmefc_ls_names[w0->ls_cmd] : ""); 1761 ret = -E2BIG; 1762 goto out_put; 1763 } 1764 1765 lsop = kzalloc(sizeof(*lsop), GFP_KERNEL); 1766 if (!lsop) { 1767 nvme_fc_rcv_ls_req_err_msg(lport, w0); 1768 ret = -ENOMEM; 1769 goto out_put; 1770 } 1771 1772 lsop->rqstbuf = kzalloc(sizeof(*lsop->rqstbuf), GFP_KERNEL); 1773 lsop->rspbuf = kzalloc(sizeof(*lsop->rspbuf), GFP_KERNEL); 1774 if (!lsop->rqstbuf || !lsop->rspbuf) { 1775 nvme_fc_rcv_ls_req_err_msg(lport, w0); 1776 ret = -ENOMEM; 1777 goto out_free; 1778 } 1779 1780 lsop->rspdma = fc_dma_map_single(lport->dev, lsop->rspbuf, 1781 sizeof(*lsop->rspbuf), 1782 DMA_TO_DEVICE); 1783 if (fc_dma_mapping_error(lport->dev, lsop->rspdma)) { 1784 dev_info(lport->dev, 1785 "RCV %s LS failed: DMA mapping failure\n", 1786 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? 1787 nvmefc_ls_names[w0->ls_cmd] : ""); 1788 ret = -EFAULT; 1789 goto out_free; 1790 } 1791 1792 lsop->rport = rport; 1793 lsop->lsrsp = lsrsp; 1794 1795 memcpy(lsop->rqstbuf, lsreqbuf, lsreqbuf_len); 1796 lsop->rqstdatalen = lsreqbuf_len; 1797 1798 spin_lock_irqsave(&rport->lock, flags); 1799 if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) { 1800 spin_unlock_irqrestore(&rport->lock, flags); 1801 ret = -ENOTCONN; 1802 goto out_unmap; 1803 } 1804 list_add_tail(&lsop->lsrcv_list, &rport->ls_rcv_list); 1805 spin_unlock_irqrestore(&rport->lock, flags); 1806 1807 schedule_work(&rport->lsrcv_work); 1808 1809 return 0; 1810 1811 out_unmap: 1812 fc_dma_unmap_single(lport->dev, lsop->rspdma, 1813 sizeof(*lsop->rspbuf), DMA_TO_DEVICE); 1814 out_free: 1815 kfree(lsop->rspbuf); 1816 kfree(lsop->rqstbuf); 1817 kfree(lsop); 1818 out_put: 1819 nvme_fc_rport_put(rport); 1820 return ret; 1821 } 1822 EXPORT_SYMBOL_GPL(nvme_fc_rcv_ls_req); 1823 1824 1825 /* *********************** NVME Ctrl Routines **************************** */ 1826 1827 static void 1828 __nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl, 1829 struct nvme_fc_fcp_op *op) 1830 { 1831 fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.rspdma, 1832 sizeof(op->rsp_iu), DMA_FROM_DEVICE); 1833 fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma, 1834 sizeof(op->cmd_iu), DMA_TO_DEVICE); 1835 1836 atomic_set(&op->state, FCPOP_STATE_UNINIT); 1837 } 1838 1839 static void 1840 nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq, 1841 unsigned int hctx_idx) 1842 { 1843 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); 1844 1845 return __nvme_fc_exit_request(to_fc_ctrl(set->driver_data), op); 1846 } 1847 1848 static int 1849 __nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op) 1850 { 1851 unsigned long flags; 1852 int opstate; 1853 1854 spin_lock_irqsave(&ctrl->lock, flags); 1855 opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED); 1856 if (opstate != FCPOP_STATE_ACTIVE) 1857 atomic_set(&op->state, opstate); 1858 else if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) { 1859 op->flags |= FCOP_FLAGS_TERMIO; 1860 ctrl->iocnt++; 1861 } 1862 spin_unlock_irqrestore(&ctrl->lock, flags); 1863 1864 if (opstate != FCPOP_STATE_ACTIVE) 1865 return -ECANCELED; 1866 1867 ctrl->lport->ops->fcp_abort(&ctrl->lport->localport, 1868 &ctrl->rport->remoteport, 1869 op->queue->lldd_handle, 1870 &op->fcp_req); 1871 1872 return 0; 1873 } 1874 1875 static void 1876 nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl) 1877 { 1878 struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops; 1879 int i; 1880 1881 /* ensure we've initialized the ops once */ 1882 if (!(aen_op->flags & FCOP_FLAGS_AEN)) 1883 return; 1884 1885 for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) 1886 __nvme_fc_abort_op(ctrl, aen_op); 1887 } 1888 1889 static inline void 1890 __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl, 1891 struct nvme_fc_fcp_op *op, int opstate) 1892 { 1893 unsigned long flags; 1894 1895 if (opstate == FCPOP_STATE_ABORTED) { 1896 spin_lock_irqsave(&ctrl->lock, flags); 1897 if (test_bit(FCCTRL_TERMIO, &ctrl->flags) && 1898 op->flags & FCOP_FLAGS_TERMIO) { 1899 if (!--ctrl->iocnt) 1900 wake_up(&ctrl->ioabort_wait); 1901 } 1902 spin_unlock_irqrestore(&ctrl->lock, flags); 1903 } 1904 } 1905 1906 static void 1907 nvme_fc_ctrl_ioerr_work(struct work_struct *work) 1908 { 1909 struct nvme_fc_ctrl *ctrl = 1910 container_of(work, struct nvme_fc_ctrl, ioerr_work); 1911 1912 nvme_fc_error_recovery(ctrl, "transport detected io error"); 1913 } 1914 1915 /* 1916 * nvme_fc_io_getuuid - Routine called to get the appid field 1917 * associated with request by the lldd 1918 * @req:IO request from nvme fc to driver 1919 * Returns: UUID if there is an appid associated with VM or 1920 * NULL if the user/libvirt has not set the appid to VM 1921 */ 1922 char *nvme_fc_io_getuuid(struct nvmefc_fcp_req *req) 1923 { 1924 struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req); 1925 struct request *rq = op->rq; 1926 1927 if (!IS_ENABLED(CONFIG_BLK_CGROUP_FC_APPID) || !rq || !rq->bio) 1928 return NULL; 1929 return blkcg_get_fc_appid(rq->bio); 1930 } 1931 EXPORT_SYMBOL_GPL(nvme_fc_io_getuuid); 1932 1933 static void 1934 nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) 1935 { 1936 struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req); 1937 struct request *rq = op->rq; 1938 struct nvmefc_fcp_req *freq = &op->fcp_req; 1939 struct nvme_fc_ctrl *ctrl = op->ctrl; 1940 struct nvme_fc_queue *queue = op->queue; 1941 struct nvme_completion *cqe = &op->rsp_iu.cqe; 1942 struct nvme_command *sqe = &op->cmd_iu.sqe; 1943 __le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1); 1944 union nvme_result result; 1945 bool terminate_assoc = true; 1946 int opstate; 1947 1948 /* 1949 * WARNING: 1950 * The current linux implementation of a nvme controller 1951 * allocates a single tag set for all io queues and sizes 1952 * the io queues to fully hold all possible tags. Thus, the 1953 * implementation does not reference or care about the sqhd 1954 * value as it never needs to use the sqhd/sqtail pointers 1955 * for submission pacing. 1956 * 1957 * This affects the FC-NVME implementation in two ways: 1958 * 1) As the value doesn't matter, we don't need to waste 1959 * cycles extracting it from ERSPs and stamping it in the 1960 * cases where the transport fabricates CQEs on successful 1961 * completions. 1962 * 2) The FC-NVME implementation requires that delivery of 1963 * ERSP completions are to go back to the nvme layer in order 1964 * relative to the rsn, such that the sqhd value will always 1965 * be "in order" for the nvme layer. As the nvme layer in 1966 * linux doesn't care about sqhd, there's no need to return 1967 * them in order. 1968 * 1969 * Additionally: 1970 * As the core nvme layer in linux currently does not look at 1971 * every field in the cqe - in cases where the FC transport must 1972 * fabricate a CQE, the following fields will not be set as they 1973 * are not referenced: 1974 * cqe.sqid, cqe.sqhd, cqe.command_id 1975 * 1976 * Failure or error of an individual i/o, in a transport 1977 * detected fashion unrelated to the nvme completion status, 1978 * potentially cause the initiator and target sides to get out 1979 * of sync on SQ head/tail (aka outstanding io count allowed). 1980 * Per FC-NVME spec, failure of an individual command requires 1981 * the connection to be terminated, which in turn requires the 1982 * association to be terminated. 1983 */ 1984 1985 opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE); 1986 1987 fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma, 1988 sizeof(op->rsp_iu), DMA_FROM_DEVICE); 1989 1990 if (opstate == FCPOP_STATE_ABORTED) 1991 status = cpu_to_le16(NVME_SC_HOST_ABORTED_CMD << 1); 1992 else if (freq->status) { 1993 status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1); 1994 dev_info(ctrl->ctrl.device, 1995 "NVME-FC{%d}: io failed due to lldd error %d\n", 1996 ctrl->cnum, freq->status); 1997 } 1998 1999 /* 2000 * For the linux implementation, if we have an unsuccesful 2001 * status, they blk-mq layer can typically be called with the 2002 * non-zero status and the content of the cqe isn't important. 2003 */ 2004 if (status) 2005 goto done; 2006 2007 /* 2008 * command completed successfully relative to the wire 2009 * protocol. However, validate anything received and 2010 * extract the status and result from the cqe (create it 2011 * where necessary). 2012 */ 2013 2014 switch (freq->rcv_rsplen) { 2015 2016 case 0: 2017 case NVME_FC_SIZEOF_ZEROS_RSP: 2018 /* 2019 * No response payload or 12 bytes of payload (which 2020 * should all be zeros) are considered successful and 2021 * no payload in the CQE by the transport. 2022 */ 2023 if (freq->transferred_length != 2024 be32_to_cpu(op->cmd_iu.data_len)) { 2025 status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1); 2026 dev_info(ctrl->ctrl.device, 2027 "NVME-FC{%d}: io failed due to bad transfer " 2028 "length: %d vs expected %d\n", 2029 ctrl->cnum, freq->transferred_length, 2030 be32_to_cpu(op->cmd_iu.data_len)); 2031 goto done; 2032 } 2033 result.u64 = 0; 2034 break; 2035 2036 case sizeof(struct nvme_fc_ersp_iu): 2037 /* 2038 * The ERSP IU contains a full completion with CQE. 2039 * Validate ERSP IU and look at cqe. 2040 */ 2041 if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) != 2042 (freq->rcv_rsplen / 4) || 2043 be32_to_cpu(op->rsp_iu.xfrd_len) != 2044 freq->transferred_length || 2045 op->rsp_iu.ersp_result || 2046 sqe->common.command_id != cqe->command_id)) { 2047 status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1); 2048 dev_info(ctrl->ctrl.device, 2049 "NVME-FC{%d}: io failed due to bad NVMe_ERSP: " 2050 "iu len %d, xfr len %d vs %d, status code " 2051 "%d, cmdid %d vs %d\n", 2052 ctrl->cnum, be16_to_cpu(op->rsp_iu.iu_len), 2053 be32_to_cpu(op->rsp_iu.xfrd_len), 2054 freq->transferred_length, 2055 op->rsp_iu.ersp_result, 2056 sqe->common.command_id, 2057 cqe->command_id); 2058 goto done; 2059 } 2060 result = cqe->result; 2061 status = cqe->status; 2062 break; 2063 2064 default: 2065 status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1); 2066 dev_info(ctrl->ctrl.device, 2067 "NVME-FC{%d}: io failed due to odd NVMe_xRSP iu " 2068 "len %d\n", 2069 ctrl->cnum, freq->rcv_rsplen); 2070 goto done; 2071 } 2072 2073 terminate_assoc = false; 2074 2075 done: 2076 if (op->flags & FCOP_FLAGS_AEN) { 2077 nvme_complete_async_event(&queue->ctrl->ctrl, status, &result); 2078 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); 2079 atomic_set(&op->state, FCPOP_STATE_IDLE); 2080 op->flags = FCOP_FLAGS_AEN; /* clear other flags */ 2081 nvme_fc_ctrl_put(ctrl); 2082 goto check_error; 2083 } 2084 2085 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); 2086 if (!nvme_try_complete_req(rq, status, result)) 2087 nvme_fc_complete_rq(rq); 2088 2089 check_error: 2090 if (terminate_assoc && ctrl->ctrl.state != NVME_CTRL_RESETTING) 2091 queue_work(nvme_reset_wq, &ctrl->ioerr_work); 2092 } 2093 2094 static int 2095 __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl, 2096 struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op, 2097 struct request *rq, u32 rqno) 2098 { 2099 struct nvme_fcp_op_w_sgl *op_w_sgl = 2100 container_of(op, typeof(*op_w_sgl), op); 2101 struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu; 2102 int ret = 0; 2103 2104 memset(op, 0, sizeof(*op)); 2105 op->fcp_req.cmdaddr = &op->cmd_iu; 2106 op->fcp_req.cmdlen = sizeof(op->cmd_iu); 2107 op->fcp_req.rspaddr = &op->rsp_iu; 2108 op->fcp_req.rsplen = sizeof(op->rsp_iu); 2109 op->fcp_req.done = nvme_fc_fcpio_done; 2110 op->ctrl = ctrl; 2111 op->queue = queue; 2112 op->rq = rq; 2113 op->rqno = rqno; 2114 2115 cmdiu->format_id = NVME_CMD_FORMAT_ID; 2116 cmdiu->fc_id = NVME_CMD_FC_ID; 2117 cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32)); 2118 if (queue->qnum) 2119 cmdiu->rsv_cat = fccmnd_set_cat_css(0, 2120 (NVME_CC_CSS_NVM >> NVME_CC_CSS_SHIFT)); 2121 else 2122 cmdiu->rsv_cat = fccmnd_set_cat_admin(0); 2123 2124 op->fcp_req.cmddma = fc_dma_map_single(ctrl->lport->dev, 2125 &op->cmd_iu, sizeof(op->cmd_iu), DMA_TO_DEVICE); 2126 if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.cmddma)) { 2127 dev_err(ctrl->dev, 2128 "FCP Op failed - cmdiu dma mapping failed.\n"); 2129 ret = -EFAULT; 2130 goto out_on_error; 2131 } 2132 2133 op->fcp_req.rspdma = fc_dma_map_single(ctrl->lport->dev, 2134 &op->rsp_iu, sizeof(op->rsp_iu), 2135 DMA_FROM_DEVICE); 2136 if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.rspdma)) { 2137 dev_err(ctrl->dev, 2138 "FCP Op failed - rspiu dma mapping failed.\n"); 2139 ret = -EFAULT; 2140 } 2141 2142 atomic_set(&op->state, FCPOP_STATE_IDLE); 2143 out_on_error: 2144 return ret; 2145 } 2146 2147 static int 2148 nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq, 2149 unsigned int hctx_idx, unsigned int numa_node) 2150 { 2151 struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data); 2152 struct nvme_fcp_op_w_sgl *op = blk_mq_rq_to_pdu(rq); 2153 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; 2154 struct nvme_fc_queue *queue = &ctrl->queues[queue_idx]; 2155 int res; 2156 2157 res = __nvme_fc_init_request(ctrl, queue, &op->op, rq, queue->rqcnt++); 2158 if (res) 2159 return res; 2160 op->op.fcp_req.first_sgl = op->sgl; 2161 op->op.fcp_req.private = &op->priv[0]; 2162 nvme_req(rq)->ctrl = &ctrl->ctrl; 2163 nvme_req(rq)->cmd = &op->op.cmd_iu.sqe; 2164 return res; 2165 } 2166 2167 static int 2168 nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl) 2169 { 2170 struct nvme_fc_fcp_op *aen_op; 2171 struct nvme_fc_cmd_iu *cmdiu; 2172 struct nvme_command *sqe; 2173 void *private = NULL; 2174 int i, ret; 2175 2176 aen_op = ctrl->aen_ops; 2177 for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) { 2178 if (ctrl->lport->ops->fcprqst_priv_sz) { 2179 private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz, 2180 GFP_KERNEL); 2181 if (!private) 2182 return -ENOMEM; 2183 } 2184 2185 cmdiu = &aen_op->cmd_iu; 2186 sqe = &cmdiu->sqe; 2187 ret = __nvme_fc_init_request(ctrl, &ctrl->queues[0], 2188 aen_op, (struct request *)NULL, 2189 (NVME_AQ_BLK_MQ_DEPTH + i)); 2190 if (ret) { 2191 kfree(private); 2192 return ret; 2193 } 2194 2195 aen_op->flags = FCOP_FLAGS_AEN; 2196 aen_op->fcp_req.private = private; 2197 2198 memset(sqe, 0, sizeof(*sqe)); 2199 sqe->common.opcode = nvme_admin_async_event; 2200 /* Note: core layer may overwrite the sqe.command_id value */ 2201 sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i; 2202 } 2203 return 0; 2204 } 2205 2206 static void 2207 nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl) 2208 { 2209 struct nvme_fc_fcp_op *aen_op; 2210 int i; 2211 2212 cancel_work_sync(&ctrl->ctrl.async_event_work); 2213 aen_op = ctrl->aen_ops; 2214 for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) { 2215 __nvme_fc_exit_request(ctrl, aen_op); 2216 2217 kfree(aen_op->fcp_req.private); 2218 aen_op->fcp_req.private = NULL; 2219 } 2220 } 2221 2222 static inline int 2223 __nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int qidx) 2224 { 2225 struct nvme_fc_ctrl *ctrl = to_fc_ctrl(data); 2226 struct nvme_fc_queue *queue = &ctrl->queues[qidx]; 2227 2228 hctx->driver_data = queue; 2229 queue->hctx = hctx; 2230 return 0; 2231 } 2232 2233 static int 2234 nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) 2235 { 2236 return __nvme_fc_init_hctx(hctx, data, hctx_idx + 1); 2237 } 2238 2239 static int 2240 nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, 2241 unsigned int hctx_idx) 2242 { 2243 return __nvme_fc_init_hctx(hctx, data, hctx_idx); 2244 } 2245 2246 static void 2247 nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx) 2248 { 2249 struct nvme_fc_queue *queue; 2250 2251 queue = &ctrl->queues[idx]; 2252 memset(queue, 0, sizeof(*queue)); 2253 queue->ctrl = ctrl; 2254 queue->qnum = idx; 2255 atomic_set(&queue->csn, 0); 2256 queue->dev = ctrl->dev; 2257 2258 if (idx > 0) 2259 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16; 2260 else 2261 queue->cmnd_capsule_len = sizeof(struct nvme_command); 2262 2263 /* 2264 * Considered whether we should allocate buffers for all SQEs 2265 * and CQEs and dma map them - mapping their respective entries 2266 * into the request structures (kernel vm addr and dma address) 2267 * thus the driver could use the buffers/mappings directly. 2268 * It only makes sense if the LLDD would use them for its 2269 * messaging api. It's very unlikely most adapter api's would use 2270 * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload 2271 * structures were used instead. 2272 */ 2273 } 2274 2275 /* 2276 * This routine terminates a queue at the transport level. 2277 * The transport has already ensured that all outstanding ios on 2278 * the queue have been terminated. 2279 * The transport will send a Disconnect LS request to terminate 2280 * the queue's connection. Termination of the admin queue will also 2281 * terminate the association at the target. 2282 */ 2283 static void 2284 nvme_fc_free_queue(struct nvme_fc_queue *queue) 2285 { 2286 if (!test_and_clear_bit(NVME_FC_Q_CONNECTED, &queue->flags)) 2287 return; 2288 2289 clear_bit(NVME_FC_Q_LIVE, &queue->flags); 2290 /* 2291 * Current implementation never disconnects a single queue. 2292 * It always terminates a whole association. So there is never 2293 * a disconnect(queue) LS sent to the target. 2294 */ 2295 2296 queue->connection_id = 0; 2297 atomic_set(&queue->csn, 0); 2298 } 2299 2300 static void 2301 __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl, 2302 struct nvme_fc_queue *queue, unsigned int qidx) 2303 { 2304 if (ctrl->lport->ops->delete_queue) 2305 ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx, 2306 queue->lldd_handle); 2307 queue->lldd_handle = NULL; 2308 } 2309 2310 static void 2311 nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl) 2312 { 2313 int i; 2314 2315 for (i = 1; i < ctrl->ctrl.queue_count; i++) 2316 nvme_fc_free_queue(&ctrl->queues[i]); 2317 } 2318 2319 static int 2320 __nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl, 2321 struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize) 2322 { 2323 int ret = 0; 2324 2325 queue->lldd_handle = NULL; 2326 if (ctrl->lport->ops->create_queue) 2327 ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport, 2328 qidx, qsize, &queue->lldd_handle); 2329 2330 return ret; 2331 } 2332 2333 static void 2334 nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl) 2335 { 2336 struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1]; 2337 int i; 2338 2339 for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--) 2340 __nvme_fc_delete_hw_queue(ctrl, queue, i); 2341 } 2342 2343 static int 2344 nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize) 2345 { 2346 struct nvme_fc_queue *queue = &ctrl->queues[1]; 2347 int i, ret; 2348 2349 for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) { 2350 ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize); 2351 if (ret) 2352 goto delete_queues; 2353 } 2354 2355 return 0; 2356 2357 delete_queues: 2358 for (; i > 0; i--) 2359 __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i); 2360 return ret; 2361 } 2362 2363 static int 2364 nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize) 2365 { 2366 int i, ret = 0; 2367 2368 for (i = 1; i < ctrl->ctrl.queue_count; i++) { 2369 ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize, 2370 (qsize / 5)); 2371 if (ret) 2372 break; 2373 ret = nvmf_connect_io_queue(&ctrl->ctrl, i); 2374 if (ret) 2375 break; 2376 2377 set_bit(NVME_FC_Q_LIVE, &ctrl->queues[i].flags); 2378 } 2379 2380 return ret; 2381 } 2382 2383 static void 2384 nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl) 2385 { 2386 int i; 2387 2388 for (i = 1; i < ctrl->ctrl.queue_count; i++) 2389 nvme_fc_init_queue(ctrl, i); 2390 } 2391 2392 static void 2393 nvme_fc_ctrl_free(struct kref *ref) 2394 { 2395 struct nvme_fc_ctrl *ctrl = 2396 container_of(ref, struct nvme_fc_ctrl, ref); 2397 unsigned long flags; 2398 2399 if (ctrl->ctrl.tagset) 2400 nvme_remove_io_tag_set(&ctrl->ctrl); 2401 2402 /* remove from rport list */ 2403 spin_lock_irqsave(&ctrl->rport->lock, flags); 2404 list_del(&ctrl->ctrl_list); 2405 spin_unlock_irqrestore(&ctrl->rport->lock, flags); 2406 2407 nvme_unquiesce_admin_queue(&ctrl->ctrl); 2408 nvme_remove_admin_tag_set(&ctrl->ctrl); 2409 2410 kfree(ctrl->queues); 2411 2412 put_device(ctrl->dev); 2413 nvme_fc_rport_put(ctrl->rport); 2414 2415 ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum); 2416 if (ctrl->ctrl.opts) 2417 nvmf_free_options(ctrl->ctrl.opts); 2418 kfree(ctrl); 2419 } 2420 2421 static void 2422 nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl) 2423 { 2424 kref_put(&ctrl->ref, nvme_fc_ctrl_free); 2425 } 2426 2427 static int 2428 nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl) 2429 { 2430 return kref_get_unless_zero(&ctrl->ref); 2431 } 2432 2433 /* 2434 * All accesses from nvme core layer done - can now free the 2435 * controller. Called after last nvme_put_ctrl() call 2436 */ 2437 static void 2438 nvme_fc_free_ctrl(struct nvme_ctrl *nctrl) 2439 { 2440 struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl); 2441 2442 WARN_ON(nctrl != &ctrl->ctrl); 2443 2444 nvme_fc_ctrl_put(ctrl); 2445 } 2446 2447 /* 2448 * This routine is used by the transport when it needs to find active 2449 * io on a queue that is to be terminated. The transport uses 2450 * blk_mq_tagset_busy_itr() to find the busy requests, which then invoke 2451 * this routine to kill them on a 1 by 1 basis. 2452 * 2453 * As FC allocates FC exchange for each io, the transport must contact 2454 * the LLDD to terminate the exchange, thus releasing the FC exchange. 2455 * After terminating the exchange the LLDD will call the transport's 2456 * normal io done path for the request, but it will have an aborted 2457 * status. The done path will return the io request back to the block 2458 * layer with an error status. 2459 */ 2460 static bool nvme_fc_terminate_exchange(struct request *req, void *data) 2461 { 2462 struct nvme_ctrl *nctrl = data; 2463 struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl); 2464 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req); 2465 2466 op->nreq.flags |= NVME_REQ_CANCELLED; 2467 __nvme_fc_abort_op(ctrl, op); 2468 return true; 2469 } 2470 2471 /* 2472 * This routine runs through all outstanding commands on the association 2473 * and aborts them. This routine is typically be called by the 2474 * delete_association routine. It is also called due to an error during 2475 * reconnect. In that scenario, it is most likely a command that initializes 2476 * the controller, including fabric Connect commands on io queues, that 2477 * may have timed out or failed thus the io must be killed for the connect 2478 * thread to see the error. 2479 */ 2480 static void 2481 __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues) 2482 { 2483 int q; 2484 2485 /* 2486 * if aborting io, the queues are no longer good, mark them 2487 * all as not live. 2488 */ 2489 if (ctrl->ctrl.queue_count > 1) { 2490 for (q = 1; q < ctrl->ctrl.queue_count; q++) 2491 clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[q].flags); 2492 } 2493 clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags); 2494 2495 /* 2496 * If io queues are present, stop them and terminate all outstanding 2497 * ios on them. As FC allocates FC exchange for each io, the 2498 * transport must contact the LLDD to terminate the exchange, 2499 * thus releasing the FC exchange. We use blk_mq_tagset_busy_itr() 2500 * to tell us what io's are busy and invoke a transport routine 2501 * to kill them with the LLDD. After terminating the exchange 2502 * the LLDD will call the transport's normal io done path, but it 2503 * will have an aborted status. The done path will return the 2504 * io requests back to the block layer as part of normal completions 2505 * (but with error status). 2506 */ 2507 if (ctrl->ctrl.queue_count > 1) { 2508 nvme_quiesce_io_queues(&ctrl->ctrl); 2509 nvme_sync_io_queues(&ctrl->ctrl); 2510 blk_mq_tagset_busy_iter(&ctrl->tag_set, 2511 nvme_fc_terminate_exchange, &ctrl->ctrl); 2512 blk_mq_tagset_wait_completed_request(&ctrl->tag_set); 2513 if (start_queues) 2514 nvme_unquiesce_io_queues(&ctrl->ctrl); 2515 } 2516 2517 /* 2518 * Other transports, which don't have link-level contexts bound 2519 * to sqe's, would try to gracefully shutdown the controller by 2520 * writing the registers for shutdown and polling (call 2521 * nvme_disable_ctrl()). Given a bunch of i/o was potentially 2522 * just aborted and we will wait on those contexts, and given 2523 * there was no indication of how live the controlelr is on the 2524 * link, don't send more io to create more contexts for the 2525 * shutdown. Let the controller fail via keepalive failure if 2526 * its still present. 2527 */ 2528 2529 /* 2530 * clean up the admin queue. Same thing as above. 2531 */ 2532 nvme_quiesce_admin_queue(&ctrl->ctrl); 2533 blk_sync_queue(ctrl->ctrl.admin_q); 2534 blk_mq_tagset_busy_iter(&ctrl->admin_tag_set, 2535 nvme_fc_terminate_exchange, &ctrl->ctrl); 2536 blk_mq_tagset_wait_completed_request(&ctrl->admin_tag_set); 2537 if (start_queues) 2538 nvme_unquiesce_admin_queue(&ctrl->ctrl); 2539 } 2540 2541 static void 2542 nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg) 2543 { 2544 /* 2545 * if an error (io timeout, etc) while (re)connecting, the remote 2546 * port requested terminating of the association (disconnect_ls) 2547 * or an error (timeout or abort) occurred on an io while creating 2548 * the controller. Abort any ios on the association and let the 2549 * create_association error path resolve things. 2550 */ 2551 if (ctrl->ctrl.state == NVME_CTRL_CONNECTING) { 2552 __nvme_fc_abort_outstanding_ios(ctrl, true); 2553 dev_warn(ctrl->ctrl.device, 2554 "NVME-FC{%d}: transport error during (re)connect\n", 2555 ctrl->cnum); 2556 return; 2557 } 2558 2559 /* Otherwise, only proceed if in LIVE state - e.g. on first error */ 2560 if (ctrl->ctrl.state != NVME_CTRL_LIVE) 2561 return; 2562 2563 dev_warn(ctrl->ctrl.device, 2564 "NVME-FC{%d}: transport association event: %s\n", 2565 ctrl->cnum, errmsg); 2566 dev_warn(ctrl->ctrl.device, 2567 "NVME-FC{%d}: resetting controller\n", ctrl->cnum); 2568 2569 nvme_reset_ctrl(&ctrl->ctrl); 2570 } 2571 2572 static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq) 2573 { 2574 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); 2575 struct nvme_fc_ctrl *ctrl = op->ctrl; 2576 u16 qnum = op->queue->qnum; 2577 struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu; 2578 struct nvme_command *sqe = &cmdiu->sqe; 2579 2580 /* 2581 * Attempt to abort the offending command. Command completion 2582 * will detect the aborted io and will fail the connection. 2583 */ 2584 dev_info(ctrl->ctrl.device, 2585 "NVME-FC{%d.%d}: io timeout: opcode %d fctype %d (%s) w10/11: " 2586 "x%08x/x%08x\n", 2587 ctrl->cnum, qnum, sqe->common.opcode, sqe->fabrics.fctype, 2588 nvme_fabrics_opcode_str(qnum, sqe), 2589 sqe->common.cdw10, sqe->common.cdw11); 2590 if (__nvme_fc_abort_op(ctrl, op)) 2591 nvme_fc_error_recovery(ctrl, "io timeout abort failed"); 2592 2593 /* 2594 * the io abort has been initiated. Have the reset timer 2595 * restarted and the abort completion will complete the io 2596 * shortly. Avoids a synchronous wait while the abort finishes. 2597 */ 2598 return BLK_EH_RESET_TIMER; 2599 } 2600 2601 static int 2602 nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq, 2603 struct nvme_fc_fcp_op *op) 2604 { 2605 struct nvmefc_fcp_req *freq = &op->fcp_req; 2606 int ret; 2607 2608 freq->sg_cnt = 0; 2609 2610 if (!blk_rq_nr_phys_segments(rq)) 2611 return 0; 2612 2613 freq->sg_table.sgl = freq->first_sgl; 2614 ret = sg_alloc_table_chained(&freq->sg_table, 2615 blk_rq_nr_phys_segments(rq), freq->sg_table.sgl, 2616 NVME_INLINE_SG_CNT); 2617 if (ret) 2618 return -ENOMEM; 2619 2620 op->nents = blk_rq_map_sg(rq->q, rq, freq->sg_table.sgl); 2621 WARN_ON(op->nents > blk_rq_nr_phys_segments(rq)); 2622 freq->sg_cnt = fc_dma_map_sg(ctrl->lport->dev, freq->sg_table.sgl, 2623 op->nents, rq_dma_dir(rq)); 2624 if (unlikely(freq->sg_cnt <= 0)) { 2625 sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT); 2626 freq->sg_cnt = 0; 2627 return -EFAULT; 2628 } 2629 2630 /* 2631 * TODO: blk_integrity_rq(rq) for DIF 2632 */ 2633 return 0; 2634 } 2635 2636 static void 2637 nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq, 2638 struct nvme_fc_fcp_op *op) 2639 { 2640 struct nvmefc_fcp_req *freq = &op->fcp_req; 2641 2642 if (!freq->sg_cnt) 2643 return; 2644 2645 fc_dma_unmap_sg(ctrl->lport->dev, freq->sg_table.sgl, op->nents, 2646 rq_dma_dir(rq)); 2647 2648 sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT); 2649 2650 freq->sg_cnt = 0; 2651 } 2652 2653 /* 2654 * In FC, the queue is a logical thing. At transport connect, the target 2655 * creates its "queue" and returns a handle that is to be given to the 2656 * target whenever it posts something to the corresponding SQ. When an 2657 * SQE is sent on a SQ, FC effectively considers the SQE, or rather the 2658 * command contained within the SQE, an io, and assigns a FC exchange 2659 * to it. The SQE and the associated SQ handle are sent in the initial 2660 * CMD IU sents on the exchange. All transfers relative to the io occur 2661 * as part of the exchange. The CQE is the last thing for the io, 2662 * which is transferred (explicitly or implicitly) with the RSP IU 2663 * sent on the exchange. After the CQE is received, the FC exchange is 2664 * terminaed and the Exchange may be used on a different io. 2665 * 2666 * The transport to LLDD api has the transport making a request for a 2667 * new fcp io request to the LLDD. The LLDD then allocates a FC exchange 2668 * resource and transfers the command. The LLDD will then process all 2669 * steps to complete the io. Upon completion, the transport done routine 2670 * is called. 2671 * 2672 * So - while the operation is outstanding to the LLDD, there is a link 2673 * level FC exchange resource that is also outstanding. This must be 2674 * considered in all cleanup operations. 2675 */ 2676 static blk_status_t 2677 nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, 2678 struct nvme_fc_fcp_op *op, u32 data_len, 2679 enum nvmefc_fcp_datadir io_dir) 2680 { 2681 struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu; 2682 struct nvme_command *sqe = &cmdiu->sqe; 2683 int ret, opstate; 2684 2685 /* 2686 * before attempting to send the io, check to see if we believe 2687 * the target device is present 2688 */ 2689 if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE) 2690 return BLK_STS_RESOURCE; 2691 2692 if (!nvme_fc_ctrl_get(ctrl)) 2693 return BLK_STS_IOERR; 2694 2695 /* format the FC-NVME CMD IU and fcp_req */ 2696 cmdiu->connection_id = cpu_to_be64(queue->connection_id); 2697 cmdiu->data_len = cpu_to_be32(data_len); 2698 switch (io_dir) { 2699 case NVMEFC_FCP_WRITE: 2700 cmdiu->flags = FCNVME_CMD_FLAGS_WRITE; 2701 break; 2702 case NVMEFC_FCP_READ: 2703 cmdiu->flags = FCNVME_CMD_FLAGS_READ; 2704 break; 2705 case NVMEFC_FCP_NODATA: 2706 cmdiu->flags = 0; 2707 break; 2708 } 2709 op->fcp_req.payload_length = data_len; 2710 op->fcp_req.io_dir = io_dir; 2711 op->fcp_req.transferred_length = 0; 2712 op->fcp_req.rcv_rsplen = 0; 2713 op->fcp_req.status = NVME_SC_SUCCESS; 2714 op->fcp_req.sqid = cpu_to_le16(queue->qnum); 2715 2716 /* 2717 * validate per fabric rules, set fields mandated by fabric spec 2718 * as well as those by FC-NVME spec. 2719 */ 2720 WARN_ON_ONCE(sqe->common.metadata); 2721 sqe->common.flags |= NVME_CMD_SGL_METABUF; 2722 2723 /* 2724 * format SQE DPTR field per FC-NVME rules: 2725 * type=0x5 Transport SGL Data Block Descriptor 2726 * subtype=0xA Transport-specific value 2727 * address=0 2728 * length=length of the data series 2729 */ 2730 sqe->rw.dptr.sgl.type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2731 NVME_SGL_FMT_TRANSPORT_A; 2732 sqe->rw.dptr.sgl.length = cpu_to_le32(data_len); 2733 sqe->rw.dptr.sgl.addr = 0; 2734 2735 if (!(op->flags & FCOP_FLAGS_AEN)) { 2736 ret = nvme_fc_map_data(ctrl, op->rq, op); 2737 if (ret < 0) { 2738 nvme_cleanup_cmd(op->rq); 2739 nvme_fc_ctrl_put(ctrl); 2740 if (ret == -ENOMEM || ret == -EAGAIN) 2741 return BLK_STS_RESOURCE; 2742 return BLK_STS_IOERR; 2743 } 2744 } 2745 2746 fc_dma_sync_single_for_device(ctrl->lport->dev, op->fcp_req.cmddma, 2747 sizeof(op->cmd_iu), DMA_TO_DEVICE); 2748 2749 atomic_set(&op->state, FCPOP_STATE_ACTIVE); 2750 2751 if (!(op->flags & FCOP_FLAGS_AEN)) 2752 nvme_start_request(op->rq); 2753 2754 cmdiu->csn = cpu_to_be32(atomic_inc_return(&queue->csn)); 2755 ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport, 2756 &ctrl->rport->remoteport, 2757 queue->lldd_handle, &op->fcp_req); 2758 2759 if (ret) { 2760 /* 2761 * If the lld fails to send the command is there an issue with 2762 * the csn value? If the command that fails is the Connect, 2763 * no - as the connection won't be live. If it is a command 2764 * post-connect, it's possible a gap in csn may be created. 2765 * Does this matter? As Linux initiators don't send fused 2766 * commands, no. The gap would exist, but as there's nothing 2767 * that depends on csn order to be delivered on the target 2768 * side, it shouldn't hurt. It would be difficult for a 2769 * target to even detect the csn gap as it has no idea when the 2770 * cmd with the csn was supposed to arrive. 2771 */ 2772 opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE); 2773 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); 2774 2775 if (!(op->flags & FCOP_FLAGS_AEN)) { 2776 nvme_fc_unmap_data(ctrl, op->rq, op); 2777 nvme_cleanup_cmd(op->rq); 2778 } 2779 2780 nvme_fc_ctrl_put(ctrl); 2781 2782 if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE && 2783 ret != -EBUSY) 2784 return BLK_STS_IOERR; 2785 2786 return BLK_STS_RESOURCE; 2787 } 2788 2789 return BLK_STS_OK; 2790 } 2791 2792 static blk_status_t 2793 nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx, 2794 const struct blk_mq_queue_data *bd) 2795 { 2796 struct nvme_ns *ns = hctx->queue->queuedata; 2797 struct nvme_fc_queue *queue = hctx->driver_data; 2798 struct nvme_fc_ctrl *ctrl = queue->ctrl; 2799 struct request *rq = bd->rq; 2800 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); 2801 enum nvmefc_fcp_datadir io_dir; 2802 bool queue_ready = test_bit(NVME_FC_Q_LIVE, &queue->flags); 2803 u32 data_len; 2804 blk_status_t ret; 2805 2806 if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE || 2807 !nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready)) 2808 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq); 2809 2810 ret = nvme_setup_cmd(ns, rq); 2811 if (ret) 2812 return ret; 2813 2814 /* 2815 * nvme core doesn't quite treat the rq opaquely. Commands such 2816 * as WRITE ZEROES will return a non-zero rq payload_bytes yet 2817 * there is no actual payload to be transferred. 2818 * To get it right, key data transmission on there being 1 or 2819 * more physical segments in the sg list. If there is no 2820 * physical segments, there is no payload. 2821 */ 2822 if (blk_rq_nr_phys_segments(rq)) { 2823 data_len = blk_rq_payload_bytes(rq); 2824 io_dir = ((rq_data_dir(rq) == WRITE) ? 2825 NVMEFC_FCP_WRITE : NVMEFC_FCP_READ); 2826 } else { 2827 data_len = 0; 2828 io_dir = NVMEFC_FCP_NODATA; 2829 } 2830 2831 2832 return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir); 2833 } 2834 2835 static void 2836 nvme_fc_submit_async_event(struct nvme_ctrl *arg) 2837 { 2838 struct nvme_fc_ctrl *ctrl = to_fc_ctrl(arg); 2839 struct nvme_fc_fcp_op *aen_op; 2840 blk_status_t ret; 2841 2842 if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) 2843 return; 2844 2845 aen_op = &ctrl->aen_ops[0]; 2846 2847 ret = nvme_fc_start_fcp_op(ctrl, aen_op->queue, aen_op, 0, 2848 NVMEFC_FCP_NODATA); 2849 if (ret) 2850 dev_err(ctrl->ctrl.device, 2851 "failed async event work\n"); 2852 } 2853 2854 static void 2855 nvme_fc_complete_rq(struct request *rq) 2856 { 2857 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); 2858 struct nvme_fc_ctrl *ctrl = op->ctrl; 2859 2860 atomic_set(&op->state, FCPOP_STATE_IDLE); 2861 op->flags &= ~FCOP_FLAGS_TERMIO; 2862 2863 nvme_fc_unmap_data(ctrl, rq, op); 2864 nvme_complete_rq(rq); 2865 nvme_fc_ctrl_put(ctrl); 2866 } 2867 2868 static void nvme_fc_map_queues(struct blk_mq_tag_set *set) 2869 { 2870 struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data); 2871 int i; 2872 2873 for (i = 0; i < set->nr_maps; i++) { 2874 struct blk_mq_queue_map *map = &set->map[i]; 2875 2876 if (!map->nr_queues) { 2877 WARN_ON(i == HCTX_TYPE_DEFAULT); 2878 continue; 2879 } 2880 2881 /* Call LLDD map queue functionality if defined */ 2882 if (ctrl->lport->ops->map_queues) 2883 ctrl->lport->ops->map_queues(&ctrl->lport->localport, 2884 map); 2885 else 2886 blk_mq_map_queues(map); 2887 } 2888 } 2889 2890 static const struct blk_mq_ops nvme_fc_mq_ops = { 2891 .queue_rq = nvme_fc_queue_rq, 2892 .complete = nvme_fc_complete_rq, 2893 .init_request = nvme_fc_init_request, 2894 .exit_request = nvme_fc_exit_request, 2895 .init_hctx = nvme_fc_init_hctx, 2896 .timeout = nvme_fc_timeout, 2897 .map_queues = nvme_fc_map_queues, 2898 }; 2899 2900 static int 2901 nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl) 2902 { 2903 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 2904 unsigned int nr_io_queues; 2905 int ret; 2906 2907 nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()), 2908 ctrl->lport->ops->max_hw_queues); 2909 ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); 2910 if (ret) { 2911 dev_info(ctrl->ctrl.device, 2912 "set_queue_count failed: %d\n", ret); 2913 return ret; 2914 } 2915 2916 ctrl->ctrl.queue_count = nr_io_queues + 1; 2917 if (!nr_io_queues) 2918 return 0; 2919 2920 nvme_fc_init_io_queues(ctrl); 2921 2922 ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set, 2923 &nvme_fc_mq_ops, 1, 2924 struct_size_t(struct nvme_fcp_op_w_sgl, priv, 2925 ctrl->lport->ops->fcprqst_priv_sz)); 2926 if (ret) 2927 return ret; 2928 2929 ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1); 2930 if (ret) 2931 goto out_cleanup_tagset; 2932 2933 ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1); 2934 if (ret) 2935 goto out_delete_hw_queues; 2936 2937 ctrl->ioq_live = true; 2938 2939 return 0; 2940 2941 out_delete_hw_queues: 2942 nvme_fc_delete_hw_io_queues(ctrl); 2943 out_cleanup_tagset: 2944 nvme_remove_io_tag_set(&ctrl->ctrl); 2945 nvme_fc_free_io_queues(ctrl); 2946 2947 /* force put free routine to ignore io queues */ 2948 ctrl->ctrl.tagset = NULL; 2949 2950 return ret; 2951 } 2952 2953 static int 2954 nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl) 2955 { 2956 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 2957 u32 prior_ioq_cnt = ctrl->ctrl.queue_count - 1; 2958 unsigned int nr_io_queues; 2959 int ret; 2960 2961 nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()), 2962 ctrl->lport->ops->max_hw_queues); 2963 ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); 2964 if (ret) { 2965 dev_info(ctrl->ctrl.device, 2966 "set_queue_count failed: %d\n", ret); 2967 return ret; 2968 } 2969 2970 if (!nr_io_queues && prior_ioq_cnt) { 2971 dev_info(ctrl->ctrl.device, 2972 "Fail Reconnect: At least 1 io queue " 2973 "required (was %d)\n", prior_ioq_cnt); 2974 return -ENOSPC; 2975 } 2976 2977 ctrl->ctrl.queue_count = nr_io_queues + 1; 2978 /* check for io queues existing */ 2979 if (ctrl->ctrl.queue_count == 1) 2980 return 0; 2981 2982 if (prior_ioq_cnt != nr_io_queues) { 2983 dev_info(ctrl->ctrl.device, 2984 "reconnect: revising io queue count from %d to %d\n", 2985 prior_ioq_cnt, nr_io_queues); 2986 blk_mq_update_nr_hw_queues(&ctrl->tag_set, nr_io_queues); 2987 } 2988 2989 ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1); 2990 if (ret) 2991 goto out_free_io_queues; 2992 2993 ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1); 2994 if (ret) 2995 goto out_delete_hw_queues; 2996 2997 return 0; 2998 2999 out_delete_hw_queues: 3000 nvme_fc_delete_hw_io_queues(ctrl); 3001 out_free_io_queues: 3002 nvme_fc_free_io_queues(ctrl); 3003 return ret; 3004 } 3005 3006 static void 3007 nvme_fc_rport_active_on_lport(struct nvme_fc_rport *rport) 3008 { 3009 struct nvme_fc_lport *lport = rport->lport; 3010 3011 atomic_inc(&lport->act_rport_cnt); 3012 } 3013 3014 static void 3015 nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport *rport) 3016 { 3017 struct nvme_fc_lport *lport = rport->lport; 3018 u32 cnt; 3019 3020 cnt = atomic_dec_return(&lport->act_rport_cnt); 3021 if (cnt == 0 && lport->localport.port_state == FC_OBJSTATE_DELETED) 3022 lport->ops->localport_delete(&lport->localport); 3023 } 3024 3025 static int 3026 nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl) 3027 { 3028 struct nvme_fc_rport *rport = ctrl->rport; 3029 u32 cnt; 3030 3031 if (test_and_set_bit(ASSOC_ACTIVE, &ctrl->flags)) 3032 return 1; 3033 3034 cnt = atomic_inc_return(&rport->act_ctrl_cnt); 3035 if (cnt == 1) 3036 nvme_fc_rport_active_on_lport(rport); 3037 3038 return 0; 3039 } 3040 3041 static int 3042 nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl) 3043 { 3044 struct nvme_fc_rport *rport = ctrl->rport; 3045 struct nvme_fc_lport *lport = rport->lport; 3046 u32 cnt; 3047 3048 /* clearing of ctrl->flags ASSOC_ACTIVE bit is in association delete */ 3049 3050 cnt = atomic_dec_return(&rport->act_ctrl_cnt); 3051 if (cnt == 0) { 3052 if (rport->remoteport.port_state == FC_OBJSTATE_DELETED) 3053 lport->ops->remoteport_delete(&rport->remoteport); 3054 nvme_fc_rport_inactive_on_lport(rport); 3055 } 3056 3057 return 0; 3058 } 3059 3060 /* 3061 * This routine restarts the controller on the host side, and 3062 * on the link side, recreates the controller association. 3063 */ 3064 static int 3065 nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) 3066 { 3067 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 3068 struct nvmefc_ls_rcv_op *disls = NULL; 3069 unsigned long flags; 3070 int ret; 3071 bool changed; 3072 3073 ++ctrl->ctrl.nr_reconnects; 3074 3075 if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE) 3076 return -ENODEV; 3077 3078 if (nvme_fc_ctlr_active_on_rport(ctrl)) 3079 return -ENOTUNIQ; 3080 3081 dev_info(ctrl->ctrl.device, 3082 "NVME-FC{%d}: create association : host wwpn 0x%016llx " 3083 " rport wwpn 0x%016llx: NQN \"%s\"\n", 3084 ctrl->cnum, ctrl->lport->localport.port_name, 3085 ctrl->rport->remoteport.port_name, ctrl->ctrl.opts->subsysnqn); 3086 3087 clear_bit(ASSOC_FAILED, &ctrl->flags); 3088 3089 /* 3090 * Create the admin queue 3091 */ 3092 3093 ret = __nvme_fc_create_hw_queue(ctrl, &ctrl->queues[0], 0, 3094 NVME_AQ_DEPTH); 3095 if (ret) 3096 goto out_free_queue; 3097 3098 ret = nvme_fc_connect_admin_queue(ctrl, &ctrl->queues[0], 3099 NVME_AQ_DEPTH, (NVME_AQ_DEPTH / 4)); 3100 if (ret) 3101 goto out_delete_hw_queue; 3102 3103 ret = nvmf_connect_admin_queue(&ctrl->ctrl); 3104 if (ret) 3105 goto out_disconnect_admin_queue; 3106 3107 set_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags); 3108 3109 /* 3110 * Check controller capabilities 3111 * 3112 * todo:- add code to check if ctrl attributes changed from 3113 * prior connection values 3114 */ 3115 3116 ret = nvme_enable_ctrl(&ctrl->ctrl); 3117 if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags)) 3118 ret = -EIO; 3119 if (ret) 3120 goto out_disconnect_admin_queue; 3121 3122 ctrl->ctrl.max_segments = ctrl->lport->ops->max_sgl_segments; 3123 ctrl->ctrl.max_hw_sectors = ctrl->ctrl.max_segments << 3124 (ilog2(SZ_4K) - 9); 3125 3126 nvme_unquiesce_admin_queue(&ctrl->ctrl); 3127 3128 ret = nvme_init_ctrl_finish(&ctrl->ctrl, false); 3129 if (ret) 3130 goto out_disconnect_admin_queue; 3131 if (test_bit(ASSOC_FAILED, &ctrl->flags)) { 3132 ret = -EIO; 3133 goto out_stop_keep_alive; 3134 } 3135 /* sanity checks */ 3136 3137 /* FC-NVME does not have other data in the capsule */ 3138 if (ctrl->ctrl.icdoff) { 3139 dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n", 3140 ctrl->ctrl.icdoff); 3141 ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 3142 goto out_stop_keep_alive; 3143 } 3144 3145 /* FC-NVME supports normal SGL Data Block Descriptors */ 3146 if (!nvme_ctrl_sgl_supported(&ctrl->ctrl)) { 3147 dev_err(ctrl->ctrl.device, 3148 "Mandatory sgls are not supported!\n"); 3149 ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 3150 goto out_stop_keep_alive; 3151 } 3152 3153 if (opts->queue_size > ctrl->ctrl.maxcmd) { 3154 /* warn if maxcmd is lower than queue_size */ 3155 dev_warn(ctrl->ctrl.device, 3156 "queue_size %zu > ctrl maxcmd %u, reducing " 3157 "to maxcmd\n", 3158 opts->queue_size, ctrl->ctrl.maxcmd); 3159 opts->queue_size = ctrl->ctrl.maxcmd; 3160 ctrl->ctrl.sqsize = opts->queue_size - 1; 3161 } 3162 3163 ret = nvme_fc_init_aen_ops(ctrl); 3164 if (ret) 3165 goto out_term_aen_ops; 3166 3167 /* 3168 * Create the io queues 3169 */ 3170 3171 if (ctrl->ctrl.queue_count > 1) { 3172 if (!ctrl->ioq_live) 3173 ret = nvme_fc_create_io_queues(ctrl); 3174 else 3175 ret = nvme_fc_recreate_io_queues(ctrl); 3176 } 3177 if (ret) 3178 goto out_term_aen_ops; 3179 3180 spin_lock_irqsave(&ctrl->lock, flags); 3181 if (!test_bit(ASSOC_FAILED, &ctrl->flags)) 3182 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE); 3183 else 3184 ret = -EIO; 3185 spin_unlock_irqrestore(&ctrl->lock, flags); 3186 3187 if (ret) 3188 goto out_term_aen_ops; 3189 3190 ctrl->ctrl.nr_reconnects = 0; 3191 3192 if (changed) 3193 nvme_start_ctrl(&ctrl->ctrl); 3194 3195 return 0; /* Success */ 3196 3197 out_term_aen_ops: 3198 nvme_fc_term_aen_ops(ctrl); 3199 out_stop_keep_alive: 3200 nvme_stop_keep_alive(&ctrl->ctrl); 3201 out_disconnect_admin_queue: 3202 dev_warn(ctrl->ctrl.device, 3203 "NVME-FC{%d}: create_assoc failed, assoc_id %llx ret %d\n", 3204 ctrl->cnum, ctrl->association_id, ret); 3205 /* send a Disconnect(association) LS to fc-nvme target */ 3206 nvme_fc_xmt_disconnect_assoc(ctrl); 3207 spin_lock_irqsave(&ctrl->lock, flags); 3208 ctrl->association_id = 0; 3209 disls = ctrl->rcv_disconn; 3210 ctrl->rcv_disconn = NULL; 3211 spin_unlock_irqrestore(&ctrl->lock, flags); 3212 if (disls) 3213 nvme_fc_xmt_ls_rsp(disls); 3214 out_delete_hw_queue: 3215 __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0); 3216 out_free_queue: 3217 nvme_fc_free_queue(&ctrl->queues[0]); 3218 clear_bit(ASSOC_ACTIVE, &ctrl->flags); 3219 nvme_fc_ctlr_inactive_on_rport(ctrl); 3220 3221 return ret; 3222 } 3223 3224 3225 /* 3226 * This routine stops operation of the controller on the host side. 3227 * On the host os stack side: Admin and IO queues are stopped, 3228 * outstanding ios on them terminated via FC ABTS. 3229 * On the link side: the association is terminated. 3230 */ 3231 static void 3232 nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl) 3233 { 3234 struct nvmefc_ls_rcv_op *disls = NULL; 3235 unsigned long flags; 3236 3237 if (!test_and_clear_bit(ASSOC_ACTIVE, &ctrl->flags)) 3238 return; 3239 3240 spin_lock_irqsave(&ctrl->lock, flags); 3241 set_bit(FCCTRL_TERMIO, &ctrl->flags); 3242 ctrl->iocnt = 0; 3243 spin_unlock_irqrestore(&ctrl->lock, flags); 3244 3245 __nvme_fc_abort_outstanding_ios(ctrl, false); 3246 3247 /* kill the aens as they are a separate path */ 3248 nvme_fc_abort_aen_ops(ctrl); 3249 3250 /* wait for all io that had to be aborted */ 3251 spin_lock_irq(&ctrl->lock); 3252 wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock); 3253 clear_bit(FCCTRL_TERMIO, &ctrl->flags); 3254 spin_unlock_irq(&ctrl->lock); 3255 3256 nvme_fc_term_aen_ops(ctrl); 3257 3258 /* 3259 * send a Disconnect(association) LS to fc-nvme target 3260 * Note: could have been sent at top of process, but 3261 * cleaner on link traffic if after the aborts complete. 3262 * Note: if association doesn't exist, association_id will be 0 3263 */ 3264 if (ctrl->association_id) 3265 nvme_fc_xmt_disconnect_assoc(ctrl); 3266 3267 spin_lock_irqsave(&ctrl->lock, flags); 3268 ctrl->association_id = 0; 3269 disls = ctrl->rcv_disconn; 3270 ctrl->rcv_disconn = NULL; 3271 spin_unlock_irqrestore(&ctrl->lock, flags); 3272 if (disls) 3273 /* 3274 * if a Disconnect Request was waiting for a response, send 3275 * now that all ABTS's have been issued (and are complete). 3276 */ 3277 nvme_fc_xmt_ls_rsp(disls); 3278 3279 if (ctrl->ctrl.tagset) { 3280 nvme_fc_delete_hw_io_queues(ctrl); 3281 nvme_fc_free_io_queues(ctrl); 3282 } 3283 3284 __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0); 3285 nvme_fc_free_queue(&ctrl->queues[0]); 3286 3287 /* re-enable the admin_q so anything new can fast fail */ 3288 nvme_unquiesce_admin_queue(&ctrl->ctrl); 3289 3290 /* resume the io queues so that things will fast fail */ 3291 nvme_unquiesce_io_queues(&ctrl->ctrl); 3292 3293 nvme_fc_ctlr_inactive_on_rport(ctrl); 3294 } 3295 3296 static void 3297 nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl) 3298 { 3299 struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl); 3300 3301 cancel_work_sync(&ctrl->ioerr_work); 3302 cancel_delayed_work_sync(&ctrl->connect_work); 3303 /* 3304 * kill the association on the link side. this will block 3305 * waiting for io to terminate 3306 */ 3307 nvme_fc_delete_association(ctrl); 3308 } 3309 3310 static void 3311 nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status) 3312 { 3313 struct nvme_fc_rport *rport = ctrl->rport; 3314 struct nvme_fc_remote_port *portptr = &rport->remoteport; 3315 unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ; 3316 bool recon = true; 3317 3318 if (nvme_ctrl_state(&ctrl->ctrl) != NVME_CTRL_CONNECTING) 3319 return; 3320 3321 if (portptr->port_state == FC_OBJSTATE_ONLINE) { 3322 dev_info(ctrl->ctrl.device, 3323 "NVME-FC{%d}: reset: Reconnect attempt failed (%d)\n", 3324 ctrl->cnum, status); 3325 } else if (time_after_eq(jiffies, rport->dev_loss_end)) 3326 recon = false; 3327 3328 if (recon && nvmf_should_reconnect(&ctrl->ctrl, status)) { 3329 if (portptr->port_state == FC_OBJSTATE_ONLINE) 3330 dev_info(ctrl->ctrl.device, 3331 "NVME-FC{%d}: Reconnect attempt in %ld " 3332 "seconds\n", 3333 ctrl->cnum, recon_delay / HZ); 3334 else if (time_after(jiffies + recon_delay, rport->dev_loss_end)) 3335 recon_delay = rport->dev_loss_end - jiffies; 3336 3337 queue_delayed_work(nvme_wq, &ctrl->connect_work, recon_delay); 3338 } else { 3339 if (portptr->port_state == FC_OBJSTATE_ONLINE) { 3340 if (status > 0 && (status & NVME_STATUS_DNR)) 3341 dev_warn(ctrl->ctrl.device, 3342 "NVME-FC{%d}: reconnect failure\n", 3343 ctrl->cnum); 3344 else 3345 dev_warn(ctrl->ctrl.device, 3346 "NVME-FC{%d}: Max reconnect attempts " 3347 "(%d) reached.\n", 3348 ctrl->cnum, ctrl->ctrl.nr_reconnects); 3349 } else 3350 dev_warn(ctrl->ctrl.device, 3351 "NVME-FC{%d}: dev_loss_tmo (%d) expired " 3352 "while waiting for remoteport connectivity.\n", 3353 ctrl->cnum, min_t(int, portptr->dev_loss_tmo, 3354 (ctrl->ctrl.opts->max_reconnects * 3355 ctrl->ctrl.opts->reconnect_delay))); 3356 WARN_ON(nvme_delete_ctrl(&ctrl->ctrl)); 3357 } 3358 } 3359 3360 static void 3361 nvme_fc_reset_ctrl_work(struct work_struct *work) 3362 { 3363 struct nvme_fc_ctrl *ctrl = 3364 container_of(work, struct nvme_fc_ctrl, ctrl.reset_work); 3365 3366 nvme_stop_ctrl(&ctrl->ctrl); 3367 3368 /* will block will waiting for io to terminate */ 3369 nvme_fc_delete_association(ctrl); 3370 3371 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) 3372 dev_err(ctrl->ctrl.device, 3373 "NVME-FC{%d}: error_recovery: Couldn't change state " 3374 "to CONNECTING\n", ctrl->cnum); 3375 3376 if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) { 3377 if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) { 3378 dev_err(ctrl->ctrl.device, 3379 "NVME-FC{%d}: failed to schedule connect " 3380 "after reset\n", ctrl->cnum); 3381 } else { 3382 flush_delayed_work(&ctrl->connect_work); 3383 } 3384 } else { 3385 nvme_fc_reconnect_or_delete(ctrl, -ENOTCONN); 3386 } 3387 } 3388 3389 3390 static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = { 3391 .name = "fc", 3392 .module = THIS_MODULE, 3393 .flags = NVME_F_FABRICS, 3394 .reg_read32 = nvmf_reg_read32, 3395 .reg_read64 = nvmf_reg_read64, 3396 .reg_write32 = nvmf_reg_write32, 3397 .subsystem_reset = nvmf_subsystem_reset, 3398 .free_ctrl = nvme_fc_free_ctrl, 3399 .submit_async_event = nvme_fc_submit_async_event, 3400 .delete_ctrl = nvme_fc_delete_ctrl, 3401 .get_address = nvmf_get_address, 3402 }; 3403 3404 static void 3405 nvme_fc_connect_ctrl_work(struct work_struct *work) 3406 { 3407 int ret; 3408 3409 struct nvme_fc_ctrl *ctrl = 3410 container_of(to_delayed_work(work), 3411 struct nvme_fc_ctrl, connect_work); 3412 3413 ret = nvme_fc_create_association(ctrl); 3414 if (ret) 3415 nvme_fc_reconnect_or_delete(ctrl, ret); 3416 else 3417 dev_info(ctrl->ctrl.device, 3418 "NVME-FC{%d}: controller connect complete\n", 3419 ctrl->cnum); 3420 } 3421 3422 3423 static const struct blk_mq_ops nvme_fc_admin_mq_ops = { 3424 .queue_rq = nvme_fc_queue_rq, 3425 .complete = nvme_fc_complete_rq, 3426 .init_request = nvme_fc_init_request, 3427 .exit_request = nvme_fc_exit_request, 3428 .init_hctx = nvme_fc_init_admin_hctx, 3429 .timeout = nvme_fc_timeout, 3430 }; 3431 3432 3433 /* 3434 * Fails a controller request if it matches an existing controller 3435 * (association) with the same tuple: 3436 * <Host NQN, Host ID, local FC port, remote FC port, SUBSYS NQN> 3437 * 3438 * The ports don't need to be compared as they are intrinsically 3439 * already matched by the port pointers supplied. 3440 */ 3441 static bool 3442 nvme_fc_existing_controller(struct nvme_fc_rport *rport, 3443 struct nvmf_ctrl_options *opts) 3444 { 3445 struct nvme_fc_ctrl *ctrl; 3446 unsigned long flags; 3447 bool found = false; 3448 3449 spin_lock_irqsave(&rport->lock, flags); 3450 list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) { 3451 found = nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts); 3452 if (found) 3453 break; 3454 } 3455 spin_unlock_irqrestore(&rport->lock, flags); 3456 3457 return found; 3458 } 3459 3460 static struct nvme_fc_ctrl * 3461 nvme_fc_alloc_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, 3462 struct nvme_fc_lport *lport, struct nvme_fc_rport *rport) 3463 { 3464 struct nvme_fc_ctrl *ctrl; 3465 int ret, idx, ctrl_loss_tmo; 3466 3467 if (!(rport->remoteport.port_role & 3468 (FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET))) { 3469 ret = -EBADR; 3470 goto out_fail; 3471 } 3472 3473 if (!opts->duplicate_connect && 3474 nvme_fc_existing_controller(rport, opts)) { 3475 ret = -EALREADY; 3476 goto out_fail; 3477 } 3478 3479 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 3480 if (!ctrl) { 3481 ret = -ENOMEM; 3482 goto out_fail; 3483 } 3484 3485 idx = ida_alloc(&nvme_fc_ctrl_cnt, GFP_KERNEL); 3486 if (idx < 0) { 3487 ret = -ENOSPC; 3488 goto out_free_ctrl; 3489 } 3490 3491 /* 3492 * if ctrl_loss_tmo is being enforced and the default reconnect delay 3493 * is being used, change to a shorter reconnect delay for FC. 3494 */ 3495 if (opts->max_reconnects != -1 && 3496 opts->reconnect_delay == NVMF_DEF_RECONNECT_DELAY && 3497 opts->reconnect_delay > NVME_FC_DEFAULT_RECONNECT_TMO) { 3498 ctrl_loss_tmo = opts->max_reconnects * opts->reconnect_delay; 3499 opts->reconnect_delay = NVME_FC_DEFAULT_RECONNECT_TMO; 3500 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo, 3501 opts->reconnect_delay); 3502 } 3503 3504 ctrl->ctrl.opts = opts; 3505 ctrl->ctrl.nr_reconnects = 0; 3506 INIT_LIST_HEAD(&ctrl->ctrl_list); 3507 ctrl->lport = lport; 3508 ctrl->rport = rport; 3509 ctrl->dev = lport->dev; 3510 ctrl->cnum = idx; 3511 ctrl->ioq_live = false; 3512 init_waitqueue_head(&ctrl->ioabort_wait); 3513 3514 get_device(ctrl->dev); 3515 kref_init(&ctrl->ref); 3516 3517 INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work); 3518 INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work); 3519 INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work); 3520 spin_lock_init(&ctrl->lock); 3521 3522 /* io queue count */ 3523 ctrl->ctrl.queue_count = min_t(unsigned int, 3524 opts->nr_io_queues, 3525 lport->ops->max_hw_queues); 3526 ctrl->ctrl.queue_count++; /* +1 for admin queue */ 3527 3528 ctrl->ctrl.sqsize = opts->queue_size - 1; 3529 ctrl->ctrl.kato = opts->kato; 3530 ctrl->ctrl.cntlid = 0xffff; 3531 3532 ret = -ENOMEM; 3533 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, 3534 sizeof(struct nvme_fc_queue), GFP_KERNEL); 3535 if (!ctrl->queues) 3536 goto out_free_ida; 3537 3538 nvme_fc_init_queue(ctrl, 0); 3539 3540 /* 3541 * Would have been nice to init io queues tag set as well. 3542 * However, we require interaction from the controller 3543 * for max io queue count before we can do so. 3544 * Defer this to the connect path. 3545 */ 3546 3547 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0); 3548 if (ret) 3549 goto out_free_queues; 3550 if (lport->dev) 3551 ctrl->ctrl.numa_node = dev_to_node(lport->dev); 3552 3553 return ctrl; 3554 3555 out_free_queues: 3556 kfree(ctrl->queues); 3557 out_free_ida: 3558 put_device(ctrl->dev); 3559 ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum); 3560 out_free_ctrl: 3561 kfree(ctrl); 3562 out_fail: 3563 /* exit via here doesn't follow ctlr ref points */ 3564 return ERR_PTR(ret); 3565 } 3566 3567 static struct nvme_ctrl * 3568 nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, 3569 struct nvme_fc_lport *lport, struct nvme_fc_rport *rport) 3570 { 3571 struct nvme_fc_ctrl *ctrl; 3572 unsigned long flags; 3573 int ret; 3574 3575 ctrl = nvme_fc_alloc_ctrl(dev, opts, lport, rport); 3576 if (IS_ERR(ctrl)) 3577 return ERR_CAST(ctrl); 3578 3579 ret = nvme_add_ctrl(&ctrl->ctrl); 3580 if (ret) 3581 goto out_put_ctrl; 3582 3583 ret = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set, 3584 &nvme_fc_admin_mq_ops, 3585 struct_size_t(struct nvme_fcp_op_w_sgl, priv, 3586 ctrl->lport->ops->fcprqst_priv_sz)); 3587 if (ret) 3588 goto fail_ctrl; 3589 3590 spin_lock_irqsave(&rport->lock, flags); 3591 list_add_tail(&ctrl->ctrl_list, &rport->ctrl_list); 3592 spin_unlock_irqrestore(&rport->lock, flags); 3593 3594 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { 3595 dev_err(ctrl->ctrl.device, 3596 "NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum); 3597 goto fail_ctrl; 3598 } 3599 3600 if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) { 3601 dev_err(ctrl->ctrl.device, 3602 "NVME-FC{%d}: failed to schedule initial connect\n", 3603 ctrl->cnum); 3604 goto fail_ctrl; 3605 } 3606 3607 flush_delayed_work(&ctrl->connect_work); 3608 3609 dev_info(ctrl->ctrl.device, 3610 "NVME-FC{%d}: new ctrl: NQN \"%s\", hostnqn: %s\n", 3611 ctrl->cnum, nvmf_ctrl_subsysnqn(&ctrl->ctrl), opts->host->nqn); 3612 3613 return &ctrl->ctrl; 3614 3615 fail_ctrl: 3616 nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING); 3617 cancel_work_sync(&ctrl->ioerr_work); 3618 cancel_work_sync(&ctrl->ctrl.reset_work); 3619 cancel_delayed_work_sync(&ctrl->connect_work); 3620 3621 ctrl->ctrl.opts = NULL; 3622 3623 /* initiate nvme ctrl ref counting teardown */ 3624 nvme_uninit_ctrl(&ctrl->ctrl); 3625 3626 out_put_ctrl: 3627 /* Remove core ctrl ref. */ 3628 nvme_put_ctrl(&ctrl->ctrl); 3629 3630 /* as we're past the point where we transition to the ref 3631 * counting teardown path, if we return a bad pointer here, 3632 * the calling routine, thinking it's prior to the 3633 * transition, will do an rport put. Since the teardown 3634 * path also does a rport put, we do an extra get here to 3635 * so proper order/teardown happens. 3636 */ 3637 nvme_fc_rport_get(rport); 3638 3639 return ERR_PTR(-EIO); 3640 } 3641 3642 struct nvmet_fc_traddr { 3643 u64 nn; 3644 u64 pn; 3645 }; 3646 3647 static int 3648 __nvme_fc_parse_u64(substring_t *sstr, u64 *val) 3649 { 3650 u64 token64; 3651 3652 if (match_u64(sstr, &token64)) 3653 return -EINVAL; 3654 *val = token64; 3655 3656 return 0; 3657 } 3658 3659 /* 3660 * This routine validates and extracts the WWN's from the TRADDR string. 3661 * As kernel parsers need the 0x to determine number base, universally 3662 * build string to parse with 0x prefix before parsing name strings. 3663 */ 3664 static int 3665 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen) 3666 { 3667 char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1]; 3668 substring_t wwn = { name, &name[sizeof(name)-1] }; 3669 int nnoffset, pnoffset; 3670 3671 /* validate if string is one of the 2 allowed formats */ 3672 if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH && 3673 !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) && 3674 !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET], 3675 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) { 3676 nnoffset = NVME_FC_TRADDR_OXNNLEN; 3677 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET + 3678 NVME_FC_TRADDR_OXNNLEN; 3679 } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH && 3680 !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) && 3681 !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET], 3682 "pn-", NVME_FC_TRADDR_NNLEN))) { 3683 nnoffset = NVME_FC_TRADDR_NNLEN; 3684 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN; 3685 } else 3686 goto out_einval; 3687 3688 name[0] = '0'; 3689 name[1] = 'x'; 3690 name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0; 3691 3692 memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN); 3693 if (__nvme_fc_parse_u64(&wwn, &traddr->nn)) 3694 goto out_einval; 3695 3696 memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN); 3697 if (__nvme_fc_parse_u64(&wwn, &traddr->pn)) 3698 goto out_einval; 3699 3700 return 0; 3701 3702 out_einval: 3703 pr_warn("%s: bad traddr string\n", __func__); 3704 return -EINVAL; 3705 } 3706 3707 static struct nvme_ctrl * 3708 nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts) 3709 { 3710 struct nvme_fc_lport *lport; 3711 struct nvme_fc_rport *rport; 3712 struct nvme_ctrl *ctrl; 3713 struct nvmet_fc_traddr laddr = { 0L, 0L }; 3714 struct nvmet_fc_traddr raddr = { 0L, 0L }; 3715 unsigned long flags; 3716 int ret; 3717 3718 ret = nvme_fc_parse_traddr(&raddr, opts->traddr, NVMF_TRADDR_SIZE); 3719 if (ret || !raddr.nn || !raddr.pn) 3720 return ERR_PTR(-EINVAL); 3721 3722 ret = nvme_fc_parse_traddr(&laddr, opts->host_traddr, NVMF_TRADDR_SIZE); 3723 if (ret || !laddr.nn || !laddr.pn) 3724 return ERR_PTR(-EINVAL); 3725 3726 /* find the host and remote ports to connect together */ 3727 spin_lock_irqsave(&nvme_fc_lock, flags); 3728 list_for_each_entry(lport, &nvme_fc_lport_list, port_list) { 3729 if (lport->localport.node_name != laddr.nn || 3730 lport->localport.port_name != laddr.pn || 3731 lport->localport.port_state != FC_OBJSTATE_ONLINE) 3732 continue; 3733 3734 list_for_each_entry(rport, &lport->endp_list, endp_list) { 3735 if (rport->remoteport.node_name != raddr.nn || 3736 rport->remoteport.port_name != raddr.pn || 3737 rport->remoteport.port_state != FC_OBJSTATE_ONLINE) 3738 continue; 3739 3740 /* if fail to get reference fall through. Will error */ 3741 if (!nvme_fc_rport_get(rport)) 3742 break; 3743 3744 spin_unlock_irqrestore(&nvme_fc_lock, flags); 3745 3746 ctrl = nvme_fc_init_ctrl(dev, opts, lport, rport); 3747 if (IS_ERR(ctrl)) 3748 nvme_fc_rport_put(rport); 3749 return ctrl; 3750 } 3751 } 3752 spin_unlock_irqrestore(&nvme_fc_lock, flags); 3753 3754 pr_warn("%s: %s - %s combination not found\n", 3755 __func__, opts->traddr, opts->host_traddr); 3756 return ERR_PTR(-ENOENT); 3757 } 3758 3759 3760 static struct nvmf_transport_ops nvme_fc_transport = { 3761 .name = "fc", 3762 .module = THIS_MODULE, 3763 .required_opts = NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR, 3764 .allowed_opts = NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO, 3765 .create_ctrl = nvme_fc_create_ctrl, 3766 }; 3767 3768 /* Arbitrary successive failures max. With lots of subsystems could be high */ 3769 #define DISCOVERY_MAX_FAIL 20 3770 3771 static ssize_t nvme_fc_nvme_discovery_store(struct device *dev, 3772 struct device_attribute *attr, const char *buf, size_t count) 3773 { 3774 unsigned long flags; 3775 LIST_HEAD(local_disc_list); 3776 struct nvme_fc_lport *lport; 3777 struct nvme_fc_rport *rport; 3778 int failcnt = 0; 3779 3780 spin_lock_irqsave(&nvme_fc_lock, flags); 3781 restart: 3782 list_for_each_entry(lport, &nvme_fc_lport_list, port_list) { 3783 list_for_each_entry(rport, &lport->endp_list, endp_list) { 3784 if (!nvme_fc_lport_get(lport)) 3785 continue; 3786 if (!nvme_fc_rport_get(rport)) { 3787 /* 3788 * This is a temporary condition. Upon restart 3789 * this rport will be gone from the list. 3790 * 3791 * Revert the lport put and retry. Anything 3792 * added to the list already will be skipped (as 3793 * they are no longer list_empty). Loops should 3794 * resume at rports that were not yet seen. 3795 */ 3796 nvme_fc_lport_put(lport); 3797 3798 if (failcnt++ < DISCOVERY_MAX_FAIL) 3799 goto restart; 3800 3801 pr_err("nvme_discovery: too many reference " 3802 "failures\n"); 3803 goto process_local_list; 3804 } 3805 if (list_empty(&rport->disc_list)) 3806 list_add_tail(&rport->disc_list, 3807 &local_disc_list); 3808 } 3809 } 3810 3811 process_local_list: 3812 while (!list_empty(&local_disc_list)) { 3813 rport = list_first_entry(&local_disc_list, 3814 struct nvme_fc_rport, disc_list); 3815 list_del_init(&rport->disc_list); 3816 spin_unlock_irqrestore(&nvme_fc_lock, flags); 3817 3818 lport = rport->lport; 3819 /* signal discovery. Won't hurt if it repeats */ 3820 nvme_fc_signal_discovery_scan(lport, rport); 3821 nvme_fc_rport_put(rport); 3822 nvme_fc_lport_put(lport); 3823 3824 spin_lock_irqsave(&nvme_fc_lock, flags); 3825 } 3826 spin_unlock_irqrestore(&nvme_fc_lock, flags); 3827 3828 return count; 3829 } 3830 3831 static DEVICE_ATTR(nvme_discovery, 0200, NULL, nvme_fc_nvme_discovery_store); 3832 3833 #ifdef CONFIG_BLK_CGROUP_FC_APPID 3834 /* Parse the cgroup id from a buf and return the length of cgrpid */ 3835 static int fc_parse_cgrpid(const char *buf, u64 *id) 3836 { 3837 char cgrp_id[16+1]; 3838 int cgrpid_len, j; 3839 3840 memset(cgrp_id, 0x0, sizeof(cgrp_id)); 3841 for (cgrpid_len = 0, j = 0; cgrpid_len < 17; cgrpid_len++) { 3842 if (buf[cgrpid_len] != ':') 3843 cgrp_id[cgrpid_len] = buf[cgrpid_len]; 3844 else { 3845 j = 1; 3846 break; 3847 } 3848 } 3849 if (!j) 3850 return -EINVAL; 3851 if (kstrtou64(cgrp_id, 16, id) < 0) 3852 return -EINVAL; 3853 return cgrpid_len; 3854 } 3855 3856 /* 3857 * Parse and update the appid in the blkcg associated with the cgroupid. 3858 */ 3859 static ssize_t fc_appid_store(struct device *dev, 3860 struct device_attribute *attr, const char *buf, size_t count) 3861 { 3862 size_t orig_count = count; 3863 u64 cgrp_id; 3864 int appid_len = 0; 3865 int cgrpid_len = 0; 3866 char app_id[FC_APPID_LEN]; 3867 int ret = 0; 3868 3869 if (buf[count-1] == '\n') 3870 count--; 3871 3872 if ((count > (16+1+FC_APPID_LEN)) || (!strchr(buf, ':'))) 3873 return -EINVAL; 3874 3875 cgrpid_len = fc_parse_cgrpid(buf, &cgrp_id); 3876 if (cgrpid_len < 0) 3877 return -EINVAL; 3878 appid_len = count - cgrpid_len - 1; 3879 if (appid_len > FC_APPID_LEN) 3880 return -EINVAL; 3881 3882 memset(app_id, 0x0, sizeof(app_id)); 3883 memcpy(app_id, &buf[cgrpid_len+1], appid_len); 3884 ret = blkcg_set_fc_appid(app_id, cgrp_id, sizeof(app_id)); 3885 if (ret < 0) 3886 return ret; 3887 return orig_count; 3888 } 3889 static DEVICE_ATTR(appid_store, 0200, NULL, fc_appid_store); 3890 #endif /* CONFIG_BLK_CGROUP_FC_APPID */ 3891 3892 static struct attribute *nvme_fc_attrs[] = { 3893 &dev_attr_nvme_discovery.attr, 3894 #ifdef CONFIG_BLK_CGROUP_FC_APPID 3895 &dev_attr_appid_store.attr, 3896 #endif 3897 NULL 3898 }; 3899 3900 static const struct attribute_group nvme_fc_attr_group = { 3901 .attrs = nvme_fc_attrs, 3902 }; 3903 3904 static const struct attribute_group *nvme_fc_attr_groups[] = { 3905 &nvme_fc_attr_group, 3906 NULL 3907 }; 3908 3909 static struct class fc_class = { 3910 .name = "fc", 3911 .dev_groups = nvme_fc_attr_groups, 3912 }; 3913 3914 static int __init nvme_fc_init_module(void) 3915 { 3916 int ret; 3917 3918 /* 3919 * NOTE: 3920 * It is expected that in the future the kernel will combine 3921 * the FC-isms that are currently under scsi and now being 3922 * added to by NVME into a new standalone FC class. The SCSI 3923 * and NVME protocols and their devices would be under this 3924 * new FC class. 3925 * 3926 * As we need something to post FC-specific udev events to, 3927 * specifically for nvme probe events, start by creating the 3928 * new device class. When the new standalone FC class is 3929 * put in place, this code will move to a more generic 3930 * location for the class. 3931 */ 3932 ret = class_register(&fc_class); 3933 if (ret) { 3934 pr_err("couldn't register class fc\n"); 3935 return ret; 3936 } 3937 3938 /* 3939 * Create a device for the FC-centric udev events 3940 */ 3941 fc_udev_device = device_create(&fc_class, NULL, MKDEV(0, 0), NULL, 3942 "fc_udev_device"); 3943 if (IS_ERR(fc_udev_device)) { 3944 pr_err("couldn't create fc_udev device!\n"); 3945 ret = PTR_ERR(fc_udev_device); 3946 goto out_destroy_class; 3947 } 3948 3949 ret = nvmf_register_transport(&nvme_fc_transport); 3950 if (ret) 3951 goto out_destroy_device; 3952 3953 return 0; 3954 3955 out_destroy_device: 3956 device_destroy(&fc_class, MKDEV(0, 0)); 3957 out_destroy_class: 3958 class_unregister(&fc_class); 3959 3960 return ret; 3961 } 3962 3963 static void 3964 nvme_fc_delete_controllers(struct nvme_fc_rport *rport) 3965 { 3966 struct nvme_fc_ctrl *ctrl; 3967 3968 spin_lock(&rport->lock); 3969 list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) { 3970 dev_warn(ctrl->ctrl.device, 3971 "NVME-FC{%d}: transport unloading: deleting ctrl\n", 3972 ctrl->cnum); 3973 nvme_delete_ctrl(&ctrl->ctrl); 3974 } 3975 spin_unlock(&rport->lock); 3976 } 3977 3978 static void __exit nvme_fc_exit_module(void) 3979 { 3980 struct nvme_fc_lport *lport; 3981 struct nvme_fc_rport *rport; 3982 unsigned long flags; 3983 3984 spin_lock_irqsave(&nvme_fc_lock, flags); 3985 list_for_each_entry(lport, &nvme_fc_lport_list, port_list) 3986 list_for_each_entry(rport, &lport->endp_list, endp_list) 3987 nvme_fc_delete_controllers(rport); 3988 spin_unlock_irqrestore(&nvme_fc_lock, flags); 3989 flush_workqueue(nvme_delete_wq); 3990 3991 nvmf_unregister_transport(&nvme_fc_transport); 3992 3993 device_destroy(&fc_class, MKDEV(0, 0)); 3994 class_unregister(&fc_class); 3995 } 3996 3997 module_init(nvme_fc_init_module); 3998 module_exit(nvme_fc_exit_module); 3999 4000 MODULE_DESCRIPTION("NVMe host FC transport driver"); 4001 MODULE_LICENSE("GPL v2"); 4002