1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2016 Avago Technologies. All rights reserved.
4 */
5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6 #include <linux/module.h>
7 #include <linux/slab.h>
8 #include <linux/blk-mq.h>
9 #include <linux/parser.h>
10 #include <linux/random.h>
11 #include <uapi/scsi/fc/fc_fs.h>
12 #include <uapi/scsi/fc/fc_els.h>
13
14 #include "nvmet.h"
15 #include <linux/nvme-fc-driver.h>
16 #include <linux/nvme-fc.h>
17 #include "../host/fc.h"
18
19
20 /* *************************** Data Structures/Defines ****************** */
21
22
23 #define NVMET_LS_CTX_COUNT 256
24
25 struct nvmet_fc_tgtport;
26 struct nvmet_fc_tgt_assoc;
27
28 struct nvmet_fc_ls_iod { /* for an LS RQST RCV */
29 struct nvmefc_ls_rsp *lsrsp;
30 struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */
31
32 struct list_head ls_rcv_list; /* tgtport->ls_rcv_list */
33
34 struct nvmet_fc_tgtport *tgtport;
35 struct nvmet_fc_tgt_assoc *assoc;
36 void *hosthandle;
37
38 union nvmefc_ls_requests *rqstbuf;
39 union nvmefc_ls_responses *rspbuf;
40 u16 rqstdatalen;
41 dma_addr_t rspdma;
42
43 struct scatterlist sg[2];
44
45 struct work_struct work;
46 } __aligned(sizeof(unsigned long long));
47
48 struct nvmet_fc_ls_req_op { /* for an LS RQST XMT */
49 struct nvmefc_ls_req ls_req;
50
51 struct nvmet_fc_tgtport *tgtport;
52 void *hosthandle;
53
54 int ls_error;
55 struct list_head lsreq_list; /* tgtport->ls_req_list */
56 bool req_queued;
57 };
58
59
60 /* desired maximum for a single sequence - if sg list allows it */
61 #define NVMET_FC_MAX_SEQ_LENGTH (256 * 1024)
62
63 enum nvmet_fcp_datadir {
64 NVMET_FCP_NODATA,
65 NVMET_FCP_WRITE,
66 NVMET_FCP_READ,
67 NVMET_FCP_ABORTED,
68 };
69
70 struct nvmet_fc_fcp_iod {
71 struct nvmefc_tgt_fcp_req *fcpreq;
72
73 struct nvme_fc_cmd_iu cmdiubuf;
74 struct nvme_fc_ersp_iu rspiubuf;
75 dma_addr_t rspdma;
76 struct scatterlist *next_sg;
77 struct scatterlist *data_sg;
78 int data_sg_cnt;
79 u32 offset;
80 enum nvmet_fcp_datadir io_dir;
81 bool active;
82 bool abort;
83 bool aborted;
84 bool writedataactive;
85 spinlock_t flock;
86
87 struct nvmet_req req;
88 struct work_struct defer_work;
89
90 struct nvmet_fc_tgtport *tgtport;
91 struct nvmet_fc_tgt_queue *queue;
92
93 struct list_head fcp_list; /* tgtport->fcp_list */
94 };
95
96 struct nvmet_fc_tgtport {
97 struct nvmet_fc_target_port fc_target_port;
98
99 struct list_head tgt_list; /* nvmet_fc_target_list */
100 struct device *dev; /* dev for dma mapping */
101 struct nvmet_fc_target_template *ops;
102
103 struct nvmet_fc_ls_iod *iod;
104 spinlock_t lock;
105 struct list_head ls_rcv_list;
106 struct list_head ls_req_list;
107 struct list_head ls_busylist;
108 struct list_head assoc_list;
109 struct list_head host_list;
110 struct ida assoc_cnt;
111 struct nvmet_fc_port_entry *pe;
112 struct kref ref;
113 u32 max_sg_cnt;
114
115 struct work_struct put_work;
116 };
117
118 struct nvmet_fc_port_entry {
119 struct nvmet_fc_tgtport *tgtport;
120 struct nvmet_port *port;
121 u64 node_name;
122 u64 port_name;
123 struct list_head pe_list;
124 };
125
126 struct nvmet_fc_defer_fcp_req {
127 struct list_head req_list;
128 struct nvmefc_tgt_fcp_req *fcp_req;
129 };
130
131 struct nvmet_fc_tgt_queue {
132 bool ninetypercent;
133 u16 qid;
134 u16 sqsize;
135 u16 ersp_ratio;
136 __le16 sqhd;
137 atomic_t connected;
138 atomic_t sqtail;
139 atomic_t zrspcnt;
140 atomic_t rsn;
141 spinlock_t qlock;
142 struct nvmet_cq nvme_cq;
143 struct nvmet_sq nvme_sq;
144 struct nvmet_fc_tgt_assoc *assoc;
145 struct list_head fod_list;
146 struct list_head pending_cmd_list;
147 struct list_head avail_defer_list;
148 struct workqueue_struct *work_q;
149 struct kref ref;
150 /* array of fcp_iods */
151 struct nvmet_fc_fcp_iod fod[] /* __counted_by(sqsize) */;
152 } __aligned(sizeof(unsigned long long));
153
154 struct nvmet_fc_hostport {
155 struct nvmet_fc_tgtport *tgtport;
156 void *hosthandle;
157 struct list_head host_list;
158 struct kref ref;
159 u8 invalid;
160 };
161
162 struct nvmet_fc_tgt_assoc {
163 u64 association_id;
164 u32 a_id;
165 atomic_t terminating;
166 struct nvmet_fc_tgtport *tgtport;
167 struct nvmet_fc_hostport *hostport;
168 struct nvmet_fc_ls_iod *rcv_disconn;
169 struct list_head a_list;
170 struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES + 1];
171 struct kref ref;
172 struct work_struct del_work;
173 };
174
175 /*
176 * Association and Connection IDs:
177 *
178 * Association ID will have random number in upper 6 bytes and zero
179 * in lower 2 bytes
180 *
181 * Connection IDs will be Association ID with QID or'd in lower 2 bytes
182 *
183 * note: Association ID = Connection ID for queue 0
184 */
185 #define BYTES_FOR_QID sizeof(u16)
186 #define BYTES_FOR_QID_SHIFT (BYTES_FOR_QID * 8)
187 #define NVMET_FC_QUEUEID_MASK ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1))
188
189 static inline u64
nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc * assoc,u16 qid)190 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid)
191 {
192 return (assoc->association_id | qid);
193 }
194
195 static inline u64
nvmet_fc_getassociationid(u64 connectionid)196 nvmet_fc_getassociationid(u64 connectionid)
197 {
198 return connectionid & ~NVMET_FC_QUEUEID_MASK;
199 }
200
201 static inline u16
nvmet_fc_getqueueid(u64 connectionid)202 nvmet_fc_getqueueid(u64 connectionid)
203 {
204 return (u16)(connectionid & NVMET_FC_QUEUEID_MASK);
205 }
206
207 static inline struct nvmet_fc_tgtport *
targetport_to_tgtport(struct nvmet_fc_target_port * targetport)208 targetport_to_tgtport(struct nvmet_fc_target_port *targetport)
209 {
210 return container_of(targetport, struct nvmet_fc_tgtport,
211 fc_target_port);
212 }
213
214 static inline struct nvmet_fc_fcp_iod *
nvmet_req_to_fod(struct nvmet_req * nvme_req)215 nvmet_req_to_fod(struct nvmet_req *nvme_req)
216 {
217 return container_of(nvme_req, struct nvmet_fc_fcp_iod, req);
218 }
219
220
221 /* *************************** Globals **************************** */
222
223
224 static DEFINE_SPINLOCK(nvmet_fc_tgtlock);
225
226 static LIST_HEAD(nvmet_fc_target_list);
227 static DEFINE_IDA(nvmet_fc_tgtport_cnt);
228 static LIST_HEAD(nvmet_fc_portentry_list);
229
230
231 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work);
232 static void nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work);
233 static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc);
234 static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc);
235 static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue);
236 static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue);
237 static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport);
nvmet_fc_put_tgtport_work(struct work_struct * work)238 static void nvmet_fc_put_tgtport_work(struct work_struct *work)
239 {
240 struct nvmet_fc_tgtport *tgtport =
241 container_of(work, struct nvmet_fc_tgtport, put_work);
242
243 nvmet_fc_tgtport_put(tgtport);
244 }
245 static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport);
246 static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
247 struct nvmet_fc_fcp_iod *fod);
248 static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc);
249 static void nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
250 struct nvmet_fc_ls_iod *iod);
251
252
253 /* *********************** FC-NVME DMA Handling **************************** */
254
255 /*
256 * The fcloop device passes in a NULL device pointer. Real LLD's will
257 * pass in a valid device pointer. If NULL is passed to the dma mapping
258 * routines, depending on the platform, it may or may not succeed, and
259 * may crash.
260 *
261 * As such:
262 * Wrapper all the dma routines and check the dev pointer.
263 *
264 * If simple mappings (return just a dma address, we'll noop them,
265 * returning a dma address of 0.
266 *
267 * On more complex mappings (dma_map_sg), a pseudo routine fills
268 * in the scatter list, setting all dma addresses to 0.
269 */
270
271 static inline dma_addr_t
fc_dma_map_single(struct device * dev,void * ptr,size_t size,enum dma_data_direction dir)272 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
273 enum dma_data_direction dir)
274 {
275 return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
276 }
277
278 static inline int
fc_dma_mapping_error(struct device * dev,dma_addr_t dma_addr)279 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
280 {
281 return dev ? dma_mapping_error(dev, dma_addr) : 0;
282 }
283
284 static inline void
fc_dma_unmap_single(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)285 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
286 enum dma_data_direction dir)
287 {
288 if (dev)
289 dma_unmap_single(dev, addr, size, dir);
290 }
291
292 static inline void
fc_dma_sync_single_for_cpu(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)293 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
294 enum dma_data_direction dir)
295 {
296 if (dev)
297 dma_sync_single_for_cpu(dev, addr, size, dir);
298 }
299
300 static inline void
fc_dma_sync_single_for_device(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)301 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
302 enum dma_data_direction dir)
303 {
304 if (dev)
305 dma_sync_single_for_device(dev, addr, size, dir);
306 }
307
308 /* pseudo dma_map_sg call */
309 static int
fc_map_sg(struct scatterlist * sg,int nents)310 fc_map_sg(struct scatterlist *sg, int nents)
311 {
312 struct scatterlist *s;
313 int i;
314
315 WARN_ON(nents == 0 || sg[0].length == 0);
316
317 for_each_sg(sg, s, nents, i) {
318 s->dma_address = 0L;
319 #ifdef CONFIG_NEED_SG_DMA_LENGTH
320 s->dma_length = s->length;
321 #endif
322 }
323 return nents;
324 }
325
326 static inline int
fc_dma_map_sg(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir)327 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
328 enum dma_data_direction dir)
329 {
330 return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
331 }
332
333 static inline void
fc_dma_unmap_sg(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir)334 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
335 enum dma_data_direction dir)
336 {
337 if (dev)
338 dma_unmap_sg(dev, sg, nents, dir);
339 }
340
341
342 /* ********************** FC-NVME LS XMT Handling ************************* */
343
344
345 static void
__nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op * lsop)346 __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop)
347 {
348 struct nvmet_fc_tgtport *tgtport = lsop->tgtport;
349 struct nvmefc_ls_req *lsreq = &lsop->ls_req;
350 unsigned long flags;
351
352 spin_lock_irqsave(&tgtport->lock, flags);
353
354 if (!lsop->req_queued) {
355 spin_unlock_irqrestore(&tgtport->lock, flags);
356 goto out_putwork;
357 }
358
359 list_del(&lsop->lsreq_list);
360
361 lsop->req_queued = false;
362
363 spin_unlock_irqrestore(&tgtport->lock, flags);
364
365 fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma,
366 (lsreq->rqstlen + lsreq->rsplen),
367 DMA_BIDIRECTIONAL);
368
369 out_putwork:
370 queue_work(nvmet_wq, &tgtport->put_work);
371 }
372
373 static int
__nvmet_fc_send_ls_req(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_req_op * lsop,void (* done)(struct nvmefc_ls_req * req,int status))374 __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport,
375 struct nvmet_fc_ls_req_op *lsop,
376 void (*done)(struct nvmefc_ls_req *req, int status))
377 {
378 struct nvmefc_ls_req *lsreq = &lsop->ls_req;
379 unsigned long flags;
380 int ret = 0;
381
382 if (!tgtport->ops->ls_req)
383 return -EOPNOTSUPP;
384
385 if (!nvmet_fc_tgtport_get(tgtport))
386 return -ESHUTDOWN;
387
388 lsreq->done = done;
389 lsop->req_queued = false;
390 INIT_LIST_HEAD(&lsop->lsreq_list);
391
392 lsreq->rqstdma = fc_dma_map_single(tgtport->dev, lsreq->rqstaddr,
393 lsreq->rqstlen + lsreq->rsplen,
394 DMA_BIDIRECTIONAL);
395 if (fc_dma_mapping_error(tgtport->dev, lsreq->rqstdma)) {
396 ret = -EFAULT;
397 goto out_puttgtport;
398 }
399 lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
400
401 spin_lock_irqsave(&tgtport->lock, flags);
402
403 list_add_tail(&lsop->lsreq_list, &tgtport->ls_req_list);
404
405 lsop->req_queued = true;
406
407 spin_unlock_irqrestore(&tgtport->lock, flags);
408
409 ret = tgtport->ops->ls_req(&tgtport->fc_target_port, lsop->hosthandle,
410 lsreq);
411 if (ret)
412 goto out_unlink;
413
414 return 0;
415
416 out_unlink:
417 lsop->ls_error = ret;
418 spin_lock_irqsave(&tgtport->lock, flags);
419 lsop->req_queued = false;
420 list_del(&lsop->lsreq_list);
421 spin_unlock_irqrestore(&tgtport->lock, flags);
422 fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma,
423 (lsreq->rqstlen + lsreq->rsplen),
424 DMA_BIDIRECTIONAL);
425 out_puttgtport:
426 nvmet_fc_tgtport_put(tgtport);
427
428 return ret;
429 }
430
431 static int
nvmet_fc_send_ls_req_async(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_req_op * lsop,void (* done)(struct nvmefc_ls_req * req,int status))432 nvmet_fc_send_ls_req_async(struct nvmet_fc_tgtport *tgtport,
433 struct nvmet_fc_ls_req_op *lsop,
434 void (*done)(struct nvmefc_ls_req *req, int status))
435 {
436 /* don't wait for completion */
437
438 return __nvmet_fc_send_ls_req(tgtport, lsop, done);
439 }
440
441 static void
nvmet_fc_disconnect_assoc_done(struct nvmefc_ls_req * lsreq,int status)442 nvmet_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
443 {
444 struct nvmet_fc_ls_req_op *lsop =
445 container_of(lsreq, struct nvmet_fc_ls_req_op, ls_req);
446
447 __nvmet_fc_finish_ls_req(lsop);
448
449 /* fc-nvme target doesn't care about success or failure of cmd */
450
451 kfree(lsop);
452 }
453
454 /*
455 * This routine sends a FC-NVME LS to disconnect (aka terminate)
456 * the FC-NVME Association. Terminating the association also
457 * terminates the FC-NVME connections (per queue, both admin and io
458 * queues) that are part of the association. E.g. things are torn
459 * down, and the related FC-NVME Association ID and Connection IDs
460 * become invalid.
461 *
462 * The behavior of the fc-nvme target is such that it's
463 * understanding of the association and connections will implicitly
464 * be torn down. The action is implicit as it may be due to a loss of
465 * connectivity with the fc-nvme host, so the target may never get a
466 * response even if it tried. As such, the action of this routine
467 * is to asynchronously send the LS, ignore any results of the LS, and
468 * continue on with terminating the association. If the fc-nvme host
469 * is present and receives the LS, it too can tear down.
470 */
471 static void
nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc * assoc)472 nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)
473 {
474 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
475 struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
476 struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
477 struct nvmet_fc_ls_req_op *lsop;
478 struct nvmefc_ls_req *lsreq;
479 int ret;
480
481 /*
482 * If ls_req is NULL or no hosthandle, it's an older lldd and no
483 * message is normal. Otherwise, send unless the hostport has
484 * already been invalidated by the lldd.
485 */
486 if (!tgtport->ops->ls_req || assoc->hostport->invalid)
487 return;
488
489 lsop = kzalloc((sizeof(*lsop) +
490 sizeof(*discon_rqst) + sizeof(*discon_acc) +
491 tgtport->ops->lsrqst_priv_sz), GFP_KERNEL);
492 if (!lsop) {
493 dev_info(tgtport->dev,
494 "{%d:%d} send Disconnect Association failed: ENOMEM\n",
495 tgtport->fc_target_port.port_num, assoc->a_id);
496 return;
497 }
498
499 discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
500 discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
501 lsreq = &lsop->ls_req;
502 if (tgtport->ops->lsrqst_priv_sz)
503 lsreq->private = (void *)&discon_acc[1];
504 else
505 lsreq->private = NULL;
506
507 lsop->tgtport = tgtport;
508 lsop->hosthandle = assoc->hostport->hosthandle;
509
510 nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
511 assoc->association_id);
512
513 ret = nvmet_fc_send_ls_req_async(tgtport, lsop,
514 nvmet_fc_disconnect_assoc_done);
515 if (ret) {
516 dev_info(tgtport->dev,
517 "{%d:%d} XMT Disconnect Association failed: %d\n",
518 tgtport->fc_target_port.port_num, assoc->a_id, ret);
519 kfree(lsop);
520 }
521 }
522
523
524 /* *********************** FC-NVME Port Management ************************ */
525
526
527 static int
nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport * tgtport)528 nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
529 {
530 struct nvmet_fc_ls_iod *iod;
531 int i;
532
533 iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod),
534 GFP_KERNEL);
535 if (!iod)
536 return -ENOMEM;
537
538 tgtport->iod = iod;
539
540 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
541 INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work);
542 iod->tgtport = tgtport;
543 list_add_tail(&iod->ls_rcv_list, &tgtport->ls_rcv_list);
544
545 iod->rqstbuf = kzalloc(sizeof(union nvmefc_ls_requests) +
546 sizeof(union nvmefc_ls_responses),
547 GFP_KERNEL);
548 if (!iod->rqstbuf)
549 goto out_fail;
550
551 iod->rspbuf = (union nvmefc_ls_responses *)&iod->rqstbuf[1];
552
553 iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf,
554 sizeof(*iod->rspbuf),
555 DMA_TO_DEVICE);
556 if (fc_dma_mapping_error(tgtport->dev, iod->rspdma))
557 goto out_fail;
558 }
559
560 return 0;
561
562 out_fail:
563 kfree(iod->rqstbuf);
564 list_del(&iod->ls_rcv_list);
565 for (iod--, i--; i >= 0; iod--, i--) {
566 fc_dma_unmap_single(tgtport->dev, iod->rspdma,
567 sizeof(*iod->rspbuf), DMA_TO_DEVICE);
568 kfree(iod->rqstbuf);
569 list_del(&iod->ls_rcv_list);
570 }
571
572 kfree(iod);
573
574 return -EFAULT;
575 }
576
577 static void
nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport * tgtport)578 nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
579 {
580 struct nvmet_fc_ls_iod *iod = tgtport->iod;
581 int i;
582
583 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
584 fc_dma_unmap_single(tgtport->dev,
585 iod->rspdma, sizeof(*iod->rspbuf),
586 DMA_TO_DEVICE);
587 kfree(iod->rqstbuf);
588 list_del(&iod->ls_rcv_list);
589 }
590 kfree(tgtport->iod);
591 }
592
593 static struct nvmet_fc_ls_iod *
nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport * tgtport)594 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport)
595 {
596 struct nvmet_fc_ls_iod *iod;
597 unsigned long flags;
598
599 spin_lock_irqsave(&tgtport->lock, flags);
600 iod = list_first_entry_or_null(&tgtport->ls_rcv_list,
601 struct nvmet_fc_ls_iod, ls_rcv_list);
602 if (iod)
603 list_move_tail(&iod->ls_rcv_list, &tgtport->ls_busylist);
604 spin_unlock_irqrestore(&tgtport->lock, flags);
605 return iod;
606 }
607
608
609 static void
nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_iod * iod)610 nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport,
611 struct nvmet_fc_ls_iod *iod)
612 {
613 unsigned long flags;
614
615 spin_lock_irqsave(&tgtport->lock, flags);
616 list_move(&iod->ls_rcv_list, &tgtport->ls_rcv_list);
617 spin_unlock_irqrestore(&tgtport->lock, flags);
618 }
619
620 static void
nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_tgt_queue * queue)621 nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
622 struct nvmet_fc_tgt_queue *queue)
623 {
624 struct nvmet_fc_fcp_iod *fod = queue->fod;
625 int i;
626
627 for (i = 0; i < queue->sqsize; fod++, i++) {
628 INIT_WORK(&fod->defer_work, nvmet_fc_fcp_rqst_op_defer_work);
629 fod->tgtport = tgtport;
630 fod->queue = queue;
631 fod->active = false;
632 fod->abort = false;
633 fod->aborted = false;
634 fod->fcpreq = NULL;
635 list_add_tail(&fod->fcp_list, &queue->fod_list);
636 spin_lock_init(&fod->flock);
637
638 fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf,
639 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
640 if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) {
641 list_del(&fod->fcp_list);
642 for (fod--, i--; i >= 0; fod--, i--) {
643 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
644 sizeof(fod->rspiubuf),
645 DMA_TO_DEVICE);
646 fod->rspdma = 0L;
647 list_del(&fod->fcp_list);
648 }
649
650 return;
651 }
652 }
653 }
654
655 static void
nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_tgt_queue * queue)656 nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
657 struct nvmet_fc_tgt_queue *queue)
658 {
659 struct nvmet_fc_fcp_iod *fod = queue->fod;
660 int i;
661
662 for (i = 0; i < queue->sqsize; fod++, i++) {
663 if (fod->rspdma)
664 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
665 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
666 }
667 }
668
669 static struct nvmet_fc_fcp_iod *
nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue * queue)670 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue)
671 {
672 struct nvmet_fc_fcp_iod *fod;
673
674 lockdep_assert_held(&queue->qlock);
675
676 fod = list_first_entry_or_null(&queue->fod_list,
677 struct nvmet_fc_fcp_iod, fcp_list);
678 if (fod) {
679 list_del(&fod->fcp_list);
680 fod->active = true;
681 /*
682 * no queue reference is taken, as it was taken by the
683 * queue lookup just prior to the allocation. The iod
684 * will "inherit" that reference.
685 */
686 }
687 return fod;
688 }
689
690
691 static void
nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_tgt_queue * queue,struct nvmefc_tgt_fcp_req * fcpreq)692 nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport,
693 struct nvmet_fc_tgt_queue *queue,
694 struct nvmefc_tgt_fcp_req *fcpreq)
695 {
696 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
697
698 /*
699 * put all admin cmds on hw queue id 0. All io commands go to
700 * the respective hw queue based on a modulo basis
701 */
702 fcpreq->hwqid = queue->qid ?
703 ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0;
704
705 nvmet_fc_handle_fcp_rqst(tgtport, fod);
706 }
707
708 static void
nvmet_fc_fcp_rqst_op_defer_work(struct work_struct * work)709 nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work)
710 {
711 struct nvmet_fc_fcp_iod *fod =
712 container_of(work, struct nvmet_fc_fcp_iod, defer_work);
713
714 /* Submit deferred IO for processing */
715 nvmet_fc_queue_fcp_req(fod->tgtport, fod->queue, fod->fcpreq);
716
717 }
718
719 static void
nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue * queue,struct nvmet_fc_fcp_iod * fod)720 nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue,
721 struct nvmet_fc_fcp_iod *fod)
722 {
723 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
724 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
725 struct nvmet_fc_defer_fcp_req *deferfcp;
726 unsigned long flags;
727
728 fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma,
729 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
730
731 fcpreq->nvmet_fc_private = NULL;
732
733 fod->active = false;
734 fod->abort = false;
735 fod->aborted = false;
736 fod->writedataactive = false;
737 fod->fcpreq = NULL;
738
739 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq);
740
741 /* release the queue lookup reference on the completed IO */
742 nvmet_fc_tgt_q_put(queue);
743
744 spin_lock_irqsave(&queue->qlock, flags);
745 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
746 struct nvmet_fc_defer_fcp_req, req_list);
747 if (!deferfcp) {
748 list_add_tail(&fod->fcp_list, &fod->queue->fod_list);
749 spin_unlock_irqrestore(&queue->qlock, flags);
750 return;
751 }
752
753 /* Re-use the fod for the next pending cmd that was deferred */
754 list_del(&deferfcp->req_list);
755
756 fcpreq = deferfcp->fcp_req;
757
758 /* deferfcp can be reused for another IO at a later date */
759 list_add_tail(&deferfcp->req_list, &queue->avail_defer_list);
760
761 spin_unlock_irqrestore(&queue->qlock, flags);
762
763 /* Save NVME CMD IO in fod */
764 memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen);
765
766 /* Setup new fcpreq to be processed */
767 fcpreq->rspaddr = NULL;
768 fcpreq->rsplen = 0;
769 fcpreq->nvmet_fc_private = fod;
770 fod->fcpreq = fcpreq;
771 fod->active = true;
772
773 /* inform LLDD IO is now being processed */
774 tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq);
775
776 /*
777 * Leave the queue lookup get reference taken when
778 * fod was originally allocated.
779 */
780
781 queue_work(queue->work_q, &fod->defer_work);
782 }
783
784 static struct nvmet_fc_tgt_queue *
nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc * assoc,u16 qid,u16 sqsize)785 nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,
786 u16 qid, u16 sqsize)
787 {
788 struct nvmet_fc_tgt_queue *queue;
789 int ret;
790
791 if (qid > NVMET_NR_QUEUES)
792 return NULL;
793
794 queue = kzalloc(struct_size(queue, fod, sqsize), GFP_KERNEL);
795 if (!queue)
796 return NULL;
797
798 queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0,
799 assoc->tgtport->fc_target_port.port_num,
800 assoc->a_id, qid);
801 if (!queue->work_q)
802 goto out_free_queue;
803
804 queue->qid = qid;
805 queue->sqsize = sqsize;
806 queue->assoc = assoc;
807 INIT_LIST_HEAD(&queue->fod_list);
808 INIT_LIST_HEAD(&queue->avail_defer_list);
809 INIT_LIST_HEAD(&queue->pending_cmd_list);
810 atomic_set(&queue->connected, 0);
811 atomic_set(&queue->sqtail, 0);
812 atomic_set(&queue->rsn, 1);
813 atomic_set(&queue->zrspcnt, 0);
814 spin_lock_init(&queue->qlock);
815 kref_init(&queue->ref);
816
817 nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue);
818
819 ret = nvmet_sq_init(&queue->nvme_sq);
820 if (ret)
821 goto out_fail_iodlist;
822
823 WARN_ON(assoc->queues[qid]);
824 assoc->queues[qid] = queue;
825
826 return queue;
827
828 out_fail_iodlist:
829 nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue);
830 destroy_workqueue(queue->work_q);
831 out_free_queue:
832 kfree(queue);
833 return NULL;
834 }
835
836
837 static void
nvmet_fc_tgt_queue_free(struct kref * ref)838 nvmet_fc_tgt_queue_free(struct kref *ref)
839 {
840 struct nvmet_fc_tgt_queue *queue =
841 container_of(ref, struct nvmet_fc_tgt_queue, ref);
842
843 nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue);
844
845 destroy_workqueue(queue->work_q);
846
847 kfree(queue);
848 }
849
850 static void
nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue * queue)851 nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue)
852 {
853 kref_put(&queue->ref, nvmet_fc_tgt_queue_free);
854 }
855
856 static int
nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue * queue)857 nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue)
858 {
859 return kref_get_unless_zero(&queue->ref);
860 }
861
862
863 static void
nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue * queue)864 nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
865 {
866 struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport;
867 struct nvmet_fc_fcp_iod *fod = queue->fod;
868 struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr;
869 unsigned long flags;
870 int i;
871 bool disconnect;
872
873 disconnect = atomic_xchg(&queue->connected, 0);
874
875 /* if not connected, nothing to do */
876 if (!disconnect)
877 return;
878
879 spin_lock_irqsave(&queue->qlock, flags);
880 /* abort outstanding io's */
881 for (i = 0; i < queue->sqsize; fod++, i++) {
882 if (fod->active) {
883 spin_lock(&fod->flock);
884 fod->abort = true;
885 /*
886 * only call lldd abort routine if waiting for
887 * writedata. other outstanding ops should finish
888 * on their own.
889 */
890 if (fod->writedataactive) {
891 fod->aborted = true;
892 spin_unlock(&fod->flock);
893 tgtport->ops->fcp_abort(
894 &tgtport->fc_target_port, fod->fcpreq);
895 } else
896 spin_unlock(&fod->flock);
897 }
898 }
899
900 /* Cleanup defer'ed IOs in queue */
901 list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list,
902 req_list) {
903 list_del(&deferfcp->req_list);
904 kfree(deferfcp);
905 }
906
907 for (;;) {
908 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
909 struct nvmet_fc_defer_fcp_req, req_list);
910 if (!deferfcp)
911 break;
912
913 list_del(&deferfcp->req_list);
914 spin_unlock_irqrestore(&queue->qlock, flags);
915
916 tgtport->ops->defer_rcv(&tgtport->fc_target_port,
917 deferfcp->fcp_req);
918
919 tgtport->ops->fcp_abort(&tgtport->fc_target_port,
920 deferfcp->fcp_req);
921
922 tgtport->ops->fcp_req_release(&tgtport->fc_target_port,
923 deferfcp->fcp_req);
924
925 /* release the queue lookup reference */
926 nvmet_fc_tgt_q_put(queue);
927
928 kfree(deferfcp);
929
930 spin_lock_irqsave(&queue->qlock, flags);
931 }
932 spin_unlock_irqrestore(&queue->qlock, flags);
933
934 flush_workqueue(queue->work_q);
935
936 nvmet_sq_destroy(&queue->nvme_sq);
937
938 nvmet_fc_tgt_q_put(queue);
939 }
940
941 static struct nvmet_fc_tgt_queue *
nvmet_fc_find_target_queue(struct nvmet_fc_tgtport * tgtport,u64 connection_id)942 nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport,
943 u64 connection_id)
944 {
945 struct nvmet_fc_tgt_assoc *assoc;
946 struct nvmet_fc_tgt_queue *queue;
947 u64 association_id = nvmet_fc_getassociationid(connection_id);
948 u16 qid = nvmet_fc_getqueueid(connection_id);
949
950 if (qid > NVMET_NR_QUEUES)
951 return NULL;
952
953 rcu_read_lock();
954 list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
955 if (association_id == assoc->association_id) {
956 queue = assoc->queues[qid];
957 if (queue &&
958 (!atomic_read(&queue->connected) ||
959 !nvmet_fc_tgt_q_get(queue)))
960 queue = NULL;
961 rcu_read_unlock();
962 return queue;
963 }
964 }
965 rcu_read_unlock();
966 return NULL;
967 }
968
969 static void
nvmet_fc_hostport_free(struct kref * ref)970 nvmet_fc_hostport_free(struct kref *ref)
971 {
972 struct nvmet_fc_hostport *hostport =
973 container_of(ref, struct nvmet_fc_hostport, ref);
974 struct nvmet_fc_tgtport *tgtport = hostport->tgtport;
975 unsigned long flags;
976
977 spin_lock_irqsave(&tgtport->lock, flags);
978 list_del(&hostport->host_list);
979 spin_unlock_irqrestore(&tgtport->lock, flags);
980 if (tgtport->ops->host_release && hostport->invalid)
981 tgtport->ops->host_release(hostport->hosthandle);
982 kfree(hostport);
983 nvmet_fc_tgtport_put(tgtport);
984 }
985
986 static void
nvmet_fc_hostport_put(struct nvmet_fc_hostport * hostport)987 nvmet_fc_hostport_put(struct nvmet_fc_hostport *hostport)
988 {
989 kref_put(&hostport->ref, nvmet_fc_hostport_free);
990 }
991
992 static int
nvmet_fc_hostport_get(struct nvmet_fc_hostport * hostport)993 nvmet_fc_hostport_get(struct nvmet_fc_hostport *hostport)
994 {
995 return kref_get_unless_zero(&hostport->ref);
996 }
997
998 static struct nvmet_fc_hostport *
nvmet_fc_match_hostport(struct nvmet_fc_tgtport * tgtport,void * hosthandle)999 nvmet_fc_match_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle)
1000 {
1001 struct nvmet_fc_hostport *host;
1002
1003 lockdep_assert_held(&tgtport->lock);
1004
1005 list_for_each_entry(host, &tgtport->host_list, host_list) {
1006 if (host->hosthandle == hosthandle && !host->invalid) {
1007 if (nvmet_fc_hostport_get(host))
1008 return host;
1009 }
1010 }
1011
1012 return NULL;
1013 }
1014
1015 static struct nvmet_fc_hostport *
nvmet_fc_alloc_hostport(struct nvmet_fc_tgtport * tgtport,void * hosthandle)1016 nvmet_fc_alloc_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle)
1017 {
1018 struct nvmet_fc_hostport *newhost, *match = NULL;
1019 unsigned long flags;
1020
1021 /*
1022 * Caller holds a reference on tgtport.
1023 */
1024
1025 /* if LLDD not implemented, leave as NULL */
1026 if (!hosthandle)
1027 return NULL;
1028
1029 spin_lock_irqsave(&tgtport->lock, flags);
1030 match = nvmet_fc_match_hostport(tgtport, hosthandle);
1031 spin_unlock_irqrestore(&tgtport->lock, flags);
1032
1033 if (match)
1034 return match;
1035
1036 newhost = kzalloc(sizeof(*newhost), GFP_KERNEL);
1037 if (!newhost)
1038 return ERR_PTR(-ENOMEM);
1039
1040 spin_lock_irqsave(&tgtport->lock, flags);
1041 match = nvmet_fc_match_hostport(tgtport, hosthandle);
1042 if (match) {
1043 /* new allocation not needed */
1044 kfree(newhost);
1045 newhost = match;
1046 } else {
1047 nvmet_fc_tgtport_get(tgtport);
1048 newhost->tgtport = tgtport;
1049 newhost->hosthandle = hosthandle;
1050 INIT_LIST_HEAD(&newhost->host_list);
1051 kref_init(&newhost->ref);
1052
1053 list_add_tail(&newhost->host_list, &tgtport->host_list);
1054 }
1055 spin_unlock_irqrestore(&tgtport->lock, flags);
1056
1057 return newhost;
1058 }
1059
1060 static void
nvmet_fc_delete_assoc_work(struct work_struct * work)1061 nvmet_fc_delete_assoc_work(struct work_struct *work)
1062 {
1063 struct nvmet_fc_tgt_assoc *assoc =
1064 container_of(work, struct nvmet_fc_tgt_assoc, del_work);
1065 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
1066
1067 nvmet_fc_delete_target_assoc(assoc);
1068 nvmet_fc_tgt_a_put(assoc);
1069 nvmet_fc_tgtport_put(tgtport);
1070 }
1071
1072 static void
nvmet_fc_schedule_delete_assoc(struct nvmet_fc_tgt_assoc * assoc)1073 nvmet_fc_schedule_delete_assoc(struct nvmet_fc_tgt_assoc *assoc)
1074 {
1075 nvmet_fc_tgtport_get(assoc->tgtport);
1076 if (!queue_work(nvmet_wq, &assoc->del_work))
1077 nvmet_fc_tgtport_put(assoc->tgtport);
1078 }
1079
1080 static bool
nvmet_fc_assoc_exists(struct nvmet_fc_tgtport * tgtport,u64 association_id)1081 nvmet_fc_assoc_exists(struct nvmet_fc_tgtport *tgtport, u64 association_id)
1082 {
1083 struct nvmet_fc_tgt_assoc *a;
1084 bool found = false;
1085
1086 rcu_read_lock();
1087 list_for_each_entry_rcu(a, &tgtport->assoc_list, a_list) {
1088 if (association_id == a->association_id) {
1089 found = true;
1090 break;
1091 }
1092 }
1093 rcu_read_unlock();
1094
1095 return found;
1096 }
1097
1098 static struct nvmet_fc_tgt_assoc *
nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport * tgtport,void * hosthandle)1099 nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle)
1100 {
1101 struct nvmet_fc_tgt_assoc *assoc;
1102 unsigned long flags;
1103 bool done;
1104 u64 ran;
1105 int idx;
1106
1107 if (!tgtport->pe)
1108 return NULL;
1109
1110 assoc = kzalloc(sizeof(*assoc), GFP_KERNEL);
1111 if (!assoc)
1112 return NULL;
1113
1114 idx = ida_alloc(&tgtport->assoc_cnt, GFP_KERNEL);
1115 if (idx < 0)
1116 goto out_free_assoc;
1117
1118 assoc->hostport = nvmet_fc_alloc_hostport(tgtport, hosthandle);
1119 if (IS_ERR(assoc->hostport))
1120 goto out_ida;
1121
1122 assoc->tgtport = tgtport;
1123 nvmet_fc_tgtport_get(tgtport);
1124 assoc->a_id = idx;
1125 INIT_LIST_HEAD(&assoc->a_list);
1126 kref_init(&assoc->ref);
1127 INIT_WORK(&assoc->del_work, nvmet_fc_delete_assoc_work);
1128 atomic_set(&assoc->terminating, 0);
1129
1130 done = false;
1131 do {
1132 get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID);
1133 ran = ran << BYTES_FOR_QID_SHIFT;
1134
1135 spin_lock_irqsave(&tgtport->lock, flags);
1136 if (!nvmet_fc_assoc_exists(tgtport, ran)) {
1137 assoc->association_id = ran;
1138 list_add_tail_rcu(&assoc->a_list, &tgtport->assoc_list);
1139 done = true;
1140 }
1141 spin_unlock_irqrestore(&tgtport->lock, flags);
1142 } while (!done);
1143
1144 return assoc;
1145
1146 out_ida:
1147 ida_free(&tgtport->assoc_cnt, idx);
1148 out_free_assoc:
1149 kfree(assoc);
1150 return NULL;
1151 }
1152
1153 static void
nvmet_fc_target_assoc_free(struct kref * ref)1154 nvmet_fc_target_assoc_free(struct kref *ref)
1155 {
1156 struct nvmet_fc_tgt_assoc *assoc =
1157 container_of(ref, struct nvmet_fc_tgt_assoc, ref);
1158 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
1159 struct nvmet_fc_ls_iod *oldls;
1160 unsigned long flags;
1161 int i;
1162
1163 for (i = NVMET_NR_QUEUES; i >= 0; i--) {
1164 if (assoc->queues[i])
1165 nvmet_fc_delete_target_queue(assoc->queues[i]);
1166 }
1167
1168 /* Send Disconnect now that all i/o has completed */
1169 nvmet_fc_xmt_disconnect_assoc(assoc);
1170
1171 nvmet_fc_hostport_put(assoc->hostport);
1172 spin_lock_irqsave(&tgtport->lock, flags);
1173 oldls = assoc->rcv_disconn;
1174 spin_unlock_irqrestore(&tgtport->lock, flags);
1175 /* if pending Rcv Disconnect Association LS, send rsp now */
1176 if (oldls)
1177 nvmet_fc_xmt_ls_rsp(tgtport, oldls);
1178 ida_free(&tgtport->assoc_cnt, assoc->a_id);
1179 dev_info(tgtport->dev,
1180 "{%d:%d} Association freed\n",
1181 tgtport->fc_target_port.port_num, assoc->a_id);
1182 kfree(assoc);
1183 }
1184
1185 static void
nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc * assoc)1186 nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc)
1187 {
1188 kref_put(&assoc->ref, nvmet_fc_target_assoc_free);
1189 }
1190
1191 static int
nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc * assoc)1192 nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc)
1193 {
1194 return kref_get_unless_zero(&assoc->ref);
1195 }
1196
1197 static void
nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc * assoc)1198 nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc)
1199 {
1200 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
1201 unsigned long flags;
1202 int i, terminating;
1203
1204 terminating = atomic_xchg(&assoc->terminating, 1);
1205
1206 /* if already terminating, do nothing */
1207 if (terminating)
1208 return;
1209
1210 spin_lock_irqsave(&tgtport->lock, flags);
1211 list_del_rcu(&assoc->a_list);
1212 spin_unlock_irqrestore(&tgtport->lock, flags);
1213
1214 synchronize_rcu();
1215
1216 /* ensure all in-flight I/Os have been processed */
1217 for (i = NVMET_NR_QUEUES; i >= 0; i--) {
1218 if (assoc->queues[i])
1219 flush_workqueue(assoc->queues[i]->work_q);
1220 }
1221
1222 dev_info(tgtport->dev,
1223 "{%d:%d} Association deleted\n",
1224 tgtport->fc_target_port.port_num, assoc->a_id);
1225
1226 nvmet_fc_tgtport_put(tgtport);
1227 }
1228
1229 static struct nvmet_fc_tgt_assoc *
nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport * tgtport,u64 association_id)1230 nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport,
1231 u64 association_id)
1232 {
1233 struct nvmet_fc_tgt_assoc *assoc;
1234 struct nvmet_fc_tgt_assoc *ret = NULL;
1235
1236 rcu_read_lock();
1237 list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
1238 if (association_id == assoc->association_id) {
1239 ret = assoc;
1240 if (!nvmet_fc_tgt_a_get(assoc))
1241 ret = NULL;
1242 break;
1243 }
1244 }
1245 rcu_read_unlock();
1246
1247 return ret;
1248 }
1249
1250 static void
nvmet_fc_portentry_bind(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_port_entry * pe,struct nvmet_port * port)1251 nvmet_fc_portentry_bind(struct nvmet_fc_tgtport *tgtport,
1252 struct nvmet_fc_port_entry *pe,
1253 struct nvmet_port *port)
1254 {
1255 lockdep_assert_held(&nvmet_fc_tgtlock);
1256
1257 pe->tgtport = tgtport;
1258 tgtport->pe = pe;
1259
1260 pe->port = port;
1261 port->priv = pe;
1262
1263 pe->node_name = tgtport->fc_target_port.node_name;
1264 pe->port_name = tgtport->fc_target_port.port_name;
1265 INIT_LIST_HEAD(&pe->pe_list);
1266
1267 list_add_tail(&pe->pe_list, &nvmet_fc_portentry_list);
1268 }
1269
1270 static void
nvmet_fc_portentry_unbind(struct nvmet_fc_port_entry * pe)1271 nvmet_fc_portentry_unbind(struct nvmet_fc_port_entry *pe)
1272 {
1273 unsigned long flags;
1274
1275 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1276 if (pe->tgtport)
1277 pe->tgtport->pe = NULL;
1278 list_del(&pe->pe_list);
1279 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1280 }
1281
1282 /*
1283 * called when a targetport deregisters. Breaks the relationship
1284 * with the nvmet port, but leaves the port_entry in place so that
1285 * re-registration can resume operation.
1286 */
1287 static void
nvmet_fc_portentry_unbind_tgt(struct nvmet_fc_tgtport * tgtport)1288 nvmet_fc_portentry_unbind_tgt(struct nvmet_fc_tgtport *tgtport)
1289 {
1290 struct nvmet_fc_port_entry *pe;
1291 unsigned long flags;
1292
1293 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1294 pe = tgtport->pe;
1295 if (pe)
1296 pe->tgtport = NULL;
1297 tgtport->pe = NULL;
1298 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1299 }
1300
1301 /*
1302 * called when a new targetport is registered. Looks in the
1303 * existing nvmet port_entries to see if the nvmet layer is
1304 * configured for the targetport's wwn's. (the targetport existed,
1305 * nvmet configured, the lldd unregistered the tgtport, and is now
1306 * reregistering the same targetport). If so, set the nvmet port
1307 * port entry on the targetport.
1308 */
1309 static void
nvmet_fc_portentry_rebind_tgt(struct nvmet_fc_tgtport * tgtport)1310 nvmet_fc_portentry_rebind_tgt(struct nvmet_fc_tgtport *tgtport)
1311 {
1312 struct nvmet_fc_port_entry *pe;
1313 unsigned long flags;
1314
1315 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1316 list_for_each_entry(pe, &nvmet_fc_portentry_list, pe_list) {
1317 if (tgtport->fc_target_port.node_name == pe->node_name &&
1318 tgtport->fc_target_port.port_name == pe->port_name) {
1319 WARN_ON(pe->tgtport);
1320 tgtport->pe = pe;
1321 pe->tgtport = tgtport;
1322 break;
1323 }
1324 }
1325 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1326 }
1327
1328 /**
1329 * nvmet_fc_register_targetport - transport entry point called by an
1330 * LLDD to register the existence of a local
1331 * NVME subystem FC port.
1332 * @pinfo: pointer to information about the port to be registered
1333 * @template: LLDD entrypoints and operational parameters for the port
1334 * @dev: physical hardware device node port corresponds to. Will be
1335 * used for DMA mappings
1336 * @portptr: pointer to a local port pointer. Upon success, the routine
1337 * will allocate a nvme_fc_local_port structure and place its
1338 * address in the local port pointer. Upon failure, local port
1339 * pointer will be set to NULL.
1340 *
1341 * Returns:
1342 * a completion status. Must be 0 upon success; a negative errno
1343 * (ex: -ENXIO) upon failure.
1344 */
1345 int
nvmet_fc_register_targetport(struct nvmet_fc_port_info * pinfo,struct nvmet_fc_target_template * template,struct device * dev,struct nvmet_fc_target_port ** portptr)1346 nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,
1347 struct nvmet_fc_target_template *template,
1348 struct device *dev,
1349 struct nvmet_fc_target_port **portptr)
1350 {
1351 struct nvmet_fc_tgtport *newrec;
1352 unsigned long flags;
1353 int ret, idx;
1354
1355 if (!template->xmt_ls_rsp || !template->fcp_op ||
1356 !template->fcp_abort ||
1357 !template->fcp_req_release || !template->targetport_delete ||
1358 !template->max_hw_queues || !template->max_sgl_segments ||
1359 !template->max_dif_sgl_segments || !template->dma_boundary) {
1360 ret = -EINVAL;
1361 goto out_regtgt_failed;
1362 }
1363
1364 newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz),
1365 GFP_KERNEL);
1366 if (!newrec) {
1367 ret = -ENOMEM;
1368 goto out_regtgt_failed;
1369 }
1370
1371 idx = ida_alloc(&nvmet_fc_tgtport_cnt, GFP_KERNEL);
1372 if (idx < 0) {
1373 ret = -ENOSPC;
1374 goto out_fail_kfree;
1375 }
1376
1377 if (!get_device(dev) && dev) {
1378 ret = -ENODEV;
1379 goto out_ida_put;
1380 }
1381
1382 newrec->fc_target_port.node_name = pinfo->node_name;
1383 newrec->fc_target_port.port_name = pinfo->port_name;
1384 if (template->target_priv_sz)
1385 newrec->fc_target_port.private = &newrec[1];
1386 else
1387 newrec->fc_target_port.private = NULL;
1388 newrec->fc_target_port.port_id = pinfo->port_id;
1389 newrec->fc_target_port.port_num = idx;
1390 INIT_LIST_HEAD(&newrec->tgt_list);
1391 newrec->dev = dev;
1392 newrec->ops = template;
1393 spin_lock_init(&newrec->lock);
1394 INIT_LIST_HEAD(&newrec->ls_rcv_list);
1395 INIT_LIST_HEAD(&newrec->ls_req_list);
1396 INIT_LIST_HEAD(&newrec->ls_busylist);
1397 INIT_LIST_HEAD(&newrec->assoc_list);
1398 INIT_LIST_HEAD(&newrec->host_list);
1399 kref_init(&newrec->ref);
1400 ida_init(&newrec->assoc_cnt);
1401 newrec->max_sg_cnt = template->max_sgl_segments;
1402 INIT_WORK(&newrec->put_work, nvmet_fc_put_tgtport_work);
1403
1404 ret = nvmet_fc_alloc_ls_iodlist(newrec);
1405 if (ret) {
1406 ret = -ENOMEM;
1407 goto out_free_newrec;
1408 }
1409
1410 nvmet_fc_portentry_rebind_tgt(newrec);
1411
1412 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1413 list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list);
1414 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1415
1416 *portptr = &newrec->fc_target_port;
1417 return 0;
1418
1419 out_free_newrec:
1420 put_device(dev);
1421 out_ida_put:
1422 ida_free(&nvmet_fc_tgtport_cnt, idx);
1423 out_fail_kfree:
1424 kfree(newrec);
1425 out_regtgt_failed:
1426 *portptr = NULL;
1427 return ret;
1428 }
1429 EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport);
1430
1431
1432 static void
nvmet_fc_free_tgtport(struct kref * ref)1433 nvmet_fc_free_tgtport(struct kref *ref)
1434 {
1435 struct nvmet_fc_tgtport *tgtport =
1436 container_of(ref, struct nvmet_fc_tgtport, ref);
1437 struct device *dev = tgtport->dev;
1438
1439 nvmet_fc_free_ls_iodlist(tgtport);
1440
1441 /* let the LLDD know we've finished tearing it down */
1442 tgtport->ops->targetport_delete(&tgtport->fc_target_port);
1443
1444 ida_free(&nvmet_fc_tgtport_cnt,
1445 tgtport->fc_target_port.port_num);
1446
1447 ida_destroy(&tgtport->assoc_cnt);
1448
1449 kfree(tgtport);
1450
1451 put_device(dev);
1452 }
1453
1454 static void
nvmet_fc_tgtport_put(struct nvmet_fc_tgtport * tgtport)1455 nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport)
1456 {
1457 kref_put(&tgtport->ref, nvmet_fc_free_tgtport);
1458 }
1459
1460 static int
nvmet_fc_tgtport_get(struct nvmet_fc_tgtport * tgtport)1461 nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport)
1462 {
1463 return kref_get_unless_zero(&tgtport->ref);
1464 }
1465
1466 static void
__nvmet_fc_free_assocs(struct nvmet_fc_tgtport * tgtport)1467 __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)
1468 {
1469 struct nvmet_fc_tgt_assoc *assoc;
1470
1471 rcu_read_lock();
1472 list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
1473 if (!nvmet_fc_tgt_a_get(assoc))
1474 continue;
1475 nvmet_fc_schedule_delete_assoc(assoc);
1476 nvmet_fc_tgt_a_put(assoc);
1477 }
1478 rcu_read_unlock();
1479 }
1480
1481 /**
1482 * nvmet_fc_invalidate_host - transport entry point called by an LLDD
1483 * to remove references to a hosthandle for LS's.
1484 *
1485 * The nvmet-fc layer ensures that any references to the hosthandle
1486 * on the targetport are forgotten (set to NULL). The LLDD will
1487 * typically call this when a login with a remote host port has been
1488 * lost, thus LS's for the remote host port are no longer possible.
1489 *
1490 * If an LS request is outstanding to the targetport/hosthandle (or
1491 * issued concurrently with the call to invalidate the host), the
1492 * LLDD is responsible for terminating/aborting the LS and completing
1493 * the LS request. It is recommended that these terminations/aborts
1494 * occur after calling to invalidate the host handle to avoid additional
1495 * retries by the nvmet-fc transport. The nvmet-fc transport may
1496 * continue to reference host handle while it cleans up outstanding
1497 * NVME associations. The nvmet-fc transport will call the
1498 * ops->host_release() callback to notify the LLDD that all references
1499 * are complete and the related host handle can be recovered.
1500 * Note: if there are no references, the callback may be called before
1501 * the invalidate host call returns.
1502 *
1503 * @target_port: pointer to the (registered) target port that a prior
1504 * LS was received on and which supplied the transport the
1505 * hosthandle.
1506 * @hosthandle: the handle (pointer) that represents the host port
1507 * that no longer has connectivity and that LS's should
1508 * no longer be directed to.
1509 */
1510 void
nvmet_fc_invalidate_host(struct nvmet_fc_target_port * target_port,void * hosthandle)1511 nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port,
1512 void *hosthandle)
1513 {
1514 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1515 struct nvmet_fc_tgt_assoc *assoc, *next;
1516 unsigned long flags;
1517 bool noassoc = true;
1518
1519 spin_lock_irqsave(&tgtport->lock, flags);
1520 list_for_each_entry_safe(assoc, next,
1521 &tgtport->assoc_list, a_list) {
1522 if (assoc->hostport->hosthandle != hosthandle)
1523 continue;
1524 if (!nvmet_fc_tgt_a_get(assoc))
1525 continue;
1526 assoc->hostport->invalid = 1;
1527 noassoc = false;
1528 nvmet_fc_schedule_delete_assoc(assoc);
1529 nvmet_fc_tgt_a_put(assoc);
1530 }
1531 spin_unlock_irqrestore(&tgtport->lock, flags);
1532
1533 /* if there's nothing to wait for - call the callback */
1534 if (noassoc && tgtport->ops->host_release)
1535 tgtport->ops->host_release(hosthandle);
1536 }
1537 EXPORT_SYMBOL_GPL(nvmet_fc_invalidate_host);
1538
1539 /*
1540 * nvmet layer has called to terminate an association
1541 */
1542 static void
nvmet_fc_delete_ctrl(struct nvmet_ctrl * ctrl)1543 nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
1544 {
1545 struct nvmet_fc_tgtport *tgtport, *next;
1546 struct nvmet_fc_tgt_assoc *assoc;
1547 struct nvmet_fc_tgt_queue *queue;
1548 unsigned long flags;
1549 bool found_ctrl = false;
1550
1551 /* this is a bit ugly, but don't want to make locks layered */
1552 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1553 list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list,
1554 tgt_list) {
1555 if (!nvmet_fc_tgtport_get(tgtport))
1556 continue;
1557 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1558
1559 rcu_read_lock();
1560 list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
1561 queue = assoc->queues[0];
1562 if (queue && queue->nvme_sq.ctrl == ctrl) {
1563 if (nvmet_fc_tgt_a_get(assoc))
1564 found_ctrl = true;
1565 break;
1566 }
1567 }
1568 rcu_read_unlock();
1569
1570 nvmet_fc_tgtport_put(tgtport);
1571
1572 if (found_ctrl) {
1573 nvmet_fc_schedule_delete_assoc(assoc);
1574 nvmet_fc_tgt_a_put(assoc);
1575 return;
1576 }
1577
1578 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1579 }
1580 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1581 }
1582
1583 /**
1584 * nvmet_fc_unregister_targetport - transport entry point called by an
1585 * LLDD to deregister/remove a previously
1586 * registered a local NVME subsystem FC port.
1587 * @target_port: pointer to the (registered) target port that is to be
1588 * deregistered.
1589 *
1590 * Returns:
1591 * a completion status. Must be 0 upon success; a negative errno
1592 * (ex: -ENXIO) upon failure.
1593 */
1594 int
nvmet_fc_unregister_targetport(struct nvmet_fc_target_port * target_port)1595 nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port)
1596 {
1597 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1598 unsigned long flags;
1599
1600 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1601 list_del(&tgtport->tgt_list);
1602 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1603
1604 nvmet_fc_portentry_unbind_tgt(tgtport);
1605
1606 /* terminate any outstanding associations */
1607 __nvmet_fc_free_assocs(tgtport);
1608
1609 flush_workqueue(nvmet_wq);
1610
1611 /*
1612 * should terminate LS's as well. However, LS's will be generated
1613 * at the tail end of association termination, so they likely don't
1614 * exist yet. And even if they did, it's worthwhile to just let
1615 * them finish and targetport ref counting will clean things up.
1616 */
1617
1618 nvmet_fc_tgtport_put(tgtport);
1619
1620 return 0;
1621 }
1622 EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport);
1623
1624
1625 /* ********************** FC-NVME LS RCV Handling ************************* */
1626
1627
1628 static void
nvmet_fc_ls_create_association(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_iod * iod)1629 nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport,
1630 struct nvmet_fc_ls_iod *iod)
1631 {
1632 struct fcnvme_ls_cr_assoc_rqst *rqst = &iod->rqstbuf->rq_cr_assoc;
1633 struct fcnvme_ls_cr_assoc_acc *acc = &iod->rspbuf->rsp_cr_assoc;
1634 struct nvmet_fc_tgt_queue *queue;
1635 int ret = 0;
1636
1637 memset(acc, 0, sizeof(*acc));
1638
1639 /*
1640 * FC-NVME spec changes. There are initiators sending different
1641 * lengths as padding sizes for Create Association Cmd descriptor
1642 * was incorrect.
1643 * Accept anything of "minimum" length. Assume format per 1.15
1644 * spec (with HOSTID reduced to 16 bytes), ignore how long the
1645 * trailing pad length is.
1646 */
1647 if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN)
1648 ret = VERR_CR_ASSOC_LEN;
1649 else if (be32_to_cpu(rqst->desc_list_len) <
1650 FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN)
1651 ret = VERR_CR_ASSOC_RQST_LEN;
1652 else if (rqst->assoc_cmd.desc_tag !=
1653 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD))
1654 ret = VERR_CR_ASSOC_CMD;
1655 else if (be32_to_cpu(rqst->assoc_cmd.desc_len) <
1656 FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN)
1657 ret = VERR_CR_ASSOC_CMD_LEN;
1658 else if (!rqst->assoc_cmd.ersp_ratio ||
1659 (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >=
1660 be16_to_cpu(rqst->assoc_cmd.sqsize)))
1661 ret = VERR_ERSP_RATIO;
1662
1663 else {
1664 /* new association w/ admin queue */
1665 iod->assoc = nvmet_fc_alloc_target_assoc(
1666 tgtport, iod->hosthandle);
1667 if (!iod->assoc)
1668 ret = VERR_ASSOC_ALLOC_FAIL;
1669 else {
1670 queue = nvmet_fc_alloc_target_queue(iod->assoc, 0,
1671 be16_to_cpu(rqst->assoc_cmd.sqsize));
1672 if (!queue) {
1673 ret = VERR_QUEUE_ALLOC_FAIL;
1674 nvmet_fc_tgt_a_put(iod->assoc);
1675 }
1676 }
1677 }
1678
1679 if (ret) {
1680 dev_err(tgtport->dev,
1681 "Create Association LS failed: %s\n",
1682 validation_errors[ret]);
1683 iod->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1684 sizeof(*acc), rqst->w0.ls_cmd,
1685 FCNVME_RJT_RC_LOGIC,
1686 FCNVME_RJT_EXP_NONE, 0);
1687 return;
1688 }
1689
1690 queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio);
1691 atomic_set(&queue->connected, 1);
1692 queue->sqhd = 0; /* best place to init value */
1693
1694 dev_info(tgtport->dev,
1695 "{%d:%d} Association created\n",
1696 tgtport->fc_target_port.port_num, iod->assoc->a_id);
1697
1698 /* format a response */
1699
1700 iod->lsrsp->rsplen = sizeof(*acc);
1701
1702 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1703 fcnvme_lsdesc_len(
1704 sizeof(struct fcnvme_ls_cr_assoc_acc)),
1705 FCNVME_LS_CREATE_ASSOCIATION);
1706 acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1707 acc->associd.desc_len =
1708 fcnvme_lsdesc_len(
1709 sizeof(struct fcnvme_lsdesc_assoc_id));
1710 acc->associd.association_id =
1711 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0));
1712 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1713 acc->connectid.desc_len =
1714 fcnvme_lsdesc_len(
1715 sizeof(struct fcnvme_lsdesc_conn_id));
1716 acc->connectid.connection_id = acc->associd.association_id;
1717 }
1718
1719 static void
nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_iod * iod)1720 nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
1721 struct nvmet_fc_ls_iod *iod)
1722 {
1723 struct fcnvme_ls_cr_conn_rqst *rqst = &iod->rqstbuf->rq_cr_conn;
1724 struct fcnvme_ls_cr_conn_acc *acc = &iod->rspbuf->rsp_cr_conn;
1725 struct nvmet_fc_tgt_queue *queue;
1726 int ret = 0;
1727
1728 memset(acc, 0, sizeof(*acc));
1729
1730 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst))
1731 ret = VERR_CR_CONN_LEN;
1732 else if (rqst->desc_list_len !=
1733 fcnvme_lsdesc_len(
1734 sizeof(struct fcnvme_ls_cr_conn_rqst)))
1735 ret = VERR_CR_CONN_RQST_LEN;
1736 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1737 ret = VERR_ASSOC_ID;
1738 else if (rqst->associd.desc_len !=
1739 fcnvme_lsdesc_len(
1740 sizeof(struct fcnvme_lsdesc_assoc_id)))
1741 ret = VERR_ASSOC_ID_LEN;
1742 else if (rqst->connect_cmd.desc_tag !=
1743 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD))
1744 ret = VERR_CR_CONN_CMD;
1745 else if (rqst->connect_cmd.desc_len !=
1746 fcnvme_lsdesc_len(
1747 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)))
1748 ret = VERR_CR_CONN_CMD_LEN;
1749 else if (!rqst->connect_cmd.ersp_ratio ||
1750 (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >=
1751 be16_to_cpu(rqst->connect_cmd.sqsize)))
1752 ret = VERR_ERSP_RATIO;
1753
1754 else {
1755 /* new io queue */
1756 iod->assoc = nvmet_fc_find_target_assoc(tgtport,
1757 be64_to_cpu(rqst->associd.association_id));
1758 if (!iod->assoc)
1759 ret = VERR_NO_ASSOC;
1760 else {
1761 queue = nvmet_fc_alloc_target_queue(iod->assoc,
1762 be16_to_cpu(rqst->connect_cmd.qid),
1763 be16_to_cpu(rqst->connect_cmd.sqsize));
1764 if (!queue)
1765 ret = VERR_QUEUE_ALLOC_FAIL;
1766
1767 /* release get taken in nvmet_fc_find_target_assoc */
1768 nvmet_fc_tgt_a_put(iod->assoc);
1769 }
1770 }
1771
1772 if (ret) {
1773 dev_err(tgtport->dev,
1774 "Create Connection LS failed: %s\n",
1775 validation_errors[ret]);
1776 iod->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1777 sizeof(*acc), rqst->w0.ls_cmd,
1778 (ret == VERR_NO_ASSOC) ?
1779 FCNVME_RJT_RC_INV_ASSOC :
1780 FCNVME_RJT_RC_LOGIC,
1781 FCNVME_RJT_EXP_NONE, 0);
1782 return;
1783 }
1784
1785 queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio);
1786 atomic_set(&queue->connected, 1);
1787 queue->sqhd = 0; /* best place to init value */
1788
1789 /* format a response */
1790
1791 iod->lsrsp->rsplen = sizeof(*acc);
1792
1793 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1794 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)),
1795 FCNVME_LS_CREATE_CONNECTION);
1796 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1797 acc->connectid.desc_len =
1798 fcnvme_lsdesc_len(
1799 sizeof(struct fcnvme_lsdesc_conn_id));
1800 acc->connectid.connection_id =
1801 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc,
1802 be16_to_cpu(rqst->connect_cmd.qid)));
1803 }
1804
1805 /*
1806 * Returns true if the LS response is to be transmit
1807 * Returns false if the LS response is to be delayed
1808 */
1809 static int
nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_iod * iod)1810 nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,
1811 struct nvmet_fc_ls_iod *iod)
1812 {
1813 struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1814 &iod->rqstbuf->rq_dis_assoc;
1815 struct fcnvme_ls_disconnect_assoc_acc *acc =
1816 &iod->rspbuf->rsp_dis_assoc;
1817 struct nvmet_fc_tgt_assoc *assoc = NULL;
1818 struct nvmet_fc_ls_iod *oldls = NULL;
1819 unsigned long flags;
1820 int ret = 0;
1821
1822 memset(acc, 0, sizeof(*acc));
1823
1824 ret = nvmefc_vldt_lsreq_discon_assoc(iod->rqstdatalen, rqst);
1825 if (!ret) {
1826 /* match an active association - takes an assoc ref if !NULL */
1827 assoc = nvmet_fc_find_target_assoc(tgtport,
1828 be64_to_cpu(rqst->associd.association_id));
1829 iod->assoc = assoc;
1830 if (!assoc)
1831 ret = VERR_NO_ASSOC;
1832 }
1833
1834 if (ret || !assoc) {
1835 dev_err(tgtport->dev,
1836 "Disconnect LS failed: %s\n",
1837 validation_errors[ret]);
1838 iod->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1839 sizeof(*acc), rqst->w0.ls_cmd,
1840 (ret == VERR_NO_ASSOC) ?
1841 FCNVME_RJT_RC_INV_ASSOC :
1842 FCNVME_RJT_RC_LOGIC,
1843 FCNVME_RJT_EXP_NONE, 0);
1844 return true;
1845 }
1846
1847 /* format a response */
1848
1849 iod->lsrsp->rsplen = sizeof(*acc);
1850
1851 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1852 fcnvme_lsdesc_len(
1853 sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1854 FCNVME_LS_DISCONNECT_ASSOC);
1855
1856 /*
1857 * The rules for LS response says the response cannot
1858 * go back until ABTS's have been sent for all outstanding
1859 * I/O and a Disconnect Association LS has been sent.
1860 * So... save off the Disconnect LS to send the response
1861 * later. If there was a prior LS already saved, replace
1862 * it with the newer one and send a can't perform reject
1863 * on the older one.
1864 */
1865 spin_lock_irqsave(&tgtport->lock, flags);
1866 oldls = assoc->rcv_disconn;
1867 assoc->rcv_disconn = iod;
1868 spin_unlock_irqrestore(&tgtport->lock, flags);
1869
1870 if (oldls) {
1871 dev_info(tgtport->dev,
1872 "{%d:%d} Multiple Disconnect Association LS's "
1873 "received\n",
1874 tgtport->fc_target_port.port_num, assoc->a_id);
1875 /* overwrite good response with bogus failure */
1876 oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf,
1877 sizeof(*iod->rspbuf),
1878 /* ok to use rqst, LS is same */
1879 rqst->w0.ls_cmd,
1880 FCNVME_RJT_RC_UNAB,
1881 FCNVME_RJT_EXP_NONE, 0);
1882 nvmet_fc_xmt_ls_rsp(tgtport, oldls);
1883 }
1884
1885 nvmet_fc_schedule_delete_assoc(assoc);
1886 nvmet_fc_tgt_a_put(assoc);
1887
1888 return false;
1889 }
1890
1891
1892 /* *********************** NVME Ctrl Routines **************************** */
1893
1894
1895 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req);
1896
1897 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
1898
1899 static void
nvmet_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp * lsrsp)1900 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1901 {
1902 struct nvmet_fc_ls_iod *iod = lsrsp->nvme_fc_private;
1903 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1904
1905 fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma,
1906 sizeof(*iod->rspbuf), DMA_TO_DEVICE);
1907 nvmet_fc_free_ls_iod(tgtport, iod);
1908 nvmet_fc_tgtport_put(tgtport);
1909 }
1910
1911 static void
nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_iod * iod)1912 nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
1913 struct nvmet_fc_ls_iod *iod)
1914 {
1915 int ret;
1916
1917 fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma,
1918 sizeof(*iod->rspbuf), DMA_TO_DEVICE);
1919
1920 ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsrsp);
1921 if (ret)
1922 nvmet_fc_xmt_ls_rsp_done(iod->lsrsp);
1923 }
1924
1925 /*
1926 * Actual processing routine for received FC-NVME LS Requests from the LLD
1927 */
1928 static void
nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_iod * iod)1929 nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport,
1930 struct nvmet_fc_ls_iod *iod)
1931 {
1932 struct fcnvme_ls_rqst_w0 *w0 = &iod->rqstbuf->rq_cr_assoc.w0;
1933 bool sendrsp = true;
1934
1935 iod->lsrsp->nvme_fc_private = iod;
1936 iod->lsrsp->rspbuf = iod->rspbuf;
1937 iod->lsrsp->rspdma = iod->rspdma;
1938 iod->lsrsp->done = nvmet_fc_xmt_ls_rsp_done;
1939 /* Be preventative. handlers will later set to valid length */
1940 iod->lsrsp->rsplen = 0;
1941
1942 iod->assoc = NULL;
1943
1944 /*
1945 * handlers:
1946 * parse request input, execute the request, and format the
1947 * LS response
1948 */
1949 switch (w0->ls_cmd) {
1950 case FCNVME_LS_CREATE_ASSOCIATION:
1951 /* Creates Association and initial Admin Queue/Connection */
1952 nvmet_fc_ls_create_association(tgtport, iod);
1953 break;
1954 case FCNVME_LS_CREATE_CONNECTION:
1955 /* Creates an IO Queue/Connection */
1956 nvmet_fc_ls_create_connection(tgtport, iod);
1957 break;
1958 case FCNVME_LS_DISCONNECT_ASSOC:
1959 /* Terminate a Queue/Connection or the Association */
1960 sendrsp = nvmet_fc_ls_disconnect(tgtport, iod);
1961 break;
1962 default:
1963 iod->lsrsp->rsplen = nvme_fc_format_rjt(iod->rspbuf,
1964 sizeof(*iod->rspbuf), w0->ls_cmd,
1965 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1966 }
1967
1968 if (sendrsp)
1969 nvmet_fc_xmt_ls_rsp(tgtport, iod);
1970 }
1971
1972 /*
1973 * Actual processing routine for received FC-NVME LS Requests from the LLD
1974 */
1975 static void
nvmet_fc_handle_ls_rqst_work(struct work_struct * work)1976 nvmet_fc_handle_ls_rqst_work(struct work_struct *work)
1977 {
1978 struct nvmet_fc_ls_iod *iod =
1979 container_of(work, struct nvmet_fc_ls_iod, work);
1980 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1981
1982 nvmet_fc_handle_ls_rqst(tgtport, iod);
1983 }
1984
1985
1986 /**
1987 * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD
1988 * upon the reception of a NVME LS request.
1989 *
1990 * The nvmet-fc layer will copy payload to an internal structure for
1991 * processing. As such, upon completion of the routine, the LLDD may
1992 * immediately free/reuse the LS request buffer passed in the call.
1993 *
1994 * If this routine returns error, the LLDD should abort the exchange.
1995 *
1996 * @target_port: pointer to the (registered) target port the LS was
1997 * received on.
1998 * @hosthandle: pointer to the host specific data, gets stored in iod.
1999 * @lsrsp: pointer to a lsrsp structure to be used to reference
2000 * the exchange corresponding to the LS.
2001 * @lsreqbuf: pointer to the buffer containing the LS Request
2002 * @lsreqbuf_len: length, in bytes, of the received LS request
2003 */
2004 int
nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port * target_port,void * hosthandle,struct nvmefc_ls_rsp * lsrsp,void * lsreqbuf,u32 lsreqbuf_len)2005 nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
2006 void *hosthandle,
2007 struct nvmefc_ls_rsp *lsrsp,
2008 void *lsreqbuf, u32 lsreqbuf_len)
2009 {
2010 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2011 struct nvmet_fc_ls_iod *iod;
2012 struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
2013
2014 if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
2015 dev_info(tgtport->dev,
2016 "RCV %s LS failed: payload too large (%d)\n",
2017 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
2018 nvmefc_ls_names[w0->ls_cmd] : "",
2019 lsreqbuf_len);
2020 return -E2BIG;
2021 }
2022
2023 if (!nvmet_fc_tgtport_get(tgtport)) {
2024 dev_info(tgtport->dev,
2025 "RCV %s LS failed: target deleting\n",
2026 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
2027 nvmefc_ls_names[w0->ls_cmd] : "");
2028 return -ESHUTDOWN;
2029 }
2030
2031 iod = nvmet_fc_alloc_ls_iod(tgtport);
2032 if (!iod) {
2033 dev_info(tgtport->dev,
2034 "RCV %s LS failed: context allocation failed\n",
2035 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
2036 nvmefc_ls_names[w0->ls_cmd] : "");
2037 nvmet_fc_tgtport_put(tgtport);
2038 return -ENOENT;
2039 }
2040
2041 iod->lsrsp = lsrsp;
2042 iod->fcpreq = NULL;
2043 memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len);
2044 iod->rqstdatalen = lsreqbuf_len;
2045 iod->hosthandle = hosthandle;
2046
2047 queue_work(nvmet_wq, &iod->work);
2048
2049 return 0;
2050 }
2051 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req);
2052
2053
2054 /*
2055 * **********************
2056 * Start of FCP handling
2057 * **********************
2058 */
2059
2060 static int
nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod * fod)2061 nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
2062 {
2063 struct scatterlist *sg;
2064 unsigned int nent;
2065
2066 sg = sgl_alloc(fod->req.transfer_len, GFP_KERNEL, &nent);
2067 if (!sg)
2068 goto out;
2069
2070 fod->data_sg = sg;
2071 fod->data_sg_cnt = nent;
2072 fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent,
2073 ((fod->io_dir == NVMET_FCP_WRITE) ?
2074 DMA_FROM_DEVICE : DMA_TO_DEVICE));
2075 /* note: write from initiator perspective */
2076 fod->next_sg = fod->data_sg;
2077
2078 return 0;
2079
2080 out:
2081 return NVME_SC_INTERNAL;
2082 }
2083
2084 static void
nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod * fod)2085 nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
2086 {
2087 if (!fod->data_sg || !fod->data_sg_cnt)
2088 return;
2089
2090 fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt,
2091 ((fod->io_dir == NVMET_FCP_WRITE) ?
2092 DMA_FROM_DEVICE : DMA_TO_DEVICE));
2093 sgl_free(fod->data_sg);
2094 fod->data_sg = NULL;
2095 fod->data_sg_cnt = 0;
2096 }
2097
2098
2099 static bool
queue_90percent_full(struct nvmet_fc_tgt_queue * q,u32 sqhd)2100 queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd)
2101 {
2102 u32 sqtail, used;
2103
2104 /* egad, this is ugly. And sqtail is just a best guess */
2105 sqtail = atomic_read(&q->sqtail) % q->sqsize;
2106
2107 used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd);
2108 return ((used * 10) >= (((u32)(q->sqsize - 1) * 9)));
2109 }
2110
2111 /*
2112 * Prep RSP payload.
2113 * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op
2114 */
2115 static void
nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_fcp_iod * fod)2116 nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
2117 struct nvmet_fc_fcp_iod *fod)
2118 {
2119 struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf;
2120 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2121 struct nvme_completion *cqe = &ersp->cqe;
2122 u32 *cqewd = (u32 *)cqe;
2123 bool send_ersp = false;
2124 u32 rsn, rspcnt, xfr_length;
2125
2126 if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP)
2127 xfr_length = fod->req.transfer_len;
2128 else
2129 xfr_length = fod->offset;
2130
2131 /*
2132 * check to see if we can send a 0's rsp.
2133 * Note: to send a 0's response, the NVME-FC host transport will
2134 * recreate the CQE. The host transport knows: sq id, SQHD (last
2135 * seen in an ersp), and command_id. Thus it will create a
2136 * zero-filled CQE with those known fields filled in. Transport
2137 * must send an ersp for any condition where the cqe won't match
2138 * this.
2139 *
2140 * Here are the FC-NVME mandated cases where we must send an ersp:
2141 * every N responses, where N=ersp_ratio
2142 * force fabric commands to send ersp's (not in FC-NVME but good
2143 * practice)
2144 * normal cmds: any time status is non-zero, or status is zero
2145 * but words 0 or 1 are non-zero.
2146 * the SQ is 90% or more full
2147 * the cmd is a fused command
2148 * transferred data length not equal to cmd iu length
2149 */
2150 rspcnt = atomic_inc_return(&fod->queue->zrspcnt);
2151 if (!(rspcnt % fod->queue->ersp_ratio) ||
2152 nvme_is_fabrics((struct nvme_command *) sqe) ||
2153 xfr_length != fod->req.transfer_len ||
2154 (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] ||
2155 (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) ||
2156 queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head)))
2157 send_ersp = true;
2158
2159 /* re-set the fields */
2160 fod->fcpreq->rspaddr = ersp;
2161 fod->fcpreq->rspdma = fod->rspdma;
2162
2163 if (!send_ersp) {
2164 memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP);
2165 fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP;
2166 } else {
2167 ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32));
2168 rsn = atomic_inc_return(&fod->queue->rsn);
2169 ersp->rsn = cpu_to_be32(rsn);
2170 ersp->xfrd_len = cpu_to_be32(xfr_length);
2171 fod->fcpreq->rsplen = sizeof(*ersp);
2172 }
2173
2174 fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma,
2175 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
2176 }
2177
2178 static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq);
2179
2180 static void
nvmet_fc_abort_op(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_fcp_iod * fod)2181 nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport,
2182 struct nvmet_fc_fcp_iod *fod)
2183 {
2184 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2185
2186 /* data no longer needed */
2187 nvmet_fc_free_tgt_pgs(fod);
2188
2189 /*
2190 * if an ABTS was received or we issued the fcp_abort early
2191 * don't call abort routine again.
2192 */
2193 /* no need to take lock - lock was taken earlier to get here */
2194 if (!fod->aborted)
2195 tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq);
2196
2197 nvmet_fc_free_fcp_iod(fod->queue, fod);
2198 }
2199
2200 static void
nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_fcp_iod * fod)2201 nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
2202 struct nvmet_fc_fcp_iod *fod)
2203 {
2204 int ret;
2205
2206 fod->fcpreq->op = NVMET_FCOP_RSP;
2207 fod->fcpreq->timeout = 0;
2208
2209 nvmet_fc_prep_fcp_rsp(tgtport, fod);
2210
2211 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
2212 if (ret)
2213 nvmet_fc_abort_op(tgtport, fod);
2214 }
2215
2216 static void
nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_fcp_iod * fod,u8 op)2217 nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport,
2218 struct nvmet_fc_fcp_iod *fod, u8 op)
2219 {
2220 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2221 struct scatterlist *sg = fod->next_sg;
2222 unsigned long flags;
2223 u32 remaininglen = fod->req.transfer_len - fod->offset;
2224 u32 tlen = 0;
2225 int ret;
2226
2227 fcpreq->op = op;
2228 fcpreq->offset = fod->offset;
2229 fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC;
2230
2231 /*
2232 * for next sequence:
2233 * break at a sg element boundary
2234 * attempt to keep sequence length capped at
2235 * NVMET_FC_MAX_SEQ_LENGTH but allow sequence to
2236 * be longer if a single sg element is larger
2237 * than that amount. This is done to avoid creating
2238 * a new sg list to use for the tgtport api.
2239 */
2240 fcpreq->sg = sg;
2241 fcpreq->sg_cnt = 0;
2242 while (tlen < remaininglen &&
2243 fcpreq->sg_cnt < tgtport->max_sg_cnt &&
2244 tlen + sg_dma_len(sg) < NVMET_FC_MAX_SEQ_LENGTH) {
2245 fcpreq->sg_cnt++;
2246 tlen += sg_dma_len(sg);
2247 sg = sg_next(sg);
2248 }
2249 if (tlen < remaininglen && fcpreq->sg_cnt == 0) {
2250 fcpreq->sg_cnt++;
2251 tlen += min_t(u32, sg_dma_len(sg), remaininglen);
2252 sg = sg_next(sg);
2253 }
2254 if (tlen < remaininglen)
2255 fod->next_sg = sg;
2256 else
2257 fod->next_sg = NULL;
2258
2259 fcpreq->transfer_length = tlen;
2260 fcpreq->transferred_length = 0;
2261 fcpreq->fcp_error = 0;
2262 fcpreq->rsplen = 0;
2263
2264 /*
2265 * If the last READDATA request: check if LLDD supports
2266 * combined xfr with response.
2267 */
2268 if ((op == NVMET_FCOP_READDATA) &&
2269 ((fod->offset + fcpreq->transfer_length) == fod->req.transfer_len) &&
2270 (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) {
2271 fcpreq->op = NVMET_FCOP_READDATA_RSP;
2272 nvmet_fc_prep_fcp_rsp(tgtport, fod);
2273 }
2274
2275 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
2276 if (ret) {
2277 /*
2278 * should be ok to set w/o lock as its in the thread of
2279 * execution (not an async timer routine) and doesn't
2280 * contend with any clearing action
2281 */
2282 fod->abort = true;
2283
2284 if (op == NVMET_FCOP_WRITEDATA) {
2285 spin_lock_irqsave(&fod->flock, flags);
2286 fod->writedataactive = false;
2287 spin_unlock_irqrestore(&fod->flock, flags);
2288 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2289 } else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ {
2290 fcpreq->fcp_error = ret;
2291 fcpreq->transferred_length = 0;
2292 nvmet_fc_xmt_fcp_op_done(fod->fcpreq);
2293 }
2294 }
2295 }
2296
2297 static inline bool
__nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod * fod,bool abort)2298 __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort)
2299 {
2300 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2301 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2302
2303 /* if in the middle of an io and we need to tear down */
2304 if (abort) {
2305 if (fcpreq->op == NVMET_FCOP_WRITEDATA) {
2306 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2307 return true;
2308 }
2309
2310 nvmet_fc_abort_op(tgtport, fod);
2311 return true;
2312 }
2313
2314 return false;
2315 }
2316
2317 /*
2318 * actual done handler for FCP operations when completed by the lldd
2319 */
2320 static void
nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod * fod)2321 nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod)
2322 {
2323 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2324 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2325 unsigned long flags;
2326 bool abort;
2327
2328 spin_lock_irqsave(&fod->flock, flags);
2329 abort = fod->abort;
2330 fod->writedataactive = false;
2331 spin_unlock_irqrestore(&fod->flock, flags);
2332
2333 switch (fcpreq->op) {
2334
2335 case NVMET_FCOP_WRITEDATA:
2336 if (__nvmet_fc_fod_op_abort(fod, abort))
2337 return;
2338 if (fcpreq->fcp_error ||
2339 fcpreq->transferred_length != fcpreq->transfer_length) {
2340 spin_lock_irqsave(&fod->flock, flags);
2341 fod->abort = true;
2342 spin_unlock_irqrestore(&fod->flock, flags);
2343
2344 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2345 return;
2346 }
2347
2348 fod->offset += fcpreq->transferred_length;
2349 if (fod->offset != fod->req.transfer_len) {
2350 spin_lock_irqsave(&fod->flock, flags);
2351 fod->writedataactive = true;
2352 spin_unlock_irqrestore(&fod->flock, flags);
2353
2354 /* transfer the next chunk */
2355 nvmet_fc_transfer_fcp_data(tgtport, fod,
2356 NVMET_FCOP_WRITEDATA);
2357 return;
2358 }
2359
2360 /* data transfer complete, resume with nvmet layer */
2361 fod->req.execute(&fod->req);
2362 break;
2363
2364 case NVMET_FCOP_READDATA:
2365 case NVMET_FCOP_READDATA_RSP:
2366 if (__nvmet_fc_fod_op_abort(fod, abort))
2367 return;
2368 if (fcpreq->fcp_error ||
2369 fcpreq->transferred_length != fcpreq->transfer_length) {
2370 nvmet_fc_abort_op(tgtport, fod);
2371 return;
2372 }
2373
2374 /* success */
2375
2376 if (fcpreq->op == NVMET_FCOP_READDATA_RSP) {
2377 /* data no longer needed */
2378 nvmet_fc_free_tgt_pgs(fod);
2379 nvmet_fc_free_fcp_iod(fod->queue, fod);
2380 return;
2381 }
2382
2383 fod->offset += fcpreq->transferred_length;
2384 if (fod->offset != fod->req.transfer_len) {
2385 /* transfer the next chunk */
2386 nvmet_fc_transfer_fcp_data(tgtport, fod,
2387 NVMET_FCOP_READDATA);
2388 return;
2389 }
2390
2391 /* data transfer complete, send response */
2392
2393 /* data no longer needed */
2394 nvmet_fc_free_tgt_pgs(fod);
2395
2396 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2397
2398 break;
2399
2400 case NVMET_FCOP_RSP:
2401 if (__nvmet_fc_fod_op_abort(fod, abort))
2402 return;
2403 nvmet_fc_free_fcp_iod(fod->queue, fod);
2404 break;
2405
2406 default:
2407 break;
2408 }
2409 }
2410
2411 static void
nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req * fcpreq)2412 nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq)
2413 {
2414 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2415
2416 nvmet_fc_fod_op_done(fod);
2417 }
2418
2419 /*
2420 * actual completion handler after execution by the nvmet layer
2421 */
2422 static void
__nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_fcp_iod * fod,int status)2423 __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport,
2424 struct nvmet_fc_fcp_iod *fod, int status)
2425 {
2426 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2427 struct nvme_completion *cqe = &fod->rspiubuf.cqe;
2428 unsigned long flags;
2429 bool abort;
2430
2431 spin_lock_irqsave(&fod->flock, flags);
2432 abort = fod->abort;
2433 spin_unlock_irqrestore(&fod->flock, flags);
2434
2435 /* if we have a CQE, snoop the last sq_head value */
2436 if (!status)
2437 fod->queue->sqhd = cqe->sq_head;
2438
2439 if (abort) {
2440 nvmet_fc_abort_op(tgtport, fod);
2441 return;
2442 }
2443
2444 /* if an error handling the cmd post initial parsing */
2445 if (status) {
2446 /* fudge up a failed CQE status for our transport error */
2447 memset(cqe, 0, sizeof(*cqe));
2448 cqe->sq_head = fod->queue->sqhd; /* echo last cqe sqhd */
2449 cqe->sq_id = cpu_to_le16(fod->queue->qid);
2450 cqe->command_id = sqe->command_id;
2451 cqe->status = cpu_to_le16(status);
2452 } else {
2453
2454 /*
2455 * try to push the data even if the SQE status is non-zero.
2456 * There may be a status where data still was intended to
2457 * be moved
2458 */
2459 if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) {
2460 /* push the data over before sending rsp */
2461 nvmet_fc_transfer_fcp_data(tgtport, fod,
2462 NVMET_FCOP_READDATA);
2463 return;
2464 }
2465
2466 /* writes & no data - fall thru */
2467 }
2468
2469 /* data no longer needed */
2470 nvmet_fc_free_tgt_pgs(fod);
2471
2472 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2473 }
2474
2475
2476 static void
nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req * nvme_req)2477 nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req)
2478 {
2479 struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req);
2480 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2481
2482 __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0);
2483 }
2484
2485
2486 /*
2487 * Actual processing routine for received FC-NVME I/O Requests from the LLD
2488 */
2489 static void
nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_fcp_iod * fod)2490 nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
2491 struct nvmet_fc_fcp_iod *fod)
2492 {
2493 struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf;
2494 u32 xfrlen = be32_to_cpu(cmdiu->data_len);
2495 int ret;
2496
2497 /*
2498 * Fused commands are currently not supported in the linux
2499 * implementation.
2500 *
2501 * As such, the implementation of the FC transport does not
2502 * look at the fused commands and order delivery to the upper
2503 * layer until we have both based on csn.
2504 */
2505
2506 fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done;
2507
2508 if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) {
2509 fod->io_dir = NVMET_FCP_WRITE;
2510 if (!nvme_is_write(&cmdiu->sqe))
2511 goto transport_error;
2512 } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) {
2513 fod->io_dir = NVMET_FCP_READ;
2514 if (nvme_is_write(&cmdiu->sqe))
2515 goto transport_error;
2516 } else {
2517 fod->io_dir = NVMET_FCP_NODATA;
2518 if (xfrlen)
2519 goto transport_error;
2520 }
2521
2522 fod->req.cmd = &fod->cmdiubuf.sqe;
2523 fod->req.cqe = &fod->rspiubuf.cqe;
2524 if (!tgtport->pe)
2525 goto transport_error;
2526 fod->req.port = tgtport->pe->port;
2527
2528 /* clear any response payload */
2529 memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf));
2530
2531 fod->data_sg = NULL;
2532 fod->data_sg_cnt = 0;
2533
2534 ret = nvmet_req_init(&fod->req,
2535 &fod->queue->nvme_cq,
2536 &fod->queue->nvme_sq,
2537 &nvmet_fc_tgt_fcp_ops);
2538 if (!ret) {
2539 /* bad SQE content or invalid ctrl state */
2540 /* nvmet layer has already called op done to send rsp. */
2541 return;
2542 }
2543
2544 fod->req.transfer_len = xfrlen;
2545
2546 /* keep a running counter of tail position */
2547 atomic_inc(&fod->queue->sqtail);
2548
2549 if (fod->req.transfer_len) {
2550 ret = nvmet_fc_alloc_tgt_pgs(fod);
2551 if (ret) {
2552 nvmet_req_complete(&fod->req, ret);
2553 return;
2554 }
2555 }
2556 fod->req.sg = fod->data_sg;
2557 fod->req.sg_cnt = fod->data_sg_cnt;
2558 fod->offset = 0;
2559
2560 if (fod->io_dir == NVMET_FCP_WRITE) {
2561 /* pull the data over before invoking nvmet layer */
2562 nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA);
2563 return;
2564 }
2565
2566 /*
2567 * Reads or no data:
2568 *
2569 * can invoke the nvmet_layer now. If read data, cmd completion will
2570 * push the data
2571 */
2572 fod->req.execute(&fod->req);
2573 return;
2574
2575 transport_error:
2576 nvmet_fc_abort_op(tgtport, fod);
2577 }
2578
2579 /**
2580 * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD
2581 * upon the reception of a NVME FCP CMD IU.
2582 *
2583 * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc
2584 * layer for processing.
2585 *
2586 * The nvmet_fc layer allocates a local job structure (struct
2587 * nvmet_fc_fcp_iod) from the queue for the io and copies the
2588 * CMD IU buffer to the job structure. As such, on a successful
2589 * completion (returns 0), the LLDD may immediately free/reuse
2590 * the CMD IU buffer passed in the call.
2591 *
2592 * However, in some circumstances, due to the packetized nature of FC
2593 * and the api of the FC LLDD which may issue a hw command to send the
2594 * response, but the LLDD may not get the hw completion for that command
2595 * and upcall the nvmet_fc layer before a new command may be
2596 * asynchronously received - its possible for a command to be received
2597 * before the LLDD and nvmet_fc have recycled the job structure. It gives
2598 * the appearance of more commands received than fits in the sq.
2599 * To alleviate this scenario, a temporary queue is maintained in the
2600 * transport for pending LLDD requests waiting for a queue job structure.
2601 * In these "overrun" cases, a temporary queue element is allocated
2602 * the LLDD request and CMD iu buffer information remembered, and the
2603 * routine returns a -EOVERFLOW status. Subsequently, when a queue job
2604 * structure is freed, it is immediately reallocated for anything on the
2605 * pending request list. The LLDDs defer_rcv() callback is called,
2606 * informing the LLDD that it may reuse the CMD IU buffer, and the io
2607 * is then started normally with the transport.
2608 *
2609 * The LLDD, when receiving an -EOVERFLOW completion status, is to treat
2610 * the completion as successful but must not reuse the CMD IU buffer
2611 * until the LLDD's defer_rcv() callback has been called for the
2612 * corresponding struct nvmefc_tgt_fcp_req pointer.
2613 *
2614 * If there is any other condition in which an error occurs, the
2615 * transport will return a non-zero status indicating the error.
2616 * In all cases other than -EOVERFLOW, the transport has not accepted the
2617 * request and the LLDD should abort the exchange.
2618 *
2619 * @target_port: pointer to the (registered) target port the FCP CMD IU
2620 * was received on.
2621 * @fcpreq: pointer to a fcpreq request structure to be used to reference
2622 * the exchange corresponding to the FCP Exchange.
2623 * @cmdiubuf: pointer to the buffer containing the FCP CMD IU
2624 * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU
2625 */
2626 int
nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port * target_port,struct nvmefc_tgt_fcp_req * fcpreq,void * cmdiubuf,u32 cmdiubuf_len)2627 nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port,
2628 struct nvmefc_tgt_fcp_req *fcpreq,
2629 void *cmdiubuf, u32 cmdiubuf_len)
2630 {
2631 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2632 struct nvme_fc_cmd_iu *cmdiu = cmdiubuf;
2633 struct nvmet_fc_tgt_queue *queue;
2634 struct nvmet_fc_fcp_iod *fod;
2635 struct nvmet_fc_defer_fcp_req *deferfcp;
2636 unsigned long flags;
2637
2638 /* validate iu, so the connection id can be used to find the queue */
2639 if ((cmdiubuf_len != sizeof(*cmdiu)) ||
2640 (cmdiu->format_id != NVME_CMD_FORMAT_ID) ||
2641 (cmdiu->fc_id != NVME_CMD_FC_ID) ||
2642 (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4)))
2643 return -EIO;
2644
2645 queue = nvmet_fc_find_target_queue(tgtport,
2646 be64_to_cpu(cmdiu->connection_id));
2647 if (!queue)
2648 return -ENOTCONN;
2649
2650 /*
2651 * note: reference taken by find_target_queue
2652 * After successful fod allocation, the fod will inherit the
2653 * ownership of that reference and will remove the reference
2654 * when the fod is freed.
2655 */
2656
2657 spin_lock_irqsave(&queue->qlock, flags);
2658
2659 fod = nvmet_fc_alloc_fcp_iod(queue);
2660 if (fod) {
2661 spin_unlock_irqrestore(&queue->qlock, flags);
2662
2663 fcpreq->nvmet_fc_private = fod;
2664 fod->fcpreq = fcpreq;
2665
2666 memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len);
2667
2668 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq);
2669
2670 return 0;
2671 }
2672
2673 if (!tgtport->ops->defer_rcv) {
2674 spin_unlock_irqrestore(&queue->qlock, flags);
2675 /* release the queue lookup reference */
2676 nvmet_fc_tgt_q_put(queue);
2677 return -ENOENT;
2678 }
2679
2680 deferfcp = list_first_entry_or_null(&queue->avail_defer_list,
2681 struct nvmet_fc_defer_fcp_req, req_list);
2682 if (deferfcp) {
2683 /* Just re-use one that was previously allocated */
2684 list_del(&deferfcp->req_list);
2685 } else {
2686 spin_unlock_irqrestore(&queue->qlock, flags);
2687
2688 /* Now we need to dynamically allocate one */
2689 deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL);
2690 if (!deferfcp) {
2691 /* release the queue lookup reference */
2692 nvmet_fc_tgt_q_put(queue);
2693 return -ENOMEM;
2694 }
2695 spin_lock_irqsave(&queue->qlock, flags);
2696 }
2697
2698 /* For now, use rspaddr / rsplen to save payload information */
2699 fcpreq->rspaddr = cmdiubuf;
2700 fcpreq->rsplen = cmdiubuf_len;
2701 deferfcp->fcp_req = fcpreq;
2702
2703 /* defer processing till a fod becomes available */
2704 list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list);
2705
2706 /* NOTE: the queue lookup reference is still valid */
2707
2708 spin_unlock_irqrestore(&queue->qlock, flags);
2709
2710 return -EOVERFLOW;
2711 }
2712 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req);
2713
2714 /**
2715 * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD
2716 * upon the reception of an ABTS for a FCP command
2717 *
2718 * Notify the transport that an ABTS has been received for a FCP command
2719 * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The
2720 * LLDD believes the command is still being worked on
2721 * (template_ops->fcp_req_release() has not been called).
2722 *
2723 * The transport will wait for any outstanding work (an op to the LLDD,
2724 * which the lldd should complete with error due to the ABTS; or the
2725 * completion from the nvmet layer of the nvme command), then will
2726 * stop processing and call the nvmet_fc_rcv_fcp_req() callback to
2727 * return the i/o context to the LLDD. The LLDD may send the BA_ACC
2728 * to the ABTS either after return from this function (assuming any
2729 * outstanding op work has been terminated) or upon the callback being
2730 * called.
2731 *
2732 * @target_port: pointer to the (registered) target port the FCP CMD IU
2733 * was received on.
2734 * @fcpreq: pointer to the fcpreq request structure that corresponds
2735 * to the exchange that received the ABTS.
2736 */
2737 void
nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port * target_port,struct nvmefc_tgt_fcp_req * fcpreq)2738 nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port,
2739 struct nvmefc_tgt_fcp_req *fcpreq)
2740 {
2741 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2742 struct nvmet_fc_tgt_queue *queue;
2743 unsigned long flags;
2744
2745 if (!fod || fod->fcpreq != fcpreq)
2746 /* job appears to have already completed, ignore abort */
2747 return;
2748
2749 queue = fod->queue;
2750
2751 spin_lock_irqsave(&queue->qlock, flags);
2752 if (fod->active) {
2753 /*
2754 * mark as abort. The abort handler, invoked upon completion
2755 * of any work, will detect the aborted status and do the
2756 * callback.
2757 */
2758 spin_lock(&fod->flock);
2759 fod->abort = true;
2760 fod->aborted = true;
2761 spin_unlock(&fod->flock);
2762 }
2763 spin_unlock_irqrestore(&queue->qlock, flags);
2764 }
2765 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort);
2766
2767
2768 struct nvmet_fc_traddr {
2769 u64 nn;
2770 u64 pn;
2771 };
2772
2773 static int
__nvme_fc_parse_u64(substring_t * sstr,u64 * val)2774 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
2775 {
2776 u64 token64;
2777
2778 if (match_u64(sstr, &token64))
2779 return -EINVAL;
2780 *val = token64;
2781
2782 return 0;
2783 }
2784
2785 /*
2786 * This routine validates and extracts the WWN's from the TRADDR string.
2787 * As kernel parsers need the 0x to determine number base, universally
2788 * build string to parse with 0x prefix before parsing name strings.
2789 */
2790 static int
nvme_fc_parse_traddr(struct nvmet_fc_traddr * traddr,char * buf,size_t blen)2791 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
2792 {
2793 char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
2794 substring_t wwn = { name, &name[sizeof(name)-1] };
2795 int nnoffset, pnoffset;
2796
2797 /* validate if string is one of the 2 allowed formats */
2798 if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
2799 !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
2800 !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
2801 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
2802 nnoffset = NVME_FC_TRADDR_OXNNLEN;
2803 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
2804 NVME_FC_TRADDR_OXNNLEN;
2805 } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
2806 !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
2807 !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
2808 "pn-", NVME_FC_TRADDR_NNLEN))) {
2809 nnoffset = NVME_FC_TRADDR_NNLEN;
2810 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
2811 } else
2812 goto out_einval;
2813
2814 name[0] = '0';
2815 name[1] = 'x';
2816 name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
2817
2818 memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2819 if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
2820 goto out_einval;
2821
2822 memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2823 if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
2824 goto out_einval;
2825
2826 return 0;
2827
2828 out_einval:
2829 pr_warn("%s: bad traddr string\n", __func__);
2830 return -EINVAL;
2831 }
2832
2833 static int
nvmet_fc_add_port(struct nvmet_port * port)2834 nvmet_fc_add_port(struct nvmet_port *port)
2835 {
2836 struct nvmet_fc_tgtport *tgtport;
2837 struct nvmet_fc_port_entry *pe;
2838 struct nvmet_fc_traddr traddr = { 0L, 0L };
2839 unsigned long flags;
2840 int ret;
2841
2842 /* validate the address info */
2843 if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) ||
2844 (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC))
2845 return -EINVAL;
2846
2847 /* map the traddr address info to a target port */
2848
2849 ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr,
2850 sizeof(port->disc_addr.traddr));
2851 if (ret)
2852 return ret;
2853
2854 pe = kzalloc(sizeof(*pe), GFP_KERNEL);
2855 if (!pe)
2856 return -ENOMEM;
2857
2858 ret = -ENXIO;
2859 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2860 list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) {
2861 if ((tgtport->fc_target_port.node_name == traddr.nn) &&
2862 (tgtport->fc_target_port.port_name == traddr.pn)) {
2863 /* a FC port can only be 1 nvmet port id */
2864 if (!tgtport->pe) {
2865 nvmet_fc_portentry_bind(tgtport, pe, port);
2866 ret = 0;
2867 } else
2868 ret = -EALREADY;
2869 break;
2870 }
2871 }
2872 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2873
2874 if (ret)
2875 kfree(pe);
2876
2877 return ret;
2878 }
2879
2880 static void
nvmet_fc_remove_port(struct nvmet_port * port)2881 nvmet_fc_remove_port(struct nvmet_port *port)
2882 {
2883 struct nvmet_fc_port_entry *pe = port->priv;
2884
2885 nvmet_fc_portentry_unbind(pe);
2886
2887 /* terminate any outstanding associations */
2888 __nvmet_fc_free_assocs(pe->tgtport);
2889
2890 kfree(pe);
2891 }
2892
2893 static void
nvmet_fc_discovery_chg(struct nvmet_port * port)2894 nvmet_fc_discovery_chg(struct nvmet_port *port)
2895 {
2896 struct nvmet_fc_port_entry *pe = port->priv;
2897 struct nvmet_fc_tgtport *tgtport = pe->tgtport;
2898
2899 if (tgtport && tgtport->ops->discovery_event)
2900 tgtport->ops->discovery_event(&tgtport->fc_target_port);
2901 }
2902
2903 static ssize_t
nvmet_fc_host_traddr(struct nvmet_ctrl * ctrl,char * traddr,size_t traddr_size)2904 nvmet_fc_host_traddr(struct nvmet_ctrl *ctrl,
2905 char *traddr, size_t traddr_size)
2906 {
2907 struct nvmet_sq *sq = ctrl->sqs[0];
2908 struct nvmet_fc_tgt_queue *queue =
2909 container_of(sq, struct nvmet_fc_tgt_queue, nvme_sq);
2910 struct nvmet_fc_tgtport *tgtport = queue->assoc ? queue->assoc->tgtport : NULL;
2911 struct nvmet_fc_hostport *hostport = queue->assoc ? queue->assoc->hostport : NULL;
2912 u64 wwnn, wwpn;
2913 ssize_t ret = 0;
2914
2915 if (!tgtport || !nvmet_fc_tgtport_get(tgtport))
2916 return -ENODEV;
2917 if (!hostport || !nvmet_fc_hostport_get(hostport)) {
2918 ret = -ENODEV;
2919 goto out_put;
2920 }
2921
2922 if (tgtport->ops->host_traddr) {
2923 ret = tgtport->ops->host_traddr(hostport->hosthandle, &wwnn, &wwpn);
2924 if (ret)
2925 goto out_put_host;
2926 ret = snprintf(traddr, traddr_size, "nn-0x%llx:pn-0x%llx", wwnn, wwpn);
2927 }
2928 out_put_host:
2929 nvmet_fc_hostport_put(hostport);
2930 out_put:
2931 nvmet_fc_tgtport_put(tgtport);
2932 return ret;
2933 }
2934
2935 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = {
2936 .owner = THIS_MODULE,
2937 .type = NVMF_TRTYPE_FC,
2938 .msdbd = 1,
2939 .add_port = nvmet_fc_add_port,
2940 .remove_port = nvmet_fc_remove_port,
2941 .queue_response = nvmet_fc_fcp_nvme_cmd_done,
2942 .delete_ctrl = nvmet_fc_delete_ctrl,
2943 .discovery_chg = nvmet_fc_discovery_chg,
2944 .host_traddr = nvmet_fc_host_traddr,
2945 };
2946
nvmet_fc_init_module(void)2947 static int __init nvmet_fc_init_module(void)
2948 {
2949 return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops);
2950 }
2951
nvmet_fc_exit_module(void)2952 static void __exit nvmet_fc_exit_module(void)
2953 {
2954 /* ensure any shutdown operation, e.g. delete ctrls have finished */
2955 flush_workqueue(nvmet_wq);
2956
2957 /* sanity check - all lports should be removed */
2958 if (!list_empty(&nvmet_fc_target_list))
2959 pr_warn("%s: targetport list not empty\n", __func__);
2960
2961 nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops);
2962
2963 ida_destroy(&nvmet_fc_tgtport_cnt);
2964 }
2965
2966 module_init(nvmet_fc_init_module);
2967 module_exit(nvmet_fc_exit_module);
2968
2969 MODULE_DESCRIPTION("NVMe target FC transport driver");
2970 MODULE_LICENSE("GPL v2");
2971