1 // SPDX-License-Identifier: GPL-2.0+
2 /*******************************************************************************
3 * Vhost kernel TCM fabric driver for virtio SCSI initiators
4 *
5 * (C) Copyright 2010-2013 Datera, Inc.
6 * (C) Copyright 2010-2012 IBM Corp.
7 *
8 * Authors: Nicholas A. Bellinger <nab@daterainc.com>
9 * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
10 ****************************************************************************/
11
12 #include <linux/module.h>
13 #include <linux/moduleparam.h>
14 #include <generated/utsrelease.h>
15 #include <linux/utsname.h>
16 #include <linux/init.h>
17 #include <linux/slab.h>
18 #include <linux/kthread.h>
19 #include <linux/types.h>
20 #include <linux/string.h>
21 #include <linux/configfs.h>
22 #include <linux/ctype.h>
23 #include <linux/compat.h>
24 #include <linux/eventfd.h>
25 #include <linux/fs.h>
26 #include <linux/vmalloc.h>
27 #include <linux/miscdevice.h>
28 #include <linux/blk_types.h>
29 #include <linux/bio.h>
30 #include <linux/unaligned.h>
31 #include <scsi/scsi_common.h>
32 #include <scsi/scsi_proto.h>
33 #include <target/target_core_base.h>
34 #include <target/target_core_fabric.h>
35 #include <linux/vhost.h>
36 #include <linux/virtio_scsi.h>
37 #include <linux/llist.h>
38 #include <linux/bitmap.h>
39
40 #include "vhost.h"
41
42 #define VHOST_SCSI_VERSION "v0.1"
43 #define VHOST_SCSI_NAMELEN 256
44 #define VHOST_SCSI_MAX_CDB_SIZE 32
45 #define VHOST_SCSI_PREALLOC_SGLS 2048
46 #define VHOST_SCSI_PREALLOC_UPAGES 2048
47 #define VHOST_SCSI_PREALLOC_PROT_SGLS 2048
48 /*
49 * For the legacy descriptor case we allocate an iov per byte in the
50 * virtio_scsi_cmd_resp struct.
51 */
52 #define VHOST_SCSI_MAX_RESP_IOVS sizeof(struct virtio_scsi_cmd_resp)
53
54 static unsigned int vhost_scsi_inline_sg_cnt = VHOST_SCSI_PREALLOC_SGLS;
55
56 #ifdef CONFIG_ARCH_NO_SG_CHAIN
vhost_scsi_set_inline_sg_cnt(const char * buf,const struct kernel_param * kp)57 static int vhost_scsi_set_inline_sg_cnt(const char *buf,
58 const struct kernel_param *kp)
59 {
60 pr_err("Setting inline_sg_cnt is not supported.\n");
61 return -EOPNOTSUPP;
62 }
63 #else
vhost_scsi_set_inline_sg_cnt(const char * buf,const struct kernel_param * kp)64 static int vhost_scsi_set_inline_sg_cnt(const char *buf,
65 const struct kernel_param *kp)
66 {
67 unsigned int cnt;
68 int ret;
69
70 ret = kstrtouint(buf, 10, &cnt);
71 if (ret)
72 return ret;
73
74 if (ret > VHOST_SCSI_PREALLOC_SGLS) {
75 pr_err("Max inline_sg_cnt is %u\n", VHOST_SCSI_PREALLOC_SGLS);
76 return -EINVAL;
77 }
78
79 vhost_scsi_inline_sg_cnt = cnt;
80 return 0;
81 }
82 #endif
83
vhost_scsi_get_inline_sg_cnt(char * buf,const struct kernel_param * kp)84 static int vhost_scsi_get_inline_sg_cnt(char *buf,
85 const struct kernel_param *kp)
86 {
87 return sprintf(buf, "%u\n", vhost_scsi_inline_sg_cnt);
88 }
89
90 static const struct kernel_param_ops vhost_scsi_inline_sg_cnt_op = {
91 .get = vhost_scsi_get_inline_sg_cnt,
92 .set = vhost_scsi_set_inline_sg_cnt,
93 };
94
95 module_param_cb(inline_sg_cnt, &vhost_scsi_inline_sg_cnt_op, NULL, 0644);
96 MODULE_PARM_DESC(inline_sg_cnt, "Set the number of scatterlist entries to pre-allocate. The default is 2048.");
97
98 /* Max number of requests before requeueing the job.
99 * Using this limit prevents one virtqueue from starving others with
100 * request.
101 */
102 #define VHOST_SCSI_WEIGHT 256
103
104 struct vhost_scsi_inflight {
105 /* Wait for the flush operation to finish */
106 struct completion comp;
107 /* Refcount for the inflight reqs */
108 struct kref kref;
109 };
110
111 struct vhost_scsi_cmd {
112 /* Descriptor from vhost_get_vq_desc() for virt_queue segment */
113 int tvc_vq_desc;
114 /* The number of scatterlists associated with this cmd */
115 u32 tvc_sgl_count;
116 u32 tvc_prot_sgl_count;
117 u32 copied_iov:1;
118 const void *read_iov;
119 struct iov_iter *read_iter;
120 struct scatterlist *sgl;
121 struct sg_table table;
122 struct scatterlist *prot_sgl;
123 struct sg_table prot_table;
124 /* Fast path response header iovec used when only one vec is needed */
125 struct iovec tvc_resp_iov;
126 /* Number of iovs for response */
127 unsigned int tvc_resp_iovs_cnt;
128 /* Pointer to response header iovecs if more than one is needed */
129 struct iovec *tvc_resp_iovs;
130 /* Pointer to vhost_virtqueue for the cmd */
131 struct vhost_virtqueue *tvc_vq;
132 /* The TCM I/O descriptor that is accessed via container_of() */
133 struct se_cmd tvc_se_cmd;
134 /* Sense buffer that will be mapped into outgoing status */
135 unsigned char tvc_sense_buf[TRANSPORT_SENSE_BUFFER];
136 /* Completed commands list, serviced from vhost worker thread */
137 struct llist_node tvc_completion_list;
138 /* Used to track inflight cmd */
139 struct vhost_scsi_inflight *inflight;
140 };
141
142 struct vhost_scsi_nexus {
143 /* Pointer to TCM session for I_T Nexus */
144 struct se_session *tvn_se_sess;
145 };
146
147 struct vhost_scsi_tpg {
148 /* Vhost port target portal group tag for TCM */
149 u16 tport_tpgt;
150 /* Used to track number of TPG Port/Lun Links wrt to explict I_T Nexus shutdown */
151 int tv_tpg_port_count;
152 /* Used for vhost_scsi device reference to tpg_nexus, protected by tv_tpg_mutex */
153 int tv_tpg_vhost_count;
154 /* Used for enabling T10-PI with legacy devices */
155 int tv_fabric_prot_type;
156 /* list for vhost_scsi_list */
157 struct list_head tv_tpg_list;
158 /* Used to protect access for tpg_nexus */
159 struct mutex tv_tpg_mutex;
160 /* Pointer to the TCM VHost I_T Nexus for this TPG endpoint */
161 struct vhost_scsi_nexus *tpg_nexus;
162 /* Pointer back to vhost_scsi_tport */
163 struct vhost_scsi_tport *tport;
164 /* Returned by vhost_scsi_make_tpg() */
165 struct se_portal_group se_tpg;
166 /* Pointer back to vhost_scsi, protected by tv_tpg_mutex */
167 struct vhost_scsi *vhost_scsi;
168 };
169
170 struct vhost_scsi_tport {
171 /* SCSI protocol the tport is providing */
172 u8 tport_proto_id;
173 /* Binary World Wide unique Port Name for Vhost Target port */
174 u64 tport_wwpn;
175 /* ASCII formatted WWPN for Vhost Target port */
176 char tport_name[VHOST_SCSI_NAMELEN];
177 /* Returned by vhost_scsi_make_tport() */
178 struct se_wwn tport_wwn;
179 };
180
181 struct vhost_scsi_evt {
182 /* event to be sent to guest */
183 struct virtio_scsi_event event;
184 /* event list, serviced from vhost worker thread */
185 struct llist_node list;
186 };
187
188 enum {
189 VHOST_SCSI_VQ_CTL = 0,
190 VHOST_SCSI_VQ_EVT = 1,
191 VHOST_SCSI_VQ_IO = 2,
192 };
193
194 /* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */
195 enum {
196 VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG) |
197 (1ULL << VIRTIO_SCSI_F_T10_PI)
198 };
199
200 #define VHOST_SCSI_MAX_TARGET 256
201 #define VHOST_SCSI_MAX_IO_VQ 1024
202 #define VHOST_SCSI_MAX_EVENT 128
203
204 static unsigned vhost_scsi_max_io_vqs = 128;
205 module_param_named(max_io_vqs, vhost_scsi_max_io_vqs, uint, 0644);
206 MODULE_PARM_DESC(max_io_vqs, "Set the max number of IO virtqueues a vhost scsi device can support. The default is 128. The max is 1024.");
207
208 struct vhost_scsi_virtqueue {
209 struct vhost_virtqueue vq;
210 struct vhost_scsi *vs;
211 /*
212 * Reference counting for inflight reqs, used for flush operation. At
213 * each time, one reference tracks new commands submitted, while we
214 * wait for another one to reach 0.
215 */
216 struct vhost_scsi_inflight inflights[2];
217 /*
218 * Indicate current inflight in use, protected by vq->mutex.
219 * Writers must also take dev mutex and flush under it.
220 */
221 int inflight_idx;
222 struct vhost_scsi_cmd *scsi_cmds;
223 struct sbitmap scsi_tags;
224 int max_cmds;
225 struct page **upages;
226
227 struct vhost_work completion_work;
228 struct llist_head completion_list;
229 };
230
231 struct vhost_scsi {
232 /* Protected by vhost_scsi->dev.mutex */
233 struct vhost_scsi_tpg **vs_tpg;
234 char vs_vhost_wwpn[TRANSPORT_IQN_LEN];
235
236 struct vhost_dev dev;
237 struct vhost_scsi_virtqueue *vqs;
238 struct vhost_scsi_inflight **old_inflight;
239
240 struct vhost_work vs_event_work; /* evt injection work item */
241 struct llist_head vs_event_list; /* evt injection queue */
242
243 bool vs_events_missed; /* any missed events, protected by vq->mutex */
244 int vs_events_nr; /* num of pending events, protected by vq->mutex */
245
246 unsigned int inline_sg_cnt;
247 };
248
249 struct vhost_scsi_tmf {
250 struct vhost_work vwork;
251 struct work_struct flush_work;
252 struct vhost_scsi *vhost;
253 struct vhost_scsi_virtqueue *svq;
254
255 struct se_cmd se_cmd;
256 u8 scsi_resp;
257 struct vhost_scsi_inflight *inflight;
258 struct iovec resp_iov;
259 int in_iovs;
260 int vq_desc;
261 };
262
263 /*
264 * Context for processing request and control queue operations.
265 */
266 struct vhost_scsi_ctx {
267 int head;
268 unsigned int out, in;
269 size_t req_size, rsp_size;
270 size_t out_size, in_size;
271 u8 *target, *lunp;
272 void *req;
273 struct iov_iter out_iter;
274 };
275
276 /*
277 * Global mutex to protect vhost_scsi TPG list for vhost IOCTLs and LIO
278 * configfs management operations.
279 */
280 static DEFINE_MUTEX(vhost_scsi_mutex);
281 static LIST_HEAD(vhost_scsi_list);
282
vhost_scsi_done_inflight(struct kref * kref)283 static void vhost_scsi_done_inflight(struct kref *kref)
284 {
285 struct vhost_scsi_inflight *inflight;
286
287 inflight = container_of(kref, struct vhost_scsi_inflight, kref);
288 complete(&inflight->comp);
289 }
290
vhost_scsi_init_inflight(struct vhost_scsi * vs,struct vhost_scsi_inflight * old_inflight[])291 static void vhost_scsi_init_inflight(struct vhost_scsi *vs,
292 struct vhost_scsi_inflight *old_inflight[])
293 {
294 struct vhost_scsi_inflight *new_inflight;
295 struct vhost_virtqueue *vq;
296 int idx, i;
297
298 for (i = 0; i < vs->dev.nvqs; i++) {
299 vq = &vs->vqs[i].vq;
300
301 mutex_lock(&vq->mutex);
302
303 /* store old infight */
304 idx = vs->vqs[i].inflight_idx;
305 if (old_inflight)
306 old_inflight[i] = &vs->vqs[i].inflights[idx];
307
308 /* setup new infight */
309 vs->vqs[i].inflight_idx = idx ^ 1;
310 new_inflight = &vs->vqs[i].inflights[idx ^ 1];
311 kref_init(&new_inflight->kref);
312 init_completion(&new_inflight->comp);
313
314 mutex_unlock(&vq->mutex);
315 }
316 }
317
318 static struct vhost_scsi_inflight *
vhost_scsi_get_inflight(struct vhost_virtqueue * vq)319 vhost_scsi_get_inflight(struct vhost_virtqueue *vq)
320 {
321 struct vhost_scsi_inflight *inflight;
322 struct vhost_scsi_virtqueue *svq;
323
324 svq = container_of(vq, struct vhost_scsi_virtqueue, vq);
325 inflight = &svq->inflights[svq->inflight_idx];
326 kref_get(&inflight->kref);
327
328 return inflight;
329 }
330
vhost_scsi_put_inflight(struct vhost_scsi_inflight * inflight)331 static void vhost_scsi_put_inflight(struct vhost_scsi_inflight *inflight)
332 {
333 kref_put(&inflight->kref, vhost_scsi_done_inflight);
334 }
335
vhost_scsi_check_true(struct se_portal_group * se_tpg)336 static int vhost_scsi_check_true(struct se_portal_group *se_tpg)
337 {
338 return 1;
339 }
340
vhost_scsi_get_fabric_wwn(struct se_portal_group * se_tpg)341 static char *vhost_scsi_get_fabric_wwn(struct se_portal_group *se_tpg)
342 {
343 struct vhost_scsi_tpg *tpg = container_of(se_tpg,
344 struct vhost_scsi_tpg, se_tpg);
345 struct vhost_scsi_tport *tport = tpg->tport;
346
347 return &tport->tport_name[0];
348 }
349
vhost_scsi_get_tpgt(struct se_portal_group * se_tpg)350 static u16 vhost_scsi_get_tpgt(struct se_portal_group *se_tpg)
351 {
352 struct vhost_scsi_tpg *tpg = container_of(se_tpg,
353 struct vhost_scsi_tpg, se_tpg);
354 return tpg->tport_tpgt;
355 }
356
vhost_scsi_check_prot_fabric_only(struct se_portal_group * se_tpg)357 static int vhost_scsi_check_prot_fabric_only(struct se_portal_group *se_tpg)
358 {
359 struct vhost_scsi_tpg *tpg = container_of(se_tpg,
360 struct vhost_scsi_tpg, se_tpg);
361
362 return tpg->tv_fabric_prot_type;
363 }
364
vhost_scsi_release_cmd_res(struct se_cmd * se_cmd)365 static void vhost_scsi_release_cmd_res(struct se_cmd *se_cmd)
366 {
367 struct vhost_scsi_cmd *tv_cmd = container_of(se_cmd,
368 struct vhost_scsi_cmd, tvc_se_cmd);
369 struct vhost_scsi_virtqueue *svq = container_of(tv_cmd->tvc_vq,
370 struct vhost_scsi_virtqueue, vq);
371 struct vhost_scsi *vs = svq->vs;
372 struct vhost_scsi_inflight *inflight = tv_cmd->inflight;
373 struct scatterlist *sg;
374 struct page *page;
375 int i;
376
377 if (tv_cmd->tvc_sgl_count) {
378 for_each_sgtable_sg(&tv_cmd->table, sg, i) {
379 page = sg_page(sg);
380 if (!page)
381 continue;
382
383 if (tv_cmd->copied_iov)
384 __free_page(page);
385 else
386 put_page(page);
387 }
388 kfree(tv_cmd->read_iter);
389 kfree(tv_cmd->read_iov);
390 sg_free_table_chained(&tv_cmd->table, vs->inline_sg_cnt);
391 }
392 if (tv_cmd->tvc_prot_sgl_count) {
393 for_each_sgtable_sg(&tv_cmd->prot_table, sg, i) {
394 page = sg_page(sg);
395 if (page)
396 put_page(page);
397 }
398 sg_free_table_chained(&tv_cmd->prot_table, vs->inline_sg_cnt);
399 }
400
401 if (tv_cmd->tvc_resp_iovs != &tv_cmd->tvc_resp_iov)
402 kfree(tv_cmd->tvc_resp_iovs);
403 sbitmap_clear_bit(&svq->scsi_tags, se_cmd->map_tag);
404 vhost_scsi_put_inflight(inflight);
405 }
406
vhost_scsi_release_tmf_res(struct vhost_scsi_tmf * tmf)407 static void vhost_scsi_release_tmf_res(struct vhost_scsi_tmf *tmf)
408 {
409 struct vhost_scsi_inflight *inflight = tmf->inflight;
410
411 kfree(tmf);
412 vhost_scsi_put_inflight(inflight);
413 }
414
vhost_scsi_drop_cmds(struct vhost_scsi_virtqueue * svq)415 static void vhost_scsi_drop_cmds(struct vhost_scsi_virtqueue *svq)
416 {
417 struct vhost_scsi_cmd *cmd, *t;
418 struct llist_node *llnode;
419
420 llnode = llist_del_all(&svq->completion_list);
421 llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list)
422 vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd);
423 }
424
vhost_scsi_release_cmd(struct se_cmd * se_cmd)425 static void vhost_scsi_release_cmd(struct se_cmd *se_cmd)
426 {
427 if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) {
428 struct vhost_scsi_tmf *tmf = container_of(se_cmd,
429 struct vhost_scsi_tmf, se_cmd);
430
431 schedule_work(&tmf->flush_work);
432 } else {
433 struct vhost_scsi_cmd *cmd = container_of(se_cmd,
434 struct vhost_scsi_cmd, tvc_se_cmd);
435 struct vhost_scsi_virtqueue *svq = container_of(cmd->tvc_vq,
436 struct vhost_scsi_virtqueue, vq);
437
438 llist_add(&cmd->tvc_completion_list, &svq->completion_list);
439 if (!vhost_vq_work_queue(&svq->vq, &svq->completion_work))
440 vhost_scsi_drop_cmds(svq);
441 }
442 }
443
vhost_scsi_write_pending(struct se_cmd * se_cmd)444 static int vhost_scsi_write_pending(struct se_cmd *se_cmd)
445 {
446 /* Go ahead and process the write immediately */
447 target_execute_cmd(se_cmd);
448 return 0;
449 }
450
vhost_scsi_queue_data_in(struct se_cmd * se_cmd)451 static int vhost_scsi_queue_data_in(struct se_cmd *se_cmd)
452 {
453 transport_generic_free_cmd(se_cmd, 0);
454 return 0;
455 }
456
vhost_scsi_queue_status(struct se_cmd * se_cmd)457 static int vhost_scsi_queue_status(struct se_cmd *se_cmd)
458 {
459 transport_generic_free_cmd(se_cmd, 0);
460 return 0;
461 }
462
vhost_scsi_queue_tm_rsp(struct se_cmd * se_cmd)463 static void vhost_scsi_queue_tm_rsp(struct se_cmd *se_cmd)
464 {
465 struct vhost_scsi_tmf *tmf = container_of(se_cmd, struct vhost_scsi_tmf,
466 se_cmd);
467
468 tmf->scsi_resp = se_cmd->se_tmr_req->response;
469 transport_generic_free_cmd(&tmf->se_cmd, 0);
470 }
471
vhost_scsi_aborted_task(struct se_cmd * se_cmd)472 static void vhost_scsi_aborted_task(struct se_cmd *se_cmd)
473 {
474 return;
475 }
476
vhost_scsi_free_evt(struct vhost_scsi * vs,struct vhost_scsi_evt * evt)477 static void vhost_scsi_free_evt(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)
478 {
479 vs->vs_events_nr--;
480 kfree(evt);
481 }
482
483 static struct vhost_scsi_evt *
vhost_scsi_allocate_evt(struct vhost_scsi * vs,u32 event,u32 reason)484 vhost_scsi_allocate_evt(struct vhost_scsi *vs,
485 u32 event, u32 reason)
486 {
487 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
488 struct vhost_scsi_evt *evt;
489
490 if (vs->vs_events_nr > VHOST_SCSI_MAX_EVENT) {
491 vs->vs_events_missed = true;
492 return NULL;
493 }
494
495 evt = kzalloc(sizeof(*evt), GFP_KERNEL);
496 if (!evt) {
497 vq_err(vq, "Failed to allocate vhost_scsi_evt\n");
498 vs->vs_events_missed = true;
499 return NULL;
500 }
501
502 evt->event.event = cpu_to_vhost32(vq, event);
503 evt->event.reason = cpu_to_vhost32(vq, reason);
504 vs->vs_events_nr++;
505
506 return evt;
507 }
508
vhost_scsi_check_stop_free(struct se_cmd * se_cmd)509 static int vhost_scsi_check_stop_free(struct se_cmd *se_cmd)
510 {
511 return target_put_sess_cmd(se_cmd);
512 }
513
514 static void
vhost_scsi_do_evt_work(struct vhost_scsi * vs,struct vhost_scsi_evt * evt)515 vhost_scsi_do_evt_work(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)
516 {
517 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
518 struct virtio_scsi_event *event = &evt->event;
519 struct virtio_scsi_event __user *eventp;
520 unsigned out, in;
521 int head, ret;
522
523 if (!vhost_vq_get_backend(vq)) {
524 vs->vs_events_missed = true;
525 return;
526 }
527
528 again:
529 vhost_disable_notify(&vs->dev, vq);
530 head = vhost_get_vq_desc(vq, vq->iov,
531 ARRAY_SIZE(vq->iov), &out, &in,
532 NULL, NULL);
533 if (head < 0) {
534 vs->vs_events_missed = true;
535 return;
536 }
537 if (head == vq->num) {
538 if (vhost_enable_notify(&vs->dev, vq))
539 goto again;
540 vs->vs_events_missed = true;
541 return;
542 }
543
544 if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {
545 vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",
546 vq->iov[out].iov_len);
547 vs->vs_events_missed = true;
548 return;
549 }
550
551 if (vs->vs_events_missed) {
552 event->event |= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED);
553 vs->vs_events_missed = false;
554 }
555
556 eventp = vq->iov[out].iov_base;
557 ret = __copy_to_user(eventp, event, sizeof(*event));
558 if (!ret)
559 vhost_add_used_and_signal(&vs->dev, vq, head, 0);
560 else
561 vq_err(vq, "Faulted on vhost_scsi_send_event\n");
562 }
563
vhost_scsi_complete_events(struct vhost_scsi * vs,bool drop)564 static void vhost_scsi_complete_events(struct vhost_scsi *vs, bool drop)
565 {
566 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
567 struct vhost_scsi_evt *evt, *t;
568 struct llist_node *llnode;
569
570 mutex_lock(&vq->mutex);
571 llnode = llist_del_all(&vs->vs_event_list);
572 llist_for_each_entry_safe(evt, t, llnode, list) {
573 if (!drop)
574 vhost_scsi_do_evt_work(vs, evt);
575 vhost_scsi_free_evt(vs, evt);
576 }
577 mutex_unlock(&vq->mutex);
578 }
579
vhost_scsi_evt_work(struct vhost_work * work)580 static void vhost_scsi_evt_work(struct vhost_work *work)
581 {
582 struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
583 vs_event_work);
584 vhost_scsi_complete_events(vs, false);
585 }
586
vhost_scsi_copy_sgl_to_iov(struct vhost_scsi_cmd * cmd)587 static int vhost_scsi_copy_sgl_to_iov(struct vhost_scsi_cmd *cmd)
588 {
589 struct iov_iter *iter = cmd->read_iter;
590 struct scatterlist *sg;
591 struct page *page;
592 size_t len;
593 int i;
594
595 for_each_sgtable_sg(&cmd->table, sg, i) {
596 page = sg_page(sg);
597 if (!page)
598 continue;
599
600 len = sg->length;
601
602 if (copy_page_to_iter(page, 0, len, iter) != len) {
603 pr_err("Could not copy data while handling misaligned cmd. Error %zu\n",
604 len);
605 return -1;
606 }
607 }
608
609 return 0;
610 }
611
612 /* Fill in status and signal that we are done processing this command
613 *
614 * This is scheduled in the vhost work queue so we are called with the owner
615 * process mm and can access the vring.
616 */
vhost_scsi_complete_cmd_work(struct vhost_work * work)617 static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
618 {
619 struct vhost_scsi_virtqueue *svq = container_of(work,
620 struct vhost_scsi_virtqueue, completion_work);
621 struct virtio_scsi_cmd_resp v_rsp;
622 struct vhost_scsi_cmd *cmd, *t;
623 struct llist_node *llnode;
624 struct se_cmd *se_cmd;
625 struct iov_iter iov_iter;
626 bool signal = false;
627 int ret;
628
629 llnode = llist_del_all(&svq->completion_list);
630
631 mutex_lock(&svq->vq.mutex);
632
633 llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) {
634 se_cmd = &cmd->tvc_se_cmd;
635
636 pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__,
637 cmd, se_cmd->residual_count, se_cmd->scsi_status);
638 memset(&v_rsp, 0, sizeof(v_rsp));
639
640 if (cmd->read_iter && vhost_scsi_copy_sgl_to_iov(cmd)) {
641 v_rsp.response = VIRTIO_SCSI_S_BAD_TARGET;
642 } else {
643 v_rsp.resid = cpu_to_vhost32(cmd->tvc_vq,
644 se_cmd->residual_count);
645 /* TODO is status_qualifier field needed? */
646 v_rsp.status = se_cmd->scsi_status;
647 v_rsp.sense_len = cpu_to_vhost32(cmd->tvc_vq,
648 se_cmd->scsi_sense_length);
649 memcpy(v_rsp.sense, cmd->tvc_sense_buf,
650 se_cmd->scsi_sense_length);
651 }
652
653 iov_iter_init(&iov_iter, ITER_DEST, cmd->tvc_resp_iovs,
654 cmd->tvc_resp_iovs_cnt, sizeof(v_rsp));
655 ret = copy_to_iter(&v_rsp, sizeof(v_rsp), &iov_iter);
656 if (likely(ret == sizeof(v_rsp))) {
657 signal = true;
658
659 vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);
660 } else
661 pr_err("Faulted on virtio_scsi_cmd_resp\n");
662
663 vhost_scsi_release_cmd_res(se_cmd);
664 }
665
666 mutex_unlock(&svq->vq.mutex);
667
668 if (signal)
669 vhost_signal(&svq->vs->dev, &svq->vq);
670 }
671
672 static struct vhost_scsi_cmd *
vhost_scsi_get_cmd(struct vhost_virtqueue * vq,u64 scsi_tag)673 vhost_scsi_get_cmd(struct vhost_virtqueue *vq, u64 scsi_tag)
674 {
675 struct vhost_scsi_virtqueue *svq = container_of(vq,
676 struct vhost_scsi_virtqueue, vq);
677 struct vhost_scsi_cmd *cmd;
678 struct scatterlist *sgl, *prot_sgl;
679 int tag;
680
681 tag = sbitmap_get(&svq->scsi_tags);
682 if (tag < 0) {
683 pr_warn_once("Guest sent too many cmds. Returning TASK_SET_FULL.\n");
684 return ERR_PTR(-ENOMEM);
685 }
686
687 cmd = &svq->scsi_cmds[tag];
688 sgl = cmd->sgl;
689 prot_sgl = cmd->prot_sgl;
690 memset(cmd, 0, sizeof(*cmd));
691 cmd->sgl = sgl;
692 cmd->prot_sgl = prot_sgl;
693 cmd->tvc_se_cmd.map_tag = tag;
694 cmd->inflight = vhost_scsi_get_inflight(vq);
695
696 return cmd;
697 }
698
vhost_scsi_revert_map_iov_to_sgl(struct iov_iter * iter,struct scatterlist * curr,struct scatterlist * end)699 static void vhost_scsi_revert_map_iov_to_sgl(struct iov_iter *iter,
700 struct scatterlist *curr,
701 struct scatterlist *end)
702 {
703 size_t revert_bytes = 0;
704 struct page *page;
705
706 while (curr != end) {
707 page = sg_page(curr);
708
709 if (page) {
710 put_page(page);
711 revert_bytes += curr->length;
712 }
713 /* Clear so we can re-use it for the copy path */
714 sg_set_page(curr, NULL, 0, 0);
715 curr = sg_next(curr);
716 }
717 iov_iter_revert(iter, revert_bytes);
718 }
719
720 /*
721 * Map a user memory range into a scatterlist
722 *
723 * Returns the number of scatterlist entries used or -errno on error.
724 */
725 static int
vhost_scsi_map_to_sgl(struct vhost_scsi_cmd * cmd,struct iov_iter * iter,struct sg_table * sg_table,struct scatterlist ** sgl,bool is_prot)726 vhost_scsi_map_to_sgl(struct vhost_scsi_cmd *cmd,
727 struct iov_iter *iter,
728 struct sg_table *sg_table,
729 struct scatterlist **sgl,
730 bool is_prot)
731 {
732 struct vhost_scsi_virtqueue *svq = container_of(cmd->tvc_vq,
733 struct vhost_scsi_virtqueue, vq);
734 struct page **pages = svq->upages;
735 struct scatterlist *sg = *sgl;
736 ssize_t bytes;
737 size_t offset;
738 unsigned int n, npages = 0;
739
740 bytes = iov_iter_get_pages2(iter, pages, LONG_MAX,
741 VHOST_SCSI_PREALLOC_UPAGES, &offset);
742 /* No pages were pinned */
743 if (bytes <= 0)
744 return bytes < 0 ? bytes : -EFAULT;
745
746 while (bytes) {
747 n = min_t(unsigned int, PAGE_SIZE - offset, bytes);
748 /*
749 * The block layer requires bios/requests to be a multiple of
750 * 512 bytes, but Windows can send us vecs that are misaligned.
751 * This can result in bios and later requests with misaligned
752 * sizes if we have to break up a cmd/scatterlist into multiple
753 * bios.
754 *
755 * We currently only break up a command into multiple bios if
756 * we hit the vec/seg limit, so check if our sgl_count is
757 * greater than the max and if a vec in the cmd has a
758 * misaligned offset/size.
759 */
760 if (!is_prot &&
761 (offset & (SECTOR_SIZE - 1) || n & (SECTOR_SIZE - 1)) &&
762 cmd->tvc_sgl_count > BIO_MAX_VECS) {
763 WARN_ONCE(true,
764 "vhost-scsi detected misaligned IO. Performance may be degraded.");
765 goto revert_iter_get_pages;
766 }
767
768 sg_set_page(sg, pages[npages++], n, offset);
769 sg = sg_next(sg);
770 bytes -= n;
771 offset = 0;
772 }
773
774 *sgl = sg;
775 return npages;
776
777 revert_iter_get_pages:
778 vhost_scsi_revert_map_iov_to_sgl(iter, *sgl, sg);
779
780 iov_iter_revert(iter, bytes);
781 while (bytes) {
782 n = min_t(unsigned int, PAGE_SIZE, bytes);
783
784 put_page(pages[npages++]);
785 bytes -= n;
786 }
787
788 return -EINVAL;
789 }
790
791 static int
vhost_scsi_calc_sgls(struct iov_iter * iter,size_t bytes,int max_sgls)792 vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls)
793 {
794 int sgl_count = 0;
795
796 if (!iter || !iter_iov(iter)) {
797 pr_err("%s: iter->iov is NULL, but expected bytes: %zu"
798 " present\n", __func__, bytes);
799 return -EINVAL;
800 }
801
802 sgl_count = iov_iter_npages(iter, 0xffff);
803 if (sgl_count > max_sgls) {
804 pr_err("%s: requested sgl_count: %d exceeds pre-allocated"
805 " max_sgls: %d\n", __func__, sgl_count, max_sgls);
806 return -EINVAL;
807 }
808 return sgl_count;
809 }
810
811 static int
vhost_scsi_copy_iov_to_sgl(struct vhost_scsi_cmd * cmd,struct iov_iter * iter,struct sg_table * sg_table,int sg_count,int data_dir)812 vhost_scsi_copy_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter,
813 struct sg_table *sg_table, int sg_count,
814 int data_dir)
815 {
816 size_t len = iov_iter_count(iter);
817 unsigned int nbytes = 0;
818 struct scatterlist *sg;
819 struct page *page;
820 int i, ret;
821
822 if (data_dir == DMA_FROM_DEVICE) {
823 cmd->read_iter = kzalloc(sizeof(*cmd->read_iter), GFP_KERNEL);
824 if (!cmd->read_iter)
825 return -ENOMEM;
826
827 cmd->read_iov = dup_iter(cmd->read_iter, iter, GFP_KERNEL);
828 if (!cmd->read_iov) {
829 ret = -ENOMEM;
830 goto free_iter;
831 }
832 }
833
834 for_each_sgtable_sg(sg_table, sg, i) {
835 page = alloc_page(GFP_KERNEL);
836 if (!page) {
837 ret = -ENOMEM;
838 goto err;
839 }
840
841 nbytes = min_t(unsigned int, PAGE_SIZE, len);
842 sg_set_page(sg, page, nbytes, 0);
843
844 if (data_dir == DMA_TO_DEVICE &&
845 copy_page_from_iter(page, 0, nbytes, iter) != nbytes) {
846 ret = -EFAULT;
847 goto err;
848 }
849
850 len -= nbytes;
851 }
852
853 cmd->copied_iov = 1;
854 return 0;
855
856 err:
857 pr_err("Could not read %u bytes while handling misaligned cmd\n",
858 nbytes);
859
860 for_each_sgtable_sg(sg_table, sg, i) {
861 page = sg_page(sg);
862 if (page)
863 __free_page(page);
864 }
865 kfree(cmd->read_iov);
866 free_iter:
867 kfree(cmd->read_iter);
868 return ret;
869 }
870
871 static int
vhost_scsi_map_iov_to_sgl(struct vhost_scsi_cmd * cmd,struct iov_iter * iter,struct sg_table * sg_table,int sg_count,bool is_prot)872 vhost_scsi_map_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter,
873 struct sg_table *sg_table, int sg_count, bool is_prot)
874 {
875 struct scatterlist *sg = sg_table->sgl;
876 int ret;
877
878 while (iov_iter_count(iter)) {
879 ret = vhost_scsi_map_to_sgl(cmd, iter, sg_table, &sg, is_prot);
880 if (ret < 0) {
881 vhost_scsi_revert_map_iov_to_sgl(iter, sg_table->sgl,
882 sg);
883 return ret;
884 }
885 }
886
887 return 0;
888 }
889
890 static int
vhost_scsi_mapal(struct vhost_scsi * vs,struct vhost_scsi_cmd * cmd,size_t prot_bytes,struct iov_iter * prot_iter,size_t data_bytes,struct iov_iter * data_iter,int data_dir)891 vhost_scsi_mapal(struct vhost_scsi *vs, struct vhost_scsi_cmd *cmd,
892 size_t prot_bytes, struct iov_iter *prot_iter,
893 size_t data_bytes, struct iov_iter *data_iter, int data_dir)
894 {
895 int sgl_count, ret;
896
897 if (prot_bytes) {
898 sgl_count = vhost_scsi_calc_sgls(prot_iter, prot_bytes,
899 VHOST_SCSI_PREALLOC_PROT_SGLS);
900 cmd->prot_table.sgl = cmd->prot_sgl;
901 ret = sg_alloc_table_chained(&cmd->prot_table, sgl_count,
902 cmd->prot_table.sgl,
903 vs->inline_sg_cnt);
904 if (ret)
905 return ret;
906
907 cmd->tvc_prot_sgl_count = sgl_count;
908 pr_debug("%s prot_sg %p prot_sgl_count %u\n", __func__,
909 cmd->prot_table.sgl, cmd->tvc_prot_sgl_count);
910
911 ret = vhost_scsi_map_iov_to_sgl(cmd, prot_iter,
912 &cmd->prot_table,
913 cmd->tvc_prot_sgl_count, true);
914 if (ret < 0) {
915 sg_free_table_chained(&cmd->prot_table,
916 vs->inline_sg_cnt);
917 cmd->tvc_prot_sgl_count = 0;
918 return ret;
919 }
920 }
921 sgl_count = vhost_scsi_calc_sgls(data_iter, data_bytes,
922 VHOST_SCSI_PREALLOC_SGLS);
923 if (sgl_count < 0)
924 return sgl_count;
925
926 cmd->table.sgl = cmd->sgl;
927 ret = sg_alloc_table_chained(&cmd->table, sgl_count, cmd->table.sgl,
928 vs->inline_sg_cnt);
929 if (ret)
930 return ret;
931
932 cmd->tvc_sgl_count = sgl_count;
933 pr_debug("%s data_sg %p data_sgl_count %u\n", __func__,
934 cmd->table.sgl, cmd->tvc_sgl_count);
935
936 ret = vhost_scsi_map_iov_to_sgl(cmd, data_iter, &cmd->table,
937 cmd->tvc_sgl_count, false);
938 if (ret == -EINVAL)
939 ret = vhost_scsi_copy_iov_to_sgl(cmd, data_iter, &cmd->table,
940 cmd->tvc_sgl_count, data_dir);
941 if (ret < 0) {
942 sg_free_table_chained(&cmd->table, vs->inline_sg_cnt);
943 cmd->tvc_sgl_count = 0;
944 return ret;
945 }
946 return 0;
947 }
948
vhost_scsi_to_tcm_attr(int attr)949 static int vhost_scsi_to_tcm_attr(int attr)
950 {
951 switch (attr) {
952 case VIRTIO_SCSI_S_SIMPLE:
953 return TCM_SIMPLE_TAG;
954 case VIRTIO_SCSI_S_ORDERED:
955 return TCM_ORDERED_TAG;
956 case VIRTIO_SCSI_S_HEAD:
957 return TCM_HEAD_TAG;
958 case VIRTIO_SCSI_S_ACA:
959 return TCM_ACA_TAG;
960 default:
961 break;
962 }
963 return TCM_SIMPLE_TAG;
964 }
965
vhost_scsi_target_queue_cmd(struct vhost_scsi_nexus * nexus,struct vhost_scsi_cmd * cmd,unsigned char * cdb,u16 lun,int task_attr,int data_dir,u32 exp_data_len)966 static void vhost_scsi_target_queue_cmd(struct vhost_scsi_nexus *nexus,
967 struct vhost_scsi_cmd *cmd,
968 unsigned char *cdb, u16 lun,
969 int task_attr, int data_dir,
970 u32 exp_data_len)
971 {
972 struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
973 struct scatterlist *sg_ptr, *sg_prot_ptr = NULL;
974
975 /* FIXME: BIDI operation */
976 if (cmd->tvc_sgl_count) {
977 sg_ptr = cmd->table.sgl;
978
979 if (cmd->tvc_prot_sgl_count)
980 sg_prot_ptr = cmd->prot_table.sgl;
981 else
982 se_cmd->prot_pto = true;
983 } else {
984 sg_ptr = NULL;
985 }
986
987 se_cmd->tag = 0;
988 target_init_cmd(se_cmd, nexus->tvn_se_sess, &cmd->tvc_sense_buf[0],
989 lun, exp_data_len, vhost_scsi_to_tcm_attr(task_attr),
990 data_dir, TARGET_SCF_ACK_KREF);
991
992 if (target_submit_prep(se_cmd, cdb, sg_ptr,
993 cmd->tvc_sgl_count, NULL, 0, sg_prot_ptr,
994 cmd->tvc_prot_sgl_count, GFP_KERNEL))
995 return;
996
997 target_submit(se_cmd);
998 }
999
1000 static void
vhost_scsi_send_status(struct vhost_scsi * vs,struct vhost_virtqueue * vq,struct vhost_scsi_ctx * vc,u8 status)1001 vhost_scsi_send_status(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
1002 struct vhost_scsi_ctx *vc, u8 status)
1003 {
1004 struct virtio_scsi_cmd_resp rsp;
1005 struct iov_iter iov_iter;
1006 int ret;
1007
1008 memset(&rsp, 0, sizeof(rsp));
1009 rsp.status = status;
1010
1011 iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[vc->out], vc->in,
1012 sizeof(rsp));
1013
1014 ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1015
1016 if (likely(ret == sizeof(rsp)))
1017 vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);
1018 else
1019 pr_err("Faulted on virtio_scsi_cmd_resp\n");
1020 }
1021
1022 #define TYPE_IO_CMD 0
1023 #define TYPE_CTRL_TMF 1
1024 #define TYPE_CTRL_AN 2
1025
1026 static void
vhost_scsi_send_bad_target(struct vhost_scsi * vs,struct vhost_virtqueue * vq,struct vhost_scsi_ctx * vc,int type)1027 vhost_scsi_send_bad_target(struct vhost_scsi *vs,
1028 struct vhost_virtqueue *vq,
1029 struct vhost_scsi_ctx *vc, int type)
1030 {
1031 union {
1032 struct virtio_scsi_cmd_resp cmd;
1033 struct virtio_scsi_ctrl_tmf_resp tmf;
1034 struct virtio_scsi_ctrl_an_resp an;
1035 } rsp;
1036 struct iov_iter iov_iter;
1037 size_t rsp_size;
1038 int ret;
1039
1040 memset(&rsp, 0, sizeof(rsp));
1041
1042 if (type == TYPE_IO_CMD) {
1043 rsp_size = sizeof(struct virtio_scsi_cmd_resp);
1044 rsp.cmd.response = VIRTIO_SCSI_S_BAD_TARGET;
1045 } else if (type == TYPE_CTRL_TMF) {
1046 rsp_size = sizeof(struct virtio_scsi_ctrl_tmf_resp);
1047 rsp.tmf.response = VIRTIO_SCSI_S_BAD_TARGET;
1048 } else {
1049 rsp_size = sizeof(struct virtio_scsi_ctrl_an_resp);
1050 rsp.an.response = VIRTIO_SCSI_S_BAD_TARGET;
1051 }
1052
1053 iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[vc->out], vc->in,
1054 rsp_size);
1055
1056 ret = copy_to_iter(&rsp, rsp_size, &iov_iter);
1057
1058 if (likely(ret == rsp_size))
1059 vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);
1060 else
1061 pr_err("Faulted on virtio scsi type=%d\n", type);
1062 }
1063
1064 static int
vhost_scsi_get_desc(struct vhost_scsi * vs,struct vhost_virtqueue * vq,struct vhost_scsi_ctx * vc)1065 vhost_scsi_get_desc(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
1066 struct vhost_scsi_ctx *vc)
1067 {
1068 int ret = -ENXIO;
1069
1070 vc->head = vhost_get_vq_desc(vq, vq->iov,
1071 ARRAY_SIZE(vq->iov), &vc->out, &vc->in,
1072 NULL, NULL);
1073
1074 pr_debug("vhost_get_vq_desc: head: %d, out: %u in: %u\n",
1075 vc->head, vc->out, vc->in);
1076
1077 /* On error, stop handling until the next kick. */
1078 if (unlikely(vc->head < 0))
1079 goto done;
1080
1081 /* Nothing new? Wait for eventfd to tell us they refilled. */
1082 if (vc->head == vq->num) {
1083 if (unlikely(vhost_enable_notify(&vs->dev, vq))) {
1084 vhost_disable_notify(&vs->dev, vq);
1085 ret = -EAGAIN;
1086 }
1087 goto done;
1088 }
1089
1090 /*
1091 * Get the size of request and response buffers.
1092 * FIXME: Not correct for BIDI operation
1093 */
1094 vc->out_size = iov_length(vq->iov, vc->out);
1095 vc->in_size = iov_length(&vq->iov[vc->out], vc->in);
1096
1097 /*
1098 * Copy over the virtio-scsi request header, which for a
1099 * ANY_LAYOUT enabled guest may span multiple iovecs, or a
1100 * single iovec may contain both the header + outgoing
1101 * WRITE payloads.
1102 *
1103 * copy_from_iter() will advance out_iter, so that it will
1104 * point at the start of the outgoing WRITE payload, if
1105 * DMA_TO_DEVICE is set.
1106 */
1107 iov_iter_init(&vc->out_iter, ITER_SOURCE, vq->iov, vc->out, vc->out_size);
1108 ret = 0;
1109
1110 done:
1111 return ret;
1112 }
1113
1114 static int
vhost_scsi_chk_size(struct vhost_virtqueue * vq,struct vhost_scsi_ctx * vc)1115 vhost_scsi_chk_size(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc)
1116 {
1117 if (unlikely(vc->in_size < vc->rsp_size)) {
1118 vq_err(vq,
1119 "Response buf too small, need min %zu bytes got %zu",
1120 vc->rsp_size, vc->in_size);
1121 return -EINVAL;
1122 } else if (unlikely(vc->out_size < vc->req_size)) {
1123 vq_err(vq,
1124 "Request buf too small, need min %zu bytes got %zu",
1125 vc->req_size, vc->out_size);
1126 return -EIO;
1127 }
1128
1129 return 0;
1130 }
1131
1132 static int
vhost_scsi_get_req(struct vhost_virtqueue * vq,struct vhost_scsi_ctx * vc,struct vhost_scsi_tpg ** tpgp)1133 vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc,
1134 struct vhost_scsi_tpg **tpgp)
1135 {
1136 int ret = -EIO;
1137
1138 if (unlikely(!copy_from_iter_full(vc->req, vc->req_size,
1139 &vc->out_iter))) {
1140 vq_err(vq, "Faulted on copy_from_iter_full\n");
1141 } else if (unlikely(*vc->lunp != 1)) {
1142 /* virtio-scsi spec requires byte 0 of the lun to be 1 */
1143 vq_err(vq, "Illegal virtio-scsi lun: %u\n", *vc->lunp);
1144 } else {
1145 struct vhost_scsi_tpg **vs_tpg, *tpg = NULL;
1146
1147 if (vc->target) {
1148 /* validated at handler entry */
1149 vs_tpg = vhost_vq_get_backend(vq);
1150 tpg = READ_ONCE(vs_tpg[*vc->target]);
1151 if (unlikely(!tpg)) {
1152 vq_err(vq, "Target 0x%x does not exist\n", *vc->target);
1153 goto out;
1154 }
1155 }
1156
1157 if (tpgp)
1158 *tpgp = tpg;
1159 ret = 0;
1160 }
1161 out:
1162 return ret;
1163 }
1164
1165 static int
vhost_scsi_setup_resp_iovs(struct vhost_scsi_cmd * cmd,struct iovec * in_iovs,unsigned int in_iovs_cnt)1166 vhost_scsi_setup_resp_iovs(struct vhost_scsi_cmd *cmd, struct iovec *in_iovs,
1167 unsigned int in_iovs_cnt)
1168 {
1169 int i, cnt;
1170
1171 if (!in_iovs_cnt)
1172 return 0;
1173 /*
1174 * Initiator's normally just put the virtio_scsi_cmd_resp in the first
1175 * iov, but just in case they wedged in some data with it we check for
1176 * greater than or equal to the response struct.
1177 */
1178 if (in_iovs[0].iov_len >= sizeof(struct virtio_scsi_cmd_resp)) {
1179 cmd->tvc_resp_iovs = &cmd->tvc_resp_iov;
1180 cmd->tvc_resp_iovs_cnt = 1;
1181 } else {
1182 /*
1183 * Legacy descriptor layouts didn't specify that we must put
1184 * the entire response in one iov. Worst case we have a
1185 * iov per byte.
1186 */
1187 cnt = min(VHOST_SCSI_MAX_RESP_IOVS, in_iovs_cnt);
1188 cmd->tvc_resp_iovs = kcalloc(cnt, sizeof(struct iovec),
1189 GFP_KERNEL);
1190 if (!cmd->tvc_resp_iovs)
1191 return -ENOMEM;
1192
1193 cmd->tvc_resp_iovs_cnt = cnt;
1194 }
1195
1196 for (i = 0; i < cmd->tvc_resp_iovs_cnt; i++)
1197 cmd->tvc_resp_iovs[i] = in_iovs[i];
1198
1199 return 0;
1200 }
1201
vhost_buf_to_lun(u8 * lun_buf)1202 static u16 vhost_buf_to_lun(u8 *lun_buf)
1203 {
1204 return ((lun_buf[2] << 8) | lun_buf[3]) & 0x3FFF;
1205 }
1206
1207 static void
vhost_scsi_handle_vq(struct vhost_scsi * vs,struct vhost_virtqueue * vq)1208 vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
1209 {
1210 struct vhost_scsi_tpg **vs_tpg, *tpg;
1211 struct virtio_scsi_cmd_req v_req;
1212 struct virtio_scsi_cmd_req_pi v_req_pi;
1213 struct vhost_scsi_nexus *nexus;
1214 struct vhost_scsi_ctx vc;
1215 struct vhost_scsi_cmd *cmd;
1216 struct iov_iter in_iter, prot_iter, data_iter;
1217 u64 tag;
1218 u32 exp_data_len, data_direction;
1219 int ret, prot_bytes, c = 0;
1220 u16 lun;
1221 u8 task_attr;
1222 bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI);
1223 u8 *cdb;
1224
1225 mutex_lock(&vq->mutex);
1226 /*
1227 * We can handle the vq only after the endpoint is setup by calling the
1228 * VHOST_SCSI_SET_ENDPOINT ioctl.
1229 */
1230 vs_tpg = vhost_vq_get_backend(vq);
1231 if (!vs_tpg)
1232 goto out;
1233
1234 memset(&vc, 0, sizeof(vc));
1235 vc.rsp_size = sizeof(struct virtio_scsi_cmd_resp);
1236
1237 vhost_disable_notify(&vs->dev, vq);
1238
1239 do {
1240 ret = vhost_scsi_get_desc(vs, vq, &vc);
1241 if (ret)
1242 goto err;
1243
1244 /*
1245 * Setup pointers and values based upon different virtio-scsi
1246 * request header if T10_PI is enabled in KVM guest.
1247 */
1248 if (t10_pi) {
1249 vc.req = &v_req_pi;
1250 vc.req_size = sizeof(v_req_pi);
1251 vc.lunp = &v_req_pi.lun[0];
1252 vc.target = &v_req_pi.lun[1];
1253 } else {
1254 vc.req = &v_req;
1255 vc.req_size = sizeof(v_req);
1256 vc.lunp = &v_req.lun[0];
1257 vc.target = &v_req.lun[1];
1258 }
1259
1260 /*
1261 * Validate the size of request and response buffers.
1262 * Check for a sane response buffer so we can report
1263 * early errors back to the guest.
1264 */
1265 ret = vhost_scsi_chk_size(vq, &vc);
1266 if (ret)
1267 goto err;
1268
1269 ret = vhost_scsi_get_req(vq, &vc, &tpg);
1270 if (ret)
1271 goto err;
1272
1273 ret = -EIO; /* bad target on any error from here on */
1274
1275 /*
1276 * Determine data_direction by calculating the total outgoing
1277 * iovec sizes + incoming iovec sizes vs. virtio-scsi request +
1278 * response headers respectively.
1279 *
1280 * For DMA_TO_DEVICE this is out_iter, which is already pointing
1281 * to the right place.
1282 *
1283 * For DMA_FROM_DEVICE, the iovec will be just past the end
1284 * of the virtio-scsi response header in either the same
1285 * or immediately following iovec.
1286 *
1287 * Any associated T10_PI bytes for the outgoing / incoming
1288 * payloads are included in calculation of exp_data_len here.
1289 */
1290 prot_bytes = 0;
1291
1292 if (vc.out_size > vc.req_size) {
1293 data_direction = DMA_TO_DEVICE;
1294 exp_data_len = vc.out_size - vc.req_size;
1295 data_iter = vc.out_iter;
1296 } else if (vc.in_size > vc.rsp_size) {
1297 data_direction = DMA_FROM_DEVICE;
1298 exp_data_len = vc.in_size - vc.rsp_size;
1299
1300 iov_iter_init(&in_iter, ITER_DEST, &vq->iov[vc.out], vc.in,
1301 vc.rsp_size + exp_data_len);
1302 iov_iter_advance(&in_iter, vc.rsp_size);
1303 data_iter = in_iter;
1304 } else {
1305 data_direction = DMA_NONE;
1306 exp_data_len = 0;
1307 }
1308 /*
1309 * If T10_PI header + payload is present, setup prot_iter values
1310 * and recalculate data_iter for vhost_scsi_mapal() mapping to
1311 * host scatterlists via get_user_pages_fast().
1312 */
1313 if (t10_pi) {
1314 if (v_req_pi.pi_bytesout) {
1315 if (data_direction != DMA_TO_DEVICE) {
1316 vq_err(vq, "Received non zero pi_bytesout,"
1317 " but wrong data_direction\n");
1318 goto err;
1319 }
1320 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout);
1321 } else if (v_req_pi.pi_bytesin) {
1322 if (data_direction != DMA_FROM_DEVICE) {
1323 vq_err(vq, "Received non zero pi_bytesin,"
1324 " but wrong data_direction\n");
1325 goto err;
1326 }
1327 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin);
1328 }
1329 /*
1330 * Set prot_iter to data_iter and truncate it to
1331 * prot_bytes, and advance data_iter past any
1332 * preceding prot_bytes that may be present.
1333 *
1334 * Also fix up the exp_data_len to reflect only the
1335 * actual data payload length.
1336 */
1337 if (prot_bytes) {
1338 exp_data_len -= prot_bytes;
1339 prot_iter = data_iter;
1340 iov_iter_truncate(&prot_iter, prot_bytes);
1341 iov_iter_advance(&data_iter, prot_bytes);
1342 }
1343 tag = vhost64_to_cpu(vq, v_req_pi.tag);
1344 task_attr = v_req_pi.task_attr;
1345 cdb = &v_req_pi.cdb[0];
1346 lun = vhost_buf_to_lun(v_req_pi.lun);
1347 } else {
1348 tag = vhost64_to_cpu(vq, v_req.tag);
1349 task_attr = v_req.task_attr;
1350 cdb = &v_req.cdb[0];
1351 lun = vhost_buf_to_lun(v_req.lun);
1352 }
1353 /*
1354 * Check that the received CDB size does not exceeded our
1355 * hardcoded max for vhost-scsi, then get a pre-allocated
1356 * cmd descriptor for the new virtio-scsi tag.
1357 *
1358 * TODO what if cdb was too small for varlen cdb header?
1359 */
1360 if (unlikely(scsi_command_size(cdb) > VHOST_SCSI_MAX_CDB_SIZE)) {
1361 vq_err(vq, "Received SCSI CDB with command_size: %d that"
1362 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1363 scsi_command_size(cdb), VHOST_SCSI_MAX_CDB_SIZE);
1364 goto err;
1365 }
1366
1367 nexus = tpg->tpg_nexus;
1368 if (!nexus) {
1369 vq_err(vq, "Unable to locate active struct vhost_scsi_nexus\n");
1370 ret = -EIO;
1371 goto err;
1372 }
1373
1374 cmd = vhost_scsi_get_cmd(vq, tag);
1375 if (IS_ERR(cmd)) {
1376 ret = PTR_ERR(cmd);
1377 vq_err(vq, "vhost_scsi_get_tag failed %dd\n", ret);
1378 goto err;
1379 }
1380 cmd->tvc_vq = vq;
1381
1382 ret = vhost_scsi_setup_resp_iovs(cmd, &vq->iov[vc.out], vc.in);
1383 if (ret) {
1384 vq_err(vq, "Failed to alloc recv iovs\n");
1385 vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd);
1386 goto err;
1387 }
1388
1389 pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n",
1390 cdb[0], lun);
1391 pr_debug("cmd: %p exp_data_len: %d, prot_bytes: %d data_direction:"
1392 " %d\n", cmd, exp_data_len, prot_bytes, data_direction);
1393
1394 if (data_direction != DMA_NONE) {
1395 ret = vhost_scsi_mapal(vs, cmd, prot_bytes, &prot_iter,
1396 exp_data_len, &data_iter,
1397 data_direction);
1398 if (unlikely(ret)) {
1399 vq_err(vq, "Failed to map iov to sgl\n");
1400 vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd);
1401 goto err;
1402 }
1403 }
1404 /*
1405 * Save the descriptor from vhost_get_vq_desc() to be used to
1406 * complete the virtio-scsi request in TCM callback context via
1407 * vhost_scsi_queue_data_in() and vhost_scsi_queue_status()
1408 */
1409 cmd->tvc_vq_desc = vc.head;
1410 vhost_scsi_target_queue_cmd(nexus, cmd, cdb, lun, task_attr,
1411 data_direction,
1412 exp_data_len + prot_bytes);
1413 ret = 0;
1414 err:
1415 /*
1416 * ENXIO: No more requests, or read error, wait for next kick
1417 * EINVAL: Invalid response buffer, drop the request
1418 * EIO: Respond with bad target
1419 * EAGAIN: Pending request
1420 * ENOMEM: Could not allocate resources for request
1421 */
1422 if (ret == -ENXIO)
1423 break;
1424 else if (ret == -EIO)
1425 vhost_scsi_send_bad_target(vs, vq, &vc, TYPE_IO_CMD);
1426 else if (ret == -ENOMEM)
1427 vhost_scsi_send_status(vs, vq, &vc,
1428 SAM_STAT_TASK_SET_FULL);
1429 } while (likely(!vhost_exceeds_weight(vq, ++c, 0)));
1430 out:
1431 mutex_unlock(&vq->mutex);
1432 }
1433
1434 static void
vhost_scsi_send_tmf_resp(struct vhost_scsi * vs,struct vhost_virtqueue * vq,int in_iovs,int vq_desc,struct iovec * resp_iov,int tmf_resp_code)1435 vhost_scsi_send_tmf_resp(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
1436 int in_iovs, int vq_desc, struct iovec *resp_iov,
1437 int tmf_resp_code)
1438 {
1439 struct virtio_scsi_ctrl_tmf_resp rsp;
1440 struct iov_iter iov_iter;
1441 int ret;
1442
1443 pr_debug("%s\n", __func__);
1444 memset(&rsp, 0, sizeof(rsp));
1445 rsp.response = tmf_resp_code;
1446
1447 iov_iter_init(&iov_iter, ITER_DEST, resp_iov, in_iovs, sizeof(rsp));
1448
1449 ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1450 if (likely(ret == sizeof(rsp)))
1451 vhost_add_used_and_signal(&vs->dev, vq, vq_desc, 0);
1452 else
1453 pr_err("Faulted on virtio_scsi_ctrl_tmf_resp\n");
1454 }
1455
vhost_scsi_tmf_resp_work(struct vhost_work * work)1456 static void vhost_scsi_tmf_resp_work(struct vhost_work *work)
1457 {
1458 struct vhost_scsi_tmf *tmf = container_of(work, struct vhost_scsi_tmf,
1459 vwork);
1460 int resp_code;
1461
1462 if (tmf->scsi_resp == TMR_FUNCTION_COMPLETE)
1463 resp_code = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED;
1464 else
1465 resp_code = VIRTIO_SCSI_S_FUNCTION_REJECTED;
1466
1467 mutex_lock(&tmf->svq->vq.mutex);
1468 vhost_scsi_send_tmf_resp(tmf->vhost, &tmf->svq->vq, tmf->in_iovs,
1469 tmf->vq_desc, &tmf->resp_iov, resp_code);
1470 mutex_unlock(&tmf->svq->vq.mutex);
1471
1472 vhost_scsi_release_tmf_res(tmf);
1473 }
1474
vhost_scsi_tmf_flush_work(struct work_struct * work)1475 static void vhost_scsi_tmf_flush_work(struct work_struct *work)
1476 {
1477 struct vhost_scsi_tmf *tmf = container_of(work, struct vhost_scsi_tmf,
1478 flush_work);
1479 struct vhost_virtqueue *vq = &tmf->svq->vq;
1480 /*
1481 * Make sure we have sent responses for other commands before we
1482 * send our response.
1483 */
1484 vhost_dev_flush(vq->dev);
1485 if (!vhost_vq_work_queue(vq, &tmf->vwork))
1486 vhost_scsi_release_tmf_res(tmf);
1487 }
1488
1489 static void
vhost_scsi_handle_tmf(struct vhost_scsi * vs,struct vhost_scsi_tpg * tpg,struct vhost_virtqueue * vq,struct virtio_scsi_ctrl_tmf_req * vtmf,struct vhost_scsi_ctx * vc)1490 vhost_scsi_handle_tmf(struct vhost_scsi *vs, struct vhost_scsi_tpg *tpg,
1491 struct vhost_virtqueue *vq,
1492 struct virtio_scsi_ctrl_tmf_req *vtmf,
1493 struct vhost_scsi_ctx *vc)
1494 {
1495 struct vhost_scsi_virtqueue *svq = container_of(vq,
1496 struct vhost_scsi_virtqueue, vq);
1497 struct vhost_scsi_tmf *tmf;
1498
1499 if (vhost32_to_cpu(vq, vtmf->subtype) !=
1500 VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET)
1501 goto send_reject;
1502
1503 if (!tpg->tpg_nexus || !tpg->tpg_nexus->tvn_se_sess) {
1504 pr_err("Unable to locate active struct vhost_scsi_nexus for LUN RESET.\n");
1505 goto send_reject;
1506 }
1507
1508 tmf = kzalloc(sizeof(*tmf), GFP_KERNEL);
1509 if (!tmf)
1510 goto send_reject;
1511
1512 INIT_WORK(&tmf->flush_work, vhost_scsi_tmf_flush_work);
1513 vhost_work_init(&tmf->vwork, vhost_scsi_tmf_resp_work);
1514 tmf->vhost = vs;
1515 tmf->svq = svq;
1516 tmf->resp_iov = vq->iov[vc->out];
1517 tmf->vq_desc = vc->head;
1518 tmf->in_iovs = vc->in;
1519 tmf->inflight = vhost_scsi_get_inflight(vq);
1520
1521 if (target_submit_tmr(&tmf->se_cmd, tpg->tpg_nexus->tvn_se_sess, NULL,
1522 vhost_buf_to_lun(vtmf->lun), NULL,
1523 TMR_LUN_RESET, GFP_KERNEL, 0,
1524 TARGET_SCF_ACK_KREF) < 0) {
1525 vhost_scsi_release_tmf_res(tmf);
1526 goto send_reject;
1527 }
1528
1529 return;
1530
1531 send_reject:
1532 vhost_scsi_send_tmf_resp(vs, vq, vc->in, vc->head, &vq->iov[vc->out],
1533 VIRTIO_SCSI_S_FUNCTION_REJECTED);
1534 }
1535
1536 static void
vhost_scsi_send_an_resp(struct vhost_scsi * vs,struct vhost_virtqueue * vq,struct vhost_scsi_ctx * vc)1537 vhost_scsi_send_an_resp(struct vhost_scsi *vs,
1538 struct vhost_virtqueue *vq,
1539 struct vhost_scsi_ctx *vc)
1540 {
1541 struct virtio_scsi_ctrl_an_resp rsp;
1542 struct iov_iter iov_iter;
1543 int ret;
1544
1545 pr_debug("%s\n", __func__);
1546 memset(&rsp, 0, sizeof(rsp)); /* event_actual = 0 */
1547 rsp.response = VIRTIO_SCSI_S_OK;
1548
1549 iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[vc->out], vc->in, sizeof(rsp));
1550
1551 ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1552 if (likely(ret == sizeof(rsp)))
1553 vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);
1554 else
1555 pr_err("Faulted on virtio_scsi_ctrl_an_resp\n");
1556 }
1557
1558 static void
vhost_scsi_ctl_handle_vq(struct vhost_scsi * vs,struct vhost_virtqueue * vq)1559 vhost_scsi_ctl_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
1560 {
1561 struct vhost_scsi_tpg *tpg;
1562 union {
1563 __virtio32 type;
1564 struct virtio_scsi_ctrl_an_req an;
1565 struct virtio_scsi_ctrl_tmf_req tmf;
1566 } v_req;
1567 struct vhost_scsi_ctx vc;
1568 size_t typ_size;
1569 int ret, c = 0;
1570
1571 mutex_lock(&vq->mutex);
1572 /*
1573 * We can handle the vq only after the endpoint is setup by calling the
1574 * VHOST_SCSI_SET_ENDPOINT ioctl.
1575 */
1576 if (!vhost_vq_get_backend(vq))
1577 goto out;
1578
1579 memset(&vc, 0, sizeof(vc));
1580
1581 vhost_disable_notify(&vs->dev, vq);
1582
1583 do {
1584 ret = vhost_scsi_get_desc(vs, vq, &vc);
1585 if (ret)
1586 goto err;
1587
1588 /*
1589 * Get the request type first in order to setup
1590 * other parameters dependent on the type.
1591 */
1592 vc.req = &v_req.type;
1593 typ_size = sizeof(v_req.type);
1594
1595 if (unlikely(!copy_from_iter_full(vc.req, typ_size,
1596 &vc.out_iter))) {
1597 vq_err(vq, "Faulted on copy_from_iter tmf type\n");
1598 /*
1599 * The size of the response buffer depends on the
1600 * request type and must be validated against it.
1601 * Since the request type is not known, don't send
1602 * a response.
1603 */
1604 continue;
1605 }
1606
1607 switch (vhost32_to_cpu(vq, v_req.type)) {
1608 case VIRTIO_SCSI_T_TMF:
1609 vc.req = &v_req.tmf;
1610 vc.req_size = sizeof(struct virtio_scsi_ctrl_tmf_req);
1611 vc.rsp_size = sizeof(struct virtio_scsi_ctrl_tmf_resp);
1612 vc.lunp = &v_req.tmf.lun[0];
1613 vc.target = &v_req.tmf.lun[1];
1614 break;
1615 case VIRTIO_SCSI_T_AN_QUERY:
1616 case VIRTIO_SCSI_T_AN_SUBSCRIBE:
1617 vc.req = &v_req.an;
1618 vc.req_size = sizeof(struct virtio_scsi_ctrl_an_req);
1619 vc.rsp_size = sizeof(struct virtio_scsi_ctrl_an_resp);
1620 vc.lunp = &v_req.an.lun[0];
1621 vc.target = NULL;
1622 break;
1623 default:
1624 vq_err(vq, "Unknown control request %d", v_req.type);
1625 continue;
1626 }
1627
1628 /*
1629 * Validate the size of request and response buffers.
1630 * Check for a sane response buffer so we can report
1631 * early errors back to the guest.
1632 */
1633 ret = vhost_scsi_chk_size(vq, &vc);
1634 if (ret)
1635 goto err;
1636
1637 /*
1638 * Get the rest of the request now that its size is known.
1639 */
1640 vc.req += typ_size;
1641 vc.req_size -= typ_size;
1642
1643 ret = vhost_scsi_get_req(vq, &vc, &tpg);
1644 if (ret)
1645 goto err;
1646
1647 if (v_req.type == VIRTIO_SCSI_T_TMF)
1648 vhost_scsi_handle_tmf(vs, tpg, vq, &v_req.tmf, &vc);
1649 else
1650 vhost_scsi_send_an_resp(vs, vq, &vc);
1651 err:
1652 /*
1653 * ENXIO: No more requests, or read error, wait for next kick
1654 * EINVAL: Invalid response buffer, drop the request
1655 * EIO: Respond with bad target
1656 * EAGAIN: Pending request
1657 */
1658 if (ret == -ENXIO)
1659 break;
1660 else if (ret == -EIO)
1661 vhost_scsi_send_bad_target(vs, vq, &vc,
1662 v_req.type == VIRTIO_SCSI_T_TMF ?
1663 TYPE_CTRL_TMF :
1664 TYPE_CTRL_AN);
1665 } while (likely(!vhost_exceeds_weight(vq, ++c, 0)));
1666 out:
1667 mutex_unlock(&vq->mutex);
1668 }
1669
vhost_scsi_ctl_handle_kick(struct vhost_work * work)1670 static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)
1671 {
1672 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1673 poll.work);
1674 struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1675
1676 pr_debug("%s: The handling func for control queue.\n", __func__);
1677 vhost_scsi_ctl_handle_vq(vs, vq);
1678 }
1679
1680 static void
vhost_scsi_send_evt(struct vhost_scsi * vs,struct vhost_virtqueue * vq,struct vhost_scsi_tpg * tpg,struct se_lun * lun,u32 event,u32 reason)1681 vhost_scsi_send_evt(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
1682 struct vhost_scsi_tpg *tpg, struct se_lun *lun,
1683 u32 event, u32 reason)
1684 {
1685 struct vhost_scsi_evt *evt;
1686
1687 evt = vhost_scsi_allocate_evt(vs, event, reason);
1688 if (!evt)
1689 return;
1690
1691 if (tpg && lun) {
1692 /* TODO: share lun setup code with virtio-scsi.ko */
1693 /*
1694 * Note: evt->event is zeroed when we allocate it and
1695 * lun[4-7] need to be zero according to virtio-scsi spec.
1696 */
1697 evt->event.lun[0] = 0x01;
1698 evt->event.lun[1] = tpg->tport_tpgt;
1699 if (lun->unpacked_lun >= 256)
1700 evt->event.lun[2] = lun->unpacked_lun >> 8 | 0x40 ;
1701 evt->event.lun[3] = lun->unpacked_lun & 0xFF;
1702 }
1703
1704 llist_add(&evt->list, &vs->vs_event_list);
1705 if (!vhost_vq_work_queue(vq, &vs->vs_event_work))
1706 vhost_scsi_complete_events(vs, true);
1707 }
1708
vhost_scsi_evt_handle_kick(struct vhost_work * work)1709 static void vhost_scsi_evt_handle_kick(struct vhost_work *work)
1710 {
1711 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1712 poll.work);
1713 struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1714
1715 mutex_lock(&vq->mutex);
1716 if (!vhost_vq_get_backend(vq))
1717 goto out;
1718
1719 if (vs->vs_events_missed)
1720 vhost_scsi_send_evt(vs, vq, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT,
1721 0);
1722 out:
1723 mutex_unlock(&vq->mutex);
1724 }
1725
vhost_scsi_handle_kick(struct vhost_work * work)1726 static void vhost_scsi_handle_kick(struct vhost_work *work)
1727 {
1728 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1729 poll.work);
1730 struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1731
1732 vhost_scsi_handle_vq(vs, vq);
1733 }
1734
1735 /* Callers must hold dev mutex */
vhost_scsi_flush(struct vhost_scsi * vs)1736 static void vhost_scsi_flush(struct vhost_scsi *vs)
1737 {
1738 int i;
1739
1740 /* Init new inflight and remember the old inflight */
1741 vhost_scsi_init_inflight(vs, vs->old_inflight);
1742
1743 /*
1744 * The inflight->kref was initialized to 1. We decrement it here to
1745 * indicate the start of the flush operation so that it will reach 0
1746 * when all the reqs are finished.
1747 */
1748 for (i = 0; i < vs->dev.nvqs; i++)
1749 kref_put(&vs->old_inflight[i]->kref, vhost_scsi_done_inflight);
1750
1751 /* Flush both the vhost poll and vhost work */
1752 vhost_dev_flush(&vs->dev);
1753
1754 /* Wait for all reqs issued before the flush to be finished */
1755 for (i = 0; i < vs->dev.nvqs; i++)
1756 wait_for_completion(&vs->old_inflight[i]->comp);
1757 }
1758
vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue * vq)1759 static void vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue *vq)
1760 {
1761 struct vhost_scsi_virtqueue *svq = container_of(vq,
1762 struct vhost_scsi_virtqueue, vq);
1763 struct vhost_scsi_cmd *tv_cmd;
1764 unsigned int i;
1765
1766 if (!svq->scsi_cmds)
1767 return;
1768
1769 for (i = 0; i < svq->max_cmds; i++) {
1770 tv_cmd = &svq->scsi_cmds[i];
1771
1772 kfree(tv_cmd->sgl);
1773 kfree(tv_cmd->prot_sgl);
1774 }
1775
1776 sbitmap_free(&svq->scsi_tags);
1777 kfree(svq->upages);
1778 kfree(svq->scsi_cmds);
1779 svq->scsi_cmds = NULL;
1780 }
1781
vhost_scsi_setup_vq_cmds(struct vhost_virtqueue * vq,int max_cmds)1782 static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds)
1783 {
1784 struct vhost_scsi_virtqueue *svq = container_of(vq,
1785 struct vhost_scsi_virtqueue, vq);
1786 struct vhost_scsi *vs = svq->vs;
1787 struct vhost_scsi_cmd *tv_cmd;
1788 unsigned int i;
1789
1790 if (svq->scsi_cmds)
1791 return 0;
1792
1793 if (sbitmap_init_node(&svq->scsi_tags, max_cmds, -1, GFP_KERNEL,
1794 NUMA_NO_NODE, false, true))
1795 return -ENOMEM;
1796 svq->max_cmds = max_cmds;
1797
1798 svq->scsi_cmds = kcalloc(max_cmds, sizeof(*tv_cmd), GFP_KERNEL);
1799 if (!svq->scsi_cmds) {
1800 sbitmap_free(&svq->scsi_tags);
1801 return -ENOMEM;
1802 }
1803
1804 svq->upages = kcalloc(VHOST_SCSI_PREALLOC_UPAGES, sizeof(struct page *),
1805 GFP_KERNEL);
1806 if (!svq->upages)
1807 goto out;
1808
1809 for (i = 0; i < max_cmds; i++) {
1810 tv_cmd = &svq->scsi_cmds[i];
1811
1812 if (vs->inline_sg_cnt) {
1813 tv_cmd->sgl = kcalloc(vs->inline_sg_cnt,
1814 sizeof(struct scatterlist),
1815 GFP_KERNEL);
1816 if (!tv_cmd->sgl) {
1817 pr_err("Unable to allocate tv_cmd->sgl\n");
1818 goto out;
1819 }
1820 }
1821
1822 if (vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI) &&
1823 vs->inline_sg_cnt) {
1824 tv_cmd->prot_sgl = kcalloc(vs->inline_sg_cnt,
1825 sizeof(struct scatterlist),
1826 GFP_KERNEL);
1827 if (!tv_cmd->prot_sgl) {
1828 pr_err("Unable to allocate tv_cmd->prot_sgl\n");
1829 goto out;
1830 }
1831 }
1832 }
1833 return 0;
1834 out:
1835 vhost_scsi_destroy_vq_cmds(vq);
1836 return -ENOMEM;
1837 }
1838
1839 /*
1840 * Called from vhost_scsi_ioctl() context to walk the list of available
1841 * vhost_scsi_tpg with an active struct vhost_scsi_nexus
1842 *
1843 * The lock nesting rule is:
1844 * vs->dev.mutex -> vhost_scsi_mutex -> tpg->tv_tpg_mutex -> vq->mutex
1845 */
1846 static int
vhost_scsi_set_endpoint(struct vhost_scsi * vs,struct vhost_scsi_target * t)1847 vhost_scsi_set_endpoint(struct vhost_scsi *vs,
1848 struct vhost_scsi_target *t)
1849 {
1850 struct se_portal_group *se_tpg;
1851 struct vhost_scsi_tport *tv_tport;
1852 struct vhost_scsi_tpg *tpg;
1853 struct vhost_scsi_tpg **vs_tpg;
1854 struct vhost_virtqueue *vq;
1855 int index, ret, i, len;
1856 bool match = false;
1857
1858 mutex_lock(&vs->dev.mutex);
1859
1860 /* Verify that ring has been setup correctly. */
1861 for (index = 0; index < vs->dev.nvqs; ++index) {
1862 /* Verify that ring has been setup correctly. */
1863 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
1864 ret = -EFAULT;
1865 goto out;
1866 }
1867 }
1868
1869 if (vs->vs_tpg) {
1870 pr_err("vhost-scsi endpoint already set for %s.\n",
1871 vs->vs_vhost_wwpn);
1872 ret = -EEXIST;
1873 goto out;
1874 }
1875
1876 len = sizeof(vs_tpg[0]) * VHOST_SCSI_MAX_TARGET;
1877 vs_tpg = kzalloc(len, GFP_KERNEL);
1878 if (!vs_tpg) {
1879 ret = -ENOMEM;
1880 goto out;
1881 }
1882
1883 mutex_lock(&vhost_scsi_mutex);
1884 list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) {
1885 mutex_lock(&tpg->tv_tpg_mutex);
1886 if (!tpg->tpg_nexus) {
1887 mutex_unlock(&tpg->tv_tpg_mutex);
1888 continue;
1889 }
1890 if (tpg->tv_tpg_vhost_count != 0) {
1891 mutex_unlock(&tpg->tv_tpg_mutex);
1892 continue;
1893 }
1894 tv_tport = tpg->tport;
1895
1896 if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1897 /*
1898 * In order to ensure individual vhost-scsi configfs
1899 * groups cannot be removed while in use by vhost ioctl,
1900 * go ahead and take an explicit se_tpg->tpg_group.cg_item
1901 * dependency now.
1902 */
1903 se_tpg = &tpg->se_tpg;
1904 ret = target_depend_item(&se_tpg->tpg_group.cg_item);
1905 if (ret) {
1906 pr_warn("target_depend_item() failed: %d\n", ret);
1907 mutex_unlock(&tpg->tv_tpg_mutex);
1908 mutex_unlock(&vhost_scsi_mutex);
1909 goto undepend;
1910 }
1911 tpg->tv_tpg_vhost_count++;
1912 tpg->vhost_scsi = vs;
1913 vs_tpg[tpg->tport_tpgt] = tpg;
1914 match = true;
1915 }
1916 mutex_unlock(&tpg->tv_tpg_mutex);
1917 }
1918 mutex_unlock(&vhost_scsi_mutex);
1919
1920 if (match) {
1921 memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
1922 sizeof(vs->vs_vhost_wwpn));
1923
1924 for (i = VHOST_SCSI_VQ_IO; i < vs->dev.nvqs; i++) {
1925 vq = &vs->vqs[i].vq;
1926 if (!vhost_vq_is_setup(vq))
1927 continue;
1928
1929 ret = vhost_scsi_setup_vq_cmds(vq, vq->num);
1930 if (ret)
1931 goto destroy_vq_cmds;
1932 }
1933
1934 for (i = 0; i < vs->dev.nvqs; i++) {
1935 vq = &vs->vqs[i].vq;
1936 mutex_lock(&vq->mutex);
1937 vhost_vq_set_backend(vq, vs_tpg);
1938 vhost_vq_init_access(vq);
1939 mutex_unlock(&vq->mutex);
1940 }
1941 ret = 0;
1942 } else {
1943 ret = -ENODEV;
1944 goto free_tpg;
1945 }
1946
1947 /*
1948 * Act as synchronize_rcu to make sure requests after this point
1949 * see a fully setup device.
1950 */
1951 vhost_scsi_flush(vs);
1952 vs->vs_tpg = vs_tpg;
1953 goto out;
1954
1955 destroy_vq_cmds:
1956 for (i--; i >= VHOST_SCSI_VQ_IO; i--) {
1957 if (!vhost_vq_get_backend(&vs->vqs[i].vq))
1958 vhost_scsi_destroy_vq_cmds(&vs->vqs[i].vq);
1959 }
1960 undepend:
1961 for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1962 tpg = vs_tpg[i];
1963 if (tpg) {
1964 mutex_lock(&tpg->tv_tpg_mutex);
1965 tpg->vhost_scsi = NULL;
1966 tpg->tv_tpg_vhost_count--;
1967 mutex_unlock(&tpg->tv_tpg_mutex);
1968 target_undepend_item(&tpg->se_tpg.tpg_group.cg_item);
1969 }
1970 }
1971 free_tpg:
1972 kfree(vs_tpg);
1973 out:
1974 mutex_unlock(&vs->dev.mutex);
1975 return ret;
1976 }
1977
1978 static int
vhost_scsi_clear_endpoint(struct vhost_scsi * vs,struct vhost_scsi_target * t)1979 vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
1980 struct vhost_scsi_target *t)
1981 {
1982 struct se_portal_group *se_tpg;
1983 struct vhost_scsi_tport *tv_tport;
1984 struct vhost_scsi_tpg *tpg;
1985 struct vhost_virtqueue *vq;
1986 bool match = false;
1987 int index, ret, i;
1988 u8 target;
1989
1990 mutex_lock(&vs->dev.mutex);
1991 /* Verify that ring has been setup correctly. */
1992 for (index = 0; index < vs->dev.nvqs; ++index) {
1993 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
1994 ret = -EFAULT;
1995 goto err_dev;
1996 }
1997 }
1998
1999 if (!vs->vs_tpg) {
2000 ret = 0;
2001 goto err_dev;
2002 }
2003
2004 for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
2005 target = i;
2006 tpg = vs->vs_tpg[target];
2007 if (!tpg)
2008 continue;
2009
2010 tv_tport = tpg->tport;
2011 if (!tv_tport) {
2012 ret = -ENODEV;
2013 goto err_dev;
2014 }
2015
2016 if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
2017 pr_warn("tv_tport->tport_name: %s, tpg->tport_tpgt: %hu"
2018 " does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n",
2019 tv_tport->tport_name, tpg->tport_tpgt,
2020 t->vhost_wwpn, t->vhost_tpgt);
2021 ret = -EINVAL;
2022 goto err_dev;
2023 }
2024 match = true;
2025 }
2026 if (!match)
2027 goto free_vs_tpg;
2028
2029 /* Prevent new cmds from starting and accessing the tpgs/sessions */
2030 for (i = 0; i < vs->dev.nvqs; i++) {
2031 vq = &vs->vqs[i].vq;
2032 mutex_lock(&vq->mutex);
2033 vhost_vq_set_backend(vq, NULL);
2034 mutex_unlock(&vq->mutex);
2035 }
2036 /* Make sure cmds are not running before tearing them down. */
2037 vhost_scsi_flush(vs);
2038
2039 for (i = 0; i < vs->dev.nvqs; i++) {
2040 vq = &vs->vqs[i].vq;
2041 vhost_scsi_destroy_vq_cmds(vq);
2042 }
2043
2044 /*
2045 * We can now release our hold on the tpg and sessions and userspace
2046 * can free them after this point.
2047 */
2048 for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
2049 target = i;
2050 tpg = vs->vs_tpg[target];
2051 if (!tpg)
2052 continue;
2053
2054 mutex_lock(&tpg->tv_tpg_mutex);
2055
2056 tpg->tv_tpg_vhost_count--;
2057 tpg->vhost_scsi = NULL;
2058 vs->vs_tpg[target] = NULL;
2059
2060 mutex_unlock(&tpg->tv_tpg_mutex);
2061
2062 se_tpg = &tpg->se_tpg;
2063 target_undepend_item(&se_tpg->tpg_group.cg_item);
2064 }
2065
2066 free_vs_tpg:
2067 /*
2068 * Act as synchronize_rcu to make sure access to
2069 * old vs->vs_tpg is finished.
2070 */
2071 vhost_scsi_flush(vs);
2072 kfree(vs->vs_tpg);
2073 vs->vs_tpg = NULL;
2074 memset(vs->vs_vhost_wwpn, 0, sizeof(vs->vs_vhost_wwpn));
2075 WARN_ON(vs->vs_events_nr);
2076 mutex_unlock(&vs->dev.mutex);
2077 return 0;
2078
2079 err_dev:
2080 mutex_unlock(&vs->dev.mutex);
2081 return ret;
2082 }
2083
vhost_scsi_set_features(struct vhost_scsi * vs,u64 features)2084 static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features)
2085 {
2086 struct vhost_virtqueue *vq;
2087 int i;
2088
2089 if (features & ~VHOST_SCSI_FEATURES)
2090 return -EOPNOTSUPP;
2091
2092 mutex_lock(&vs->dev.mutex);
2093 if ((features & (1 << VHOST_F_LOG_ALL)) &&
2094 !vhost_log_access_ok(&vs->dev)) {
2095 mutex_unlock(&vs->dev.mutex);
2096 return -EFAULT;
2097 }
2098
2099 for (i = 0; i < vs->dev.nvqs; i++) {
2100 vq = &vs->vqs[i].vq;
2101 mutex_lock(&vq->mutex);
2102 vq->acked_features = features;
2103 mutex_unlock(&vq->mutex);
2104 }
2105 mutex_unlock(&vs->dev.mutex);
2106 return 0;
2107 }
2108
vhost_scsi_open(struct inode * inode,struct file * f)2109 static int vhost_scsi_open(struct inode *inode, struct file *f)
2110 {
2111 struct vhost_scsi_virtqueue *svq;
2112 struct vhost_scsi *vs;
2113 struct vhost_virtqueue **vqs;
2114 int r = -ENOMEM, i, nvqs = vhost_scsi_max_io_vqs;
2115
2116 vs = kvzalloc(sizeof(*vs), GFP_KERNEL);
2117 if (!vs)
2118 goto err_vs;
2119 vs->inline_sg_cnt = vhost_scsi_inline_sg_cnt;
2120
2121 if (nvqs > VHOST_SCSI_MAX_IO_VQ) {
2122 pr_err("Invalid max_io_vqs of %d. Using %d.\n", nvqs,
2123 VHOST_SCSI_MAX_IO_VQ);
2124 nvqs = VHOST_SCSI_MAX_IO_VQ;
2125 } else if (nvqs == 0) {
2126 pr_err("Invalid max_io_vqs of %d. Using 1.\n", nvqs);
2127 nvqs = 1;
2128 }
2129 nvqs += VHOST_SCSI_VQ_IO;
2130
2131 vs->old_inflight = kmalloc_array(nvqs, sizeof(*vs->old_inflight),
2132 GFP_KERNEL | __GFP_ZERO);
2133 if (!vs->old_inflight)
2134 goto err_inflight;
2135
2136 vs->vqs = kmalloc_array(nvqs, sizeof(*vs->vqs),
2137 GFP_KERNEL | __GFP_ZERO);
2138 if (!vs->vqs)
2139 goto err_vqs;
2140
2141 vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
2142 if (!vqs)
2143 goto err_local_vqs;
2144
2145 vhost_work_init(&vs->vs_event_work, vhost_scsi_evt_work);
2146
2147 vs->vs_events_nr = 0;
2148 vs->vs_events_missed = false;
2149
2150 vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;
2151 vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
2152 vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;
2153 vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;
2154 for (i = VHOST_SCSI_VQ_IO; i < nvqs; i++) {
2155 svq = &vs->vqs[i];
2156
2157 vqs[i] = &svq->vq;
2158 svq->vs = vs;
2159 init_llist_head(&svq->completion_list);
2160 vhost_work_init(&svq->completion_work,
2161 vhost_scsi_complete_cmd_work);
2162 svq->vq.handle_kick = vhost_scsi_handle_kick;
2163 }
2164 vhost_dev_init(&vs->dev, vqs, nvqs, UIO_MAXIOV,
2165 VHOST_SCSI_WEIGHT, 0, true, NULL);
2166
2167 vhost_scsi_init_inflight(vs, NULL);
2168
2169 f->private_data = vs;
2170 return 0;
2171
2172 err_local_vqs:
2173 kfree(vs->vqs);
2174 err_vqs:
2175 kfree(vs->old_inflight);
2176 err_inflight:
2177 kvfree(vs);
2178 err_vs:
2179 return r;
2180 }
2181
vhost_scsi_release(struct inode * inode,struct file * f)2182 static int vhost_scsi_release(struct inode *inode, struct file *f)
2183 {
2184 struct vhost_scsi *vs = f->private_data;
2185 struct vhost_scsi_target t;
2186
2187 mutex_lock(&vs->dev.mutex);
2188 memcpy(t.vhost_wwpn, vs->vs_vhost_wwpn, sizeof(t.vhost_wwpn));
2189 mutex_unlock(&vs->dev.mutex);
2190 vhost_scsi_clear_endpoint(vs, &t);
2191 vhost_dev_stop(&vs->dev);
2192 vhost_dev_cleanup(&vs->dev);
2193 kfree(vs->dev.vqs);
2194 kfree(vs->vqs);
2195 kfree(vs->old_inflight);
2196 kvfree(vs);
2197 return 0;
2198 }
2199
2200 static long
vhost_scsi_ioctl(struct file * f,unsigned int ioctl,unsigned long arg)2201 vhost_scsi_ioctl(struct file *f,
2202 unsigned int ioctl,
2203 unsigned long arg)
2204 {
2205 struct vhost_scsi *vs = f->private_data;
2206 struct vhost_scsi_target backend;
2207 void __user *argp = (void __user *)arg;
2208 u64 __user *featurep = argp;
2209 u32 __user *eventsp = argp;
2210 u32 events_missed;
2211 u64 features;
2212 int r, abi_version = VHOST_SCSI_ABI_VERSION;
2213 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
2214
2215 switch (ioctl) {
2216 case VHOST_SCSI_SET_ENDPOINT:
2217 if (copy_from_user(&backend, argp, sizeof backend))
2218 return -EFAULT;
2219 if (backend.reserved != 0)
2220 return -EOPNOTSUPP;
2221
2222 return vhost_scsi_set_endpoint(vs, &backend);
2223 case VHOST_SCSI_CLEAR_ENDPOINT:
2224 if (copy_from_user(&backend, argp, sizeof backend))
2225 return -EFAULT;
2226 if (backend.reserved != 0)
2227 return -EOPNOTSUPP;
2228
2229 return vhost_scsi_clear_endpoint(vs, &backend);
2230 case VHOST_SCSI_GET_ABI_VERSION:
2231 if (copy_to_user(argp, &abi_version, sizeof abi_version))
2232 return -EFAULT;
2233 return 0;
2234 case VHOST_SCSI_SET_EVENTS_MISSED:
2235 if (get_user(events_missed, eventsp))
2236 return -EFAULT;
2237 mutex_lock(&vq->mutex);
2238 vs->vs_events_missed = events_missed;
2239 mutex_unlock(&vq->mutex);
2240 return 0;
2241 case VHOST_SCSI_GET_EVENTS_MISSED:
2242 mutex_lock(&vq->mutex);
2243 events_missed = vs->vs_events_missed;
2244 mutex_unlock(&vq->mutex);
2245 if (put_user(events_missed, eventsp))
2246 return -EFAULT;
2247 return 0;
2248 case VHOST_GET_FEATURES:
2249 features = VHOST_SCSI_FEATURES;
2250 if (copy_to_user(featurep, &features, sizeof features))
2251 return -EFAULT;
2252 return 0;
2253 case VHOST_SET_FEATURES:
2254 if (copy_from_user(&features, featurep, sizeof features))
2255 return -EFAULT;
2256 return vhost_scsi_set_features(vs, features);
2257 case VHOST_NEW_WORKER:
2258 case VHOST_FREE_WORKER:
2259 case VHOST_ATTACH_VRING_WORKER:
2260 case VHOST_GET_VRING_WORKER:
2261 mutex_lock(&vs->dev.mutex);
2262 r = vhost_worker_ioctl(&vs->dev, ioctl, argp);
2263 mutex_unlock(&vs->dev.mutex);
2264 return r;
2265 default:
2266 mutex_lock(&vs->dev.mutex);
2267 r = vhost_dev_ioctl(&vs->dev, ioctl, argp);
2268 /* TODO: flush backend after dev ioctl. */
2269 if (r == -ENOIOCTLCMD)
2270 r = vhost_vring_ioctl(&vs->dev, ioctl, argp);
2271 mutex_unlock(&vs->dev.mutex);
2272 return r;
2273 }
2274 }
2275
2276 static const struct file_operations vhost_scsi_fops = {
2277 .owner = THIS_MODULE,
2278 .release = vhost_scsi_release,
2279 .unlocked_ioctl = vhost_scsi_ioctl,
2280 .compat_ioctl = compat_ptr_ioctl,
2281 .open = vhost_scsi_open,
2282 .llseek = noop_llseek,
2283 };
2284
2285 static struct miscdevice vhost_scsi_misc = {
2286 MISC_DYNAMIC_MINOR,
2287 "vhost-scsi",
2288 &vhost_scsi_fops,
2289 };
2290
vhost_scsi_register(void)2291 static int __init vhost_scsi_register(void)
2292 {
2293 return misc_register(&vhost_scsi_misc);
2294 }
2295
vhost_scsi_deregister(void)2296 static void vhost_scsi_deregister(void)
2297 {
2298 misc_deregister(&vhost_scsi_misc);
2299 }
2300
vhost_scsi_dump_proto_id(struct vhost_scsi_tport * tport)2301 static char *vhost_scsi_dump_proto_id(struct vhost_scsi_tport *tport)
2302 {
2303 switch (tport->tport_proto_id) {
2304 case SCSI_PROTOCOL_SAS:
2305 return "SAS";
2306 case SCSI_PROTOCOL_FCP:
2307 return "FCP";
2308 case SCSI_PROTOCOL_ISCSI:
2309 return "iSCSI";
2310 default:
2311 break;
2312 }
2313
2314 return "Unknown";
2315 }
2316
2317 static void
vhost_scsi_do_plug(struct vhost_scsi_tpg * tpg,struct se_lun * lun,bool plug)2318 vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
2319 struct se_lun *lun, bool plug)
2320 {
2321
2322 struct vhost_scsi *vs = tpg->vhost_scsi;
2323 struct vhost_virtqueue *vq;
2324 u32 reason;
2325
2326 if (!vs)
2327 return;
2328
2329 if (plug)
2330 reason = VIRTIO_SCSI_EVT_RESET_RESCAN;
2331 else
2332 reason = VIRTIO_SCSI_EVT_RESET_REMOVED;
2333
2334 vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
2335 mutex_lock(&vq->mutex);
2336 /*
2337 * We can't queue events if the backend has been cleared, because
2338 * we could end up queueing an event after the flush.
2339 */
2340 if (!vhost_vq_get_backend(vq))
2341 goto unlock;
2342
2343 if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))
2344 vhost_scsi_send_evt(vs, vq, tpg, lun,
2345 VIRTIO_SCSI_T_TRANSPORT_RESET, reason);
2346 unlock:
2347 mutex_unlock(&vq->mutex);
2348 }
2349
vhost_scsi_hotplug(struct vhost_scsi_tpg * tpg,struct se_lun * lun)2350 static void vhost_scsi_hotplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
2351 {
2352 vhost_scsi_do_plug(tpg, lun, true);
2353 }
2354
vhost_scsi_hotunplug(struct vhost_scsi_tpg * tpg,struct se_lun * lun)2355 static void vhost_scsi_hotunplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
2356 {
2357 vhost_scsi_do_plug(tpg, lun, false);
2358 }
2359
vhost_scsi_port_link(struct se_portal_group * se_tpg,struct se_lun * lun)2360 static int vhost_scsi_port_link(struct se_portal_group *se_tpg,
2361 struct se_lun *lun)
2362 {
2363 struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2364 struct vhost_scsi_tpg, se_tpg);
2365
2366 mutex_lock(&tpg->tv_tpg_mutex);
2367 tpg->tv_tpg_port_count++;
2368 vhost_scsi_hotplug(tpg, lun);
2369 mutex_unlock(&tpg->tv_tpg_mutex);
2370
2371 return 0;
2372 }
2373
vhost_scsi_port_unlink(struct se_portal_group * se_tpg,struct se_lun * lun)2374 static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg,
2375 struct se_lun *lun)
2376 {
2377 struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2378 struct vhost_scsi_tpg, se_tpg);
2379
2380 mutex_lock(&tpg->tv_tpg_mutex);
2381 tpg->tv_tpg_port_count--;
2382 vhost_scsi_hotunplug(tpg, lun);
2383 mutex_unlock(&tpg->tv_tpg_mutex);
2384 }
2385
vhost_scsi_tpg_attrib_fabric_prot_type_store(struct config_item * item,const char * page,size_t count)2386 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store(
2387 struct config_item *item, const char *page, size_t count)
2388 {
2389 struct se_portal_group *se_tpg = attrib_to_tpg(item);
2390 struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2391 struct vhost_scsi_tpg, se_tpg);
2392 unsigned long val;
2393 int ret = kstrtoul(page, 0, &val);
2394
2395 if (ret) {
2396 pr_err("kstrtoul() returned %d for fabric_prot_type\n", ret);
2397 return ret;
2398 }
2399 if (val != 0 && val != 1 && val != 3) {
2400 pr_err("Invalid vhost_scsi fabric_prot_type: %lu\n", val);
2401 return -EINVAL;
2402 }
2403 tpg->tv_fabric_prot_type = val;
2404
2405 return count;
2406 }
2407
vhost_scsi_tpg_attrib_fabric_prot_type_show(struct config_item * item,char * page)2408 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_show(
2409 struct config_item *item, char *page)
2410 {
2411 struct se_portal_group *se_tpg = attrib_to_tpg(item);
2412 struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2413 struct vhost_scsi_tpg, se_tpg);
2414
2415 return sysfs_emit(page, "%d\n", tpg->tv_fabric_prot_type);
2416 }
2417
2418 CONFIGFS_ATTR(vhost_scsi_tpg_attrib_, fabric_prot_type);
2419
2420 static struct configfs_attribute *vhost_scsi_tpg_attrib_attrs[] = {
2421 &vhost_scsi_tpg_attrib_attr_fabric_prot_type,
2422 NULL,
2423 };
2424
vhost_scsi_make_nexus(struct vhost_scsi_tpg * tpg,const char * name)2425 static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg,
2426 const char *name)
2427 {
2428 struct vhost_scsi_nexus *tv_nexus;
2429
2430 mutex_lock(&tpg->tv_tpg_mutex);
2431 if (tpg->tpg_nexus) {
2432 mutex_unlock(&tpg->tv_tpg_mutex);
2433 pr_debug("tpg->tpg_nexus already exists\n");
2434 return -EEXIST;
2435 }
2436
2437 tv_nexus = kzalloc(sizeof(*tv_nexus), GFP_KERNEL);
2438 if (!tv_nexus) {
2439 mutex_unlock(&tpg->tv_tpg_mutex);
2440 pr_err("Unable to allocate struct vhost_scsi_nexus\n");
2441 return -ENOMEM;
2442 }
2443 /*
2444 * Since we are running in 'demo mode' this call with generate a
2445 * struct se_node_acl for the vhost_scsi struct se_portal_group with
2446 * the SCSI Initiator port name of the passed configfs group 'name'.
2447 */
2448 tv_nexus->tvn_se_sess = target_setup_session(&tpg->se_tpg, 0, 0,
2449 TARGET_PROT_DIN_PASS | TARGET_PROT_DOUT_PASS,
2450 (unsigned char *)name, tv_nexus, NULL);
2451 if (IS_ERR(tv_nexus->tvn_se_sess)) {
2452 mutex_unlock(&tpg->tv_tpg_mutex);
2453 kfree(tv_nexus);
2454 return -ENOMEM;
2455 }
2456 tpg->tpg_nexus = tv_nexus;
2457
2458 mutex_unlock(&tpg->tv_tpg_mutex);
2459 return 0;
2460 }
2461
vhost_scsi_drop_nexus(struct vhost_scsi_tpg * tpg)2462 static int vhost_scsi_drop_nexus(struct vhost_scsi_tpg *tpg)
2463 {
2464 struct se_session *se_sess;
2465 struct vhost_scsi_nexus *tv_nexus;
2466
2467 mutex_lock(&tpg->tv_tpg_mutex);
2468 tv_nexus = tpg->tpg_nexus;
2469 if (!tv_nexus) {
2470 mutex_unlock(&tpg->tv_tpg_mutex);
2471 return -ENODEV;
2472 }
2473
2474 se_sess = tv_nexus->tvn_se_sess;
2475 if (!se_sess) {
2476 mutex_unlock(&tpg->tv_tpg_mutex);
2477 return -ENODEV;
2478 }
2479
2480 if (tpg->tv_tpg_port_count != 0) {
2481 mutex_unlock(&tpg->tv_tpg_mutex);
2482 pr_err("Unable to remove TCM_vhost I_T Nexus with"
2483 " active TPG port count: %d\n",
2484 tpg->tv_tpg_port_count);
2485 return -EBUSY;
2486 }
2487
2488 if (tpg->tv_tpg_vhost_count != 0) {
2489 mutex_unlock(&tpg->tv_tpg_mutex);
2490 pr_err("Unable to remove TCM_vhost I_T Nexus with"
2491 " active TPG vhost count: %d\n",
2492 tpg->tv_tpg_vhost_count);
2493 return -EBUSY;
2494 }
2495
2496 pr_debug("TCM_vhost_ConfigFS: Removing I_T Nexus to emulated"
2497 " %s Initiator Port: %s\n", vhost_scsi_dump_proto_id(tpg->tport),
2498 tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2499
2500 /*
2501 * Release the SCSI I_T Nexus to the emulated vhost Target Port
2502 */
2503 target_remove_session(se_sess);
2504 tpg->tpg_nexus = NULL;
2505 mutex_unlock(&tpg->tv_tpg_mutex);
2506
2507 kfree(tv_nexus);
2508 return 0;
2509 }
2510
vhost_scsi_tpg_nexus_show(struct config_item * item,char * page)2511 static ssize_t vhost_scsi_tpg_nexus_show(struct config_item *item, char *page)
2512 {
2513 struct se_portal_group *se_tpg = to_tpg(item);
2514 struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2515 struct vhost_scsi_tpg, se_tpg);
2516 struct vhost_scsi_nexus *tv_nexus;
2517 ssize_t ret;
2518
2519 mutex_lock(&tpg->tv_tpg_mutex);
2520 tv_nexus = tpg->tpg_nexus;
2521 if (!tv_nexus) {
2522 mutex_unlock(&tpg->tv_tpg_mutex);
2523 return -ENODEV;
2524 }
2525 ret = sysfs_emit(page, "%s\n",
2526 tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2527 mutex_unlock(&tpg->tv_tpg_mutex);
2528
2529 return ret;
2530 }
2531
vhost_scsi_tpg_nexus_store(struct config_item * item,const char * page,size_t count)2532 static ssize_t vhost_scsi_tpg_nexus_store(struct config_item *item,
2533 const char *page, size_t count)
2534 {
2535 struct se_portal_group *se_tpg = to_tpg(item);
2536 struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2537 struct vhost_scsi_tpg, se_tpg);
2538 struct vhost_scsi_tport *tport_wwn = tpg->tport;
2539 unsigned char i_port[VHOST_SCSI_NAMELEN], *ptr, *port_ptr;
2540 int ret;
2541 /*
2542 * Shutdown the active I_T nexus if 'NULL' is passed..
2543 */
2544 if (!strncmp(page, "NULL", 4)) {
2545 ret = vhost_scsi_drop_nexus(tpg);
2546 return (!ret) ? count : ret;
2547 }
2548 /*
2549 * Otherwise make sure the passed virtual Initiator port WWN matches
2550 * the fabric protocol_id set in vhost_scsi_make_tport(), and call
2551 * vhost_scsi_make_nexus().
2552 */
2553 if (strlen(page) >= VHOST_SCSI_NAMELEN) {
2554 pr_err("Emulated NAA Sas Address: %s, exceeds"
2555 " max: %d\n", page, VHOST_SCSI_NAMELEN);
2556 return -EINVAL;
2557 }
2558 snprintf(&i_port[0], VHOST_SCSI_NAMELEN, "%s", page);
2559
2560 ptr = strstr(i_port, "naa.");
2561 if (ptr) {
2562 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_SAS) {
2563 pr_err("Passed SAS Initiator Port %s does not"
2564 " match target port protoid: %s\n", i_port,
2565 vhost_scsi_dump_proto_id(tport_wwn));
2566 return -EINVAL;
2567 }
2568 port_ptr = &i_port[0];
2569 goto check_newline;
2570 }
2571 ptr = strstr(i_port, "fc.");
2572 if (ptr) {
2573 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_FCP) {
2574 pr_err("Passed FCP Initiator Port %s does not"
2575 " match target port protoid: %s\n", i_port,
2576 vhost_scsi_dump_proto_id(tport_wwn));
2577 return -EINVAL;
2578 }
2579 port_ptr = &i_port[3]; /* Skip over "fc." */
2580 goto check_newline;
2581 }
2582 ptr = strstr(i_port, "iqn.");
2583 if (ptr) {
2584 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_ISCSI) {
2585 pr_err("Passed iSCSI Initiator Port %s does not"
2586 " match target port protoid: %s\n", i_port,
2587 vhost_scsi_dump_proto_id(tport_wwn));
2588 return -EINVAL;
2589 }
2590 port_ptr = &i_port[0];
2591 goto check_newline;
2592 }
2593 pr_err("Unable to locate prefix for emulated Initiator Port:"
2594 " %s\n", i_port);
2595 return -EINVAL;
2596 /*
2597 * Clear any trailing newline for the NAA WWN
2598 */
2599 check_newline:
2600 if (i_port[strlen(i_port)-1] == '\n')
2601 i_port[strlen(i_port)-1] = '\0';
2602
2603 ret = vhost_scsi_make_nexus(tpg, port_ptr);
2604 if (ret < 0)
2605 return ret;
2606
2607 return count;
2608 }
2609
2610 CONFIGFS_ATTR(vhost_scsi_tpg_, nexus);
2611
2612 static struct configfs_attribute *vhost_scsi_tpg_attrs[] = {
2613 &vhost_scsi_tpg_attr_nexus,
2614 NULL,
2615 };
2616
2617 static struct se_portal_group *
vhost_scsi_make_tpg(struct se_wwn * wwn,const char * name)2618 vhost_scsi_make_tpg(struct se_wwn *wwn, const char *name)
2619 {
2620 struct vhost_scsi_tport *tport = container_of(wwn,
2621 struct vhost_scsi_tport, tport_wwn);
2622
2623 struct vhost_scsi_tpg *tpg;
2624 u16 tpgt;
2625 int ret;
2626
2627 if (strstr(name, "tpgt_") != name)
2628 return ERR_PTR(-EINVAL);
2629 if (kstrtou16(name + 5, 10, &tpgt) || tpgt >= VHOST_SCSI_MAX_TARGET)
2630 return ERR_PTR(-EINVAL);
2631
2632 tpg = kzalloc(sizeof(*tpg), GFP_KERNEL);
2633 if (!tpg) {
2634 pr_err("Unable to allocate struct vhost_scsi_tpg");
2635 return ERR_PTR(-ENOMEM);
2636 }
2637 mutex_init(&tpg->tv_tpg_mutex);
2638 INIT_LIST_HEAD(&tpg->tv_tpg_list);
2639 tpg->tport = tport;
2640 tpg->tport_tpgt = tpgt;
2641
2642 ret = core_tpg_register(wwn, &tpg->se_tpg, tport->tport_proto_id);
2643 if (ret < 0) {
2644 kfree(tpg);
2645 return NULL;
2646 }
2647 mutex_lock(&vhost_scsi_mutex);
2648 list_add_tail(&tpg->tv_tpg_list, &vhost_scsi_list);
2649 mutex_unlock(&vhost_scsi_mutex);
2650
2651 return &tpg->se_tpg;
2652 }
2653
vhost_scsi_drop_tpg(struct se_portal_group * se_tpg)2654 static void vhost_scsi_drop_tpg(struct se_portal_group *se_tpg)
2655 {
2656 struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2657 struct vhost_scsi_tpg, se_tpg);
2658
2659 mutex_lock(&vhost_scsi_mutex);
2660 list_del(&tpg->tv_tpg_list);
2661 mutex_unlock(&vhost_scsi_mutex);
2662 /*
2663 * Release the virtual I_T Nexus for this vhost TPG
2664 */
2665 vhost_scsi_drop_nexus(tpg);
2666 /*
2667 * Deregister the se_tpg from TCM..
2668 */
2669 core_tpg_deregister(se_tpg);
2670 kfree(tpg);
2671 }
2672
2673 static struct se_wwn *
vhost_scsi_make_tport(struct target_fabric_configfs * tf,struct config_group * group,const char * name)2674 vhost_scsi_make_tport(struct target_fabric_configfs *tf,
2675 struct config_group *group,
2676 const char *name)
2677 {
2678 struct vhost_scsi_tport *tport;
2679 char *ptr;
2680 u64 wwpn = 0;
2681 int off = 0;
2682
2683 /* if (vhost_scsi_parse_wwn(name, &wwpn, 1) < 0)
2684 return ERR_PTR(-EINVAL); */
2685
2686 tport = kzalloc(sizeof(*tport), GFP_KERNEL);
2687 if (!tport) {
2688 pr_err("Unable to allocate struct vhost_scsi_tport");
2689 return ERR_PTR(-ENOMEM);
2690 }
2691 tport->tport_wwpn = wwpn;
2692 /*
2693 * Determine the emulated Protocol Identifier and Target Port Name
2694 * based on the incoming configfs directory name.
2695 */
2696 ptr = strstr(name, "naa.");
2697 if (ptr) {
2698 tport->tport_proto_id = SCSI_PROTOCOL_SAS;
2699 goto check_len;
2700 }
2701 ptr = strstr(name, "fc.");
2702 if (ptr) {
2703 tport->tport_proto_id = SCSI_PROTOCOL_FCP;
2704 off = 3; /* Skip over "fc." */
2705 goto check_len;
2706 }
2707 ptr = strstr(name, "iqn.");
2708 if (ptr) {
2709 tport->tport_proto_id = SCSI_PROTOCOL_ISCSI;
2710 goto check_len;
2711 }
2712
2713 pr_err("Unable to locate prefix for emulated Target Port:"
2714 " %s\n", name);
2715 kfree(tport);
2716 return ERR_PTR(-EINVAL);
2717
2718 check_len:
2719 if (strlen(name) >= VHOST_SCSI_NAMELEN) {
2720 pr_err("Emulated %s Address: %s, exceeds"
2721 " max: %d\n", name, vhost_scsi_dump_proto_id(tport),
2722 VHOST_SCSI_NAMELEN);
2723 kfree(tport);
2724 return ERR_PTR(-EINVAL);
2725 }
2726 snprintf(&tport->tport_name[0], VHOST_SCSI_NAMELEN, "%s", &name[off]);
2727
2728 pr_debug("TCM_VHost_ConfigFS: Allocated emulated Target"
2729 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport), name);
2730
2731 return &tport->tport_wwn;
2732 }
2733
vhost_scsi_drop_tport(struct se_wwn * wwn)2734 static void vhost_scsi_drop_tport(struct se_wwn *wwn)
2735 {
2736 struct vhost_scsi_tport *tport = container_of(wwn,
2737 struct vhost_scsi_tport, tport_wwn);
2738
2739 pr_debug("TCM_VHost_ConfigFS: Deallocating emulated Target"
2740 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport),
2741 tport->tport_name);
2742
2743 kfree(tport);
2744 }
2745
2746 static ssize_t
vhost_scsi_wwn_version_show(struct config_item * item,char * page)2747 vhost_scsi_wwn_version_show(struct config_item *item, char *page)
2748 {
2749 return sysfs_emit(page, "TCM_VHOST fabric module %s on %s/%s"
2750 "on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2751 utsname()->machine);
2752 }
2753
2754 CONFIGFS_ATTR_RO(vhost_scsi_wwn_, version);
2755
2756 static struct configfs_attribute *vhost_scsi_wwn_attrs[] = {
2757 &vhost_scsi_wwn_attr_version,
2758 NULL,
2759 };
2760
2761 static const struct target_core_fabric_ops vhost_scsi_ops = {
2762 .module = THIS_MODULE,
2763 .fabric_name = "vhost",
2764 .max_data_sg_nents = VHOST_SCSI_PREALLOC_SGLS,
2765 .tpg_get_wwn = vhost_scsi_get_fabric_wwn,
2766 .tpg_get_tag = vhost_scsi_get_tpgt,
2767 .tpg_check_demo_mode = vhost_scsi_check_true,
2768 .tpg_check_demo_mode_cache = vhost_scsi_check_true,
2769 .tpg_check_prot_fabric_only = vhost_scsi_check_prot_fabric_only,
2770 .release_cmd = vhost_scsi_release_cmd,
2771 .check_stop_free = vhost_scsi_check_stop_free,
2772 .sess_get_initiator_sid = NULL,
2773 .write_pending = vhost_scsi_write_pending,
2774 .queue_data_in = vhost_scsi_queue_data_in,
2775 .queue_status = vhost_scsi_queue_status,
2776 .queue_tm_rsp = vhost_scsi_queue_tm_rsp,
2777 .aborted_task = vhost_scsi_aborted_task,
2778 /*
2779 * Setup callers for generic logic in target_core_fabric_configfs.c
2780 */
2781 .fabric_make_wwn = vhost_scsi_make_tport,
2782 .fabric_drop_wwn = vhost_scsi_drop_tport,
2783 .fabric_make_tpg = vhost_scsi_make_tpg,
2784 .fabric_drop_tpg = vhost_scsi_drop_tpg,
2785 .fabric_post_link = vhost_scsi_port_link,
2786 .fabric_pre_unlink = vhost_scsi_port_unlink,
2787
2788 .tfc_wwn_attrs = vhost_scsi_wwn_attrs,
2789 .tfc_tpg_base_attrs = vhost_scsi_tpg_attrs,
2790 .tfc_tpg_attrib_attrs = vhost_scsi_tpg_attrib_attrs,
2791
2792 .default_submit_type = TARGET_QUEUE_SUBMIT,
2793 .direct_submit_supp = 1,
2794 };
2795
vhost_scsi_init(void)2796 static int __init vhost_scsi_init(void)
2797 {
2798 int ret = -ENOMEM;
2799
2800 pr_debug("TCM_VHOST fabric module %s on %s/%s"
2801 " on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2802 utsname()->machine);
2803
2804 ret = vhost_scsi_register();
2805 if (ret < 0)
2806 goto out;
2807
2808 ret = target_register_template(&vhost_scsi_ops);
2809 if (ret < 0)
2810 goto out_vhost_scsi_deregister;
2811
2812 return 0;
2813
2814 out_vhost_scsi_deregister:
2815 vhost_scsi_deregister();
2816 out:
2817 return ret;
2818 };
2819
vhost_scsi_exit(void)2820 static void vhost_scsi_exit(void)
2821 {
2822 target_unregister_template(&vhost_scsi_ops);
2823 vhost_scsi_deregister();
2824 };
2825
2826 MODULE_DESCRIPTION("VHOST_SCSI series fabric driver");
2827 MODULE_ALIAS("tcm_vhost");
2828 MODULE_LICENSE("GPL");
2829 module_init(vhost_scsi_init);
2830 module_exit(vhost_scsi_exit);
2831