xref: /qemu/contrib/vhost-user-scsi/vhost-user-scsi.c (revision 3e7bd3ad47b86d0ec4d85c0ae839bdd9f49dad46) 
1 /*
2  * vhost-user-scsi sample application
3  *
4  * Copyright (c) 2016 Nutanix Inc. All rights reserved.
5  *
6  * Author:
7  *  Felipe Franciosi <felipe@nutanix.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2 only.
10  * See the COPYING file in the top-level directory.
11  */
12 
13 #include "qemu/osdep.h"
14 #include "contrib/libvhost-user/libvhost-user.h"
15 #include "standard-headers/linux/virtio_scsi.h"
16 #include "iscsi/iscsi.h"
17 #include "iscsi/scsi-lowlevel.h"
18 
19 #include <glib.h>
20 
21 /* #define VUS_DEBUG 1 */
22 
23 /** Log helpers **/
24 
25 #define PPRE                                                          \
26     struct timespec ts;                                               \
27     char   timebuf[64];                                               \
28     struct tm tm;                                                     \
29     (void)clock_gettime(CLOCK_REALTIME, &ts);                         \
30     (void)strftime(timebuf, 64, "%Y%m%d %T", gmtime_r(&ts.tv_sec, &tm))
31 
32 #define PEXT(lvl, msg, ...) do {                                      \
33     PPRE;                                                             \
34     fprintf(stderr, "%s.%06ld " lvl ": %s:%s():%d: " msg "\n",        \
35             timebuf, ts.tv_nsec / 1000,                               \
36             __FILE__, __func__, __LINE__, ## __VA_ARGS__);            \
37 } while (0)
38 
39 #define PNOR(lvl, msg, ...) do {                                      \
40     PPRE;                                                             \
41     fprintf(stderr, "%s.%06ld " lvl ": " msg "\n",                    \
42             timebuf, ts.tv_nsec / 1000, ## __VA_ARGS__);              \
43 } while (0)
44 
45 #ifdef VUS_DEBUG
46 #define PDBG(msg, ...) PEXT("DBG", msg, ## __VA_ARGS__)
47 #define PERR(msg, ...) PEXT("ERR", msg, ## __VA_ARGS__)
48 #define PLOG(msg, ...) PEXT("LOG", msg, ## __VA_ARGS__)
49 #else
50 #define PDBG(msg, ...) { }
51 #define PERR(msg, ...) PNOR("ERR", msg, ## __VA_ARGS__)
52 #define PLOG(msg, ...) PNOR("LOG", msg, ## __VA_ARGS__)
53 #endif
54 
55 /** vhost-user-scsi specific definitions **/
56 
57 #define VUS_ISCSI_INITIATOR "iqn.2016-11.com.nutanix:vhost-user-scsi"
58 
59 typedef struct VusIscsiLun {
60     struct iscsi_context *iscsi_ctx;
61     int iscsi_lun;
62 } VusIscsiLun;
63 
64 typedef struct VusDev {
65     VuDev vu_dev;
66     int server_sock;
67     GMainLoop *loop;
68     GTree *fdmap;   /* fd -> gsource context id */
69     VusIscsiLun lun;
70 } VusDev;
71 
72 /** glib event loop integration for libvhost-user and misc callbacks **/
73 
74 QEMU_BUILD_BUG_ON((int)G_IO_IN != (int)VU_WATCH_IN);
75 QEMU_BUILD_BUG_ON((int)G_IO_OUT != (int)VU_WATCH_OUT);
76 QEMU_BUILD_BUG_ON((int)G_IO_PRI != (int)VU_WATCH_PRI);
77 QEMU_BUILD_BUG_ON((int)G_IO_ERR != (int)VU_WATCH_ERR);
78 QEMU_BUILD_BUG_ON((int)G_IO_HUP != (int)VU_WATCH_HUP);
79 
80 typedef struct vus_gsrc {
81     GSource parent;
82     VusDev *vdev_scsi;
83     GPollFD gfd;
84 } vus_gsrc_t;
85 
86 static gint vus_fdmap_compare(gconstpointer a, gconstpointer b)
87 {
88     return (b > a) - (b < a);
89 }
90 
91 static gboolean vus_gsrc_prepare(GSource *src, gint *timeout)
92 {
93     assert(timeout);
94 
95     *timeout = -1;
96     return FALSE;
97 }
98 
99 static gboolean vus_gsrc_check(GSource *src)
100 {
101     vus_gsrc_t *vus_src = (vus_gsrc_t *)src;
102 
103     assert(vus_src);
104 
105     return vus_src->gfd.revents & vus_src->gfd.events;
106 }
107 
108 static gboolean vus_gsrc_dispatch(GSource *src, GSourceFunc cb, gpointer data)
109 {
110     VusDev *vdev_scsi;
111     vus_gsrc_t *vus_src = (vus_gsrc_t *)src;
112 
113     assert(vus_src);
114 
115     vdev_scsi = vus_src->vdev_scsi;
116 
117     assert(vdev_scsi);
118 
119     ((vu_watch_cb)cb)(&vdev_scsi->vu_dev, vus_src->gfd.revents, data);
120 
121     return G_SOURCE_CONTINUE;
122 }
123 
124 static GSourceFuncs vus_gsrc_funcs = {
125     vus_gsrc_prepare,
126     vus_gsrc_check,
127     vus_gsrc_dispatch,
128     NULL
129 };
130 
131 static void vus_gsrc_new(VusDev *vdev_scsi, int fd, GIOCondition cond,
132                          vu_watch_cb vu_cb, GSourceFunc gsrc_cb, gpointer data)
133 {
134     GSource *vus_gsrc;
135     vus_gsrc_t *vus_src;
136     guint id;
137 
138     assert(vdev_scsi);
139     assert(fd >= 0);
140     assert(vu_cb || gsrc_cb);
141     assert(!(vu_cb && gsrc_cb));
142 
143     vus_gsrc = g_source_new(&vus_gsrc_funcs, sizeof(vus_gsrc_t));
144     g_source_set_callback(vus_gsrc, (GSourceFunc) vu_cb, data, NULL);
145     vus_src = (vus_gsrc_t *)vus_gsrc;
146 
147     vus_src->vdev_scsi = vdev_scsi;
148     vus_src->gfd.fd = fd;
149     vus_src->gfd.events = cond;
150 
151     g_source_add_poll(vus_gsrc, &vus_src->gfd);
152     g_source_set_callback(vus_gsrc, gsrc_cb, data, NULL);
153     id = g_source_attach(vus_gsrc, NULL);
154     assert(id);
155     g_source_unref(vus_gsrc);
156 
157     g_tree_insert(vdev_scsi->fdmap, (gpointer)(uintptr_t)fd,
158                                     (gpointer)(uintptr_t)id);
159 }
160 
161 /** libiscsi integration **/
162 
163 typedef struct virtio_scsi_cmd_req VirtIOSCSICmdReq;
164 typedef struct virtio_scsi_cmd_resp VirtIOSCSICmdResp;
165 
166 static int vus_iscsi_add_lun(VusIscsiLun *lun, char *iscsi_uri)
167 {
168     struct iscsi_url *iscsi_url;
169     struct iscsi_context *iscsi_ctx;
170     int ret = 0;
171 
172     assert(lun);
173     assert(iscsi_uri);
174     assert(!lun->iscsi_ctx);
175 
176     iscsi_ctx = iscsi_create_context(VUS_ISCSI_INITIATOR);
177     if (!iscsi_ctx) {
178         PERR("Unable to create iSCSI context");
179         return -1;
180     }
181 
182     iscsi_url = iscsi_parse_full_url(iscsi_ctx, iscsi_uri);
183     if (!iscsi_url) {
184         PERR("Unable to parse iSCSI URL: %s", iscsi_get_error(iscsi_ctx));
185         goto fail;
186     }
187 
188     iscsi_set_session_type(iscsi_ctx, ISCSI_SESSION_NORMAL);
189     iscsi_set_header_digest(iscsi_ctx, ISCSI_HEADER_DIGEST_NONE_CRC32C);
190     if (iscsi_full_connect_sync(iscsi_ctx, iscsi_url->portal, iscsi_url->lun)) {
191         PERR("Unable to login to iSCSI portal: %s", iscsi_get_error(iscsi_ctx));
192         goto fail;
193     }
194 
195     lun->iscsi_ctx = iscsi_ctx;
196     lun->iscsi_lun = iscsi_url->lun;
197 
198     PDBG("Context %p created for lun 0: %s", iscsi_ctx, iscsi_uri);
199 
200 out:
201     if (iscsi_url) {
202         iscsi_destroy_url(iscsi_url);
203     }
204     return ret;
205 
206 fail:
207     (void)iscsi_destroy_context(iscsi_ctx);
208     ret = -1;
209     goto out;
210 }
211 
212 static struct scsi_task *scsi_task_new(int cdb_len, uint8_t *cdb, int dir,
213                                        int xfer_len)
214 {
215     struct scsi_task *task;
216 
217     assert(cdb_len > 0);
218     assert(cdb);
219 
220     task = g_new0(struct scsi_task, 1);
221     memcpy(task->cdb, cdb, cdb_len);
222     task->cdb_size = cdb_len;
223     task->xfer_dir = dir;
224     task->expxferlen = xfer_len;
225 
226     return task;
227 }
228 
229 static int get_cdb_len(uint8_t *cdb)
230 {
231     assert(cdb);
232 
233     switch (cdb[0] >> 5) {
234     case 0: return 6;
235     case 1: /* fall through */
236     case 2: return 10;
237     case 4: return 16;
238     case 5: return 12;
239     }
240     PERR("Unable to determine cdb len (0x%02hhX)", cdb[0] >> 5);
241     return -1;
242 }
243 
244 static int handle_cmd_sync(struct iscsi_context *ctx,
245                            VirtIOSCSICmdReq *req,
246                            struct iovec *out, unsigned int out_len,
247                            VirtIOSCSICmdResp *rsp,
248                            struct iovec *in, unsigned int in_len)
249 {
250     struct scsi_task *task;
251     uint32_t dir;
252     uint32_t len;
253     int cdb_len;
254     int i;
255 
256     assert(ctx);
257     assert(req);
258     assert(rsp);
259 
260     if (!(!req->lun[1] && req->lun[2] == 0x40 && !req->lun[3])) {
261         /* Ignore anything different than target=0, lun=0 */
262         PDBG("Ignoring unconnected lun (0x%hhX, 0x%hhX)",
263              req->lun[1], req->lun[3]);
264         rsp->status = SCSI_STATUS_CHECK_CONDITION;
265         memset(rsp->sense, 0, sizeof(rsp->sense));
266         rsp->sense_len = 18;
267         rsp->sense[0] = 0x70;
268         rsp->sense[2] = SCSI_SENSE_ILLEGAL_REQUEST;
269         rsp->sense[7] = 10;
270         rsp->sense[12] = 0x24;
271 
272         return 0;
273     }
274 
275     cdb_len = get_cdb_len(req->cdb);
276     if (cdb_len == -1) {
277         return -1;
278     }
279 
280     len = 0;
281     if (!out_len && !in_len) {
282         dir = SCSI_XFER_NONE;
283     } else if (out_len) {
284         dir = SCSI_XFER_WRITE;
285         for (i = 0; i < out_len; i++) {
286             len += out[i].iov_len;
287         }
288     } else {
289         dir = SCSI_XFER_READ;
290         for (i = 0; i < in_len; i++) {
291             len += in[i].iov_len;
292         }
293     }
294 
295     task = scsi_task_new(cdb_len, req->cdb, dir, len);
296 
297     if (dir == SCSI_XFER_WRITE) {
298         task->iovector_out.iov = (struct scsi_iovec *)out;
299         task->iovector_out.niov = out_len;
300     } else if (dir == SCSI_XFER_READ) {
301         task->iovector_in.iov = (struct scsi_iovec *)in;
302         task->iovector_in.niov = in_len;
303     }
304 
305     PDBG("Sending iscsi cmd (cdb_len=%d, dir=%d, task=%p)",
306          cdb_len, dir, task);
307     if (!iscsi_scsi_command_sync(ctx, 0, task, NULL)) {
308         PERR("Error serving SCSI command");
309         g_free(task);
310         return -1;
311     }
312 
313     memset(rsp, 0, sizeof(*rsp));
314 
315     rsp->status = task->status;
316     rsp->resid  = task->residual;
317 
318     if (task->status == SCSI_STATUS_CHECK_CONDITION) {
319         rsp->response = VIRTIO_SCSI_S_FAILURE;
320         rsp->sense_len = task->datain.size - 2;
321         memcpy(rsp->sense, &task->datain.data[2], rsp->sense_len);
322     }
323 
324     g_free(task);
325 
326     PDBG("Filled in rsp: status=%hhX, resid=%u, response=%hhX, sense_len=%u",
327          rsp->status, rsp->resid, rsp->response, rsp->sense_len);
328 
329     return 0;
330 }
331 
332 /** libvhost-user callbacks **/
333 
334 static void vus_panic_cb(VuDev *vu_dev, const char *buf)
335 {
336     VusDev *vdev_scsi;
337 
338     assert(vu_dev);
339 
340     vdev_scsi = container_of(vu_dev, VusDev, vu_dev);
341     if (buf) {
342         PERR("vu_panic: %s", buf);
343     }
344 
345     g_main_loop_quit(vdev_scsi->loop);
346 }
347 
348 static void vus_add_watch_cb(VuDev *vu_dev, int fd, int vu_evt, vu_watch_cb cb,
349                              void *pvt)
350 {
351     VusDev *vdev_scsi;
352     guint id;
353 
354     assert(vu_dev);
355     assert(fd >= 0);
356     assert(cb);
357 
358     vdev_scsi = container_of(vu_dev, VusDev, vu_dev);
359     id = (guint)(uintptr_t)g_tree_lookup(vdev_scsi->fdmap,
360                                          (gpointer)(uintptr_t)fd);
361     if (id) {
362         GSource *vus_src = g_main_context_find_source_by_id(NULL, id);
363         assert(vus_src);
364         g_source_destroy(vus_src);
365         (void)g_tree_remove(vdev_scsi->fdmap, (gpointer)(uintptr_t)fd);
366     }
367 
368     vus_gsrc_new(vdev_scsi, fd, vu_evt, cb, NULL, pvt);
369 }
370 
371 static void vus_del_watch_cb(VuDev *vu_dev, int fd)
372 {
373     VusDev *vdev_scsi;
374     guint id;
375 
376     assert(vu_dev);
377     assert(fd >= 0);
378 
379     vdev_scsi = container_of(vu_dev, VusDev, vu_dev);
380     id = (guint)(uintptr_t)g_tree_lookup(vdev_scsi->fdmap,
381                                          (gpointer)(uintptr_t)fd);
382     if (id) {
383         GSource *vus_src = g_main_context_find_source_by_id(NULL, id);
384         assert(vus_src);
385         g_source_destroy(vus_src);
386         (void)g_tree_remove(vdev_scsi->fdmap, (gpointer)(uintptr_t)fd);
387     }
388 }
389 
390 static void vus_proc_req(VuDev *vu_dev, int idx)
391 {
392     VusDev *vdev_scsi;
393     VuVirtq *vq;
394 
395     assert(vu_dev);
396 
397     vdev_scsi = container_of(vu_dev, VusDev, vu_dev);
398     if (idx < 0 || idx >= VHOST_MAX_NR_VIRTQUEUE) {
399         PERR("VQ Index out of range: %d", idx);
400         vus_panic_cb(vu_dev, NULL);
401         return;
402     }
403 
404     vq = vu_get_queue(vu_dev, idx);
405     if (!vq) {
406         PERR("Error fetching VQ (dev=%p, idx=%d)", vu_dev, idx);
407         vus_panic_cb(vu_dev, NULL);
408         return;
409     }
410 
411     PDBG("Got kicked on vq[%d]@%p", idx, vq);
412 
413     while (1) {
414         VuVirtqElement *elem;
415         VirtIOSCSICmdReq *req;
416         VirtIOSCSICmdResp *rsp;
417 
418         elem = vu_queue_pop(vu_dev, vq, sizeof(VuVirtqElement));
419         if (!elem) {
420             PDBG("No more elements pending on vq[%d]@%p", idx, vq);
421             break;
422         }
423         PDBG("Popped elem@%p", elem);
424 
425         assert(!(elem->out_num > 1 && elem->in_num > 1));
426         assert(elem->out_num > 0 && elem->in_num > 0);
427 
428         if (elem->out_sg[0].iov_len < sizeof(VirtIOSCSICmdReq)) {
429             PERR("Invalid virtio-scsi req header");
430             vus_panic_cb(vu_dev, NULL);
431             break;
432         }
433         req = (VirtIOSCSICmdReq *)elem->out_sg[0].iov_base;
434 
435         if (elem->in_sg[0].iov_len < sizeof(VirtIOSCSICmdResp)) {
436             PERR("Invalid virtio-scsi rsp header");
437             vus_panic_cb(vu_dev, NULL);
438             break;
439         }
440         rsp = (VirtIOSCSICmdResp *)elem->in_sg[0].iov_base;
441 
442         if (handle_cmd_sync(vdev_scsi->lun.iscsi_ctx,
443                             req, &elem->out_sg[1], elem->out_num - 1,
444                             rsp, &elem->in_sg[1], elem->in_num - 1) != 0) {
445             vus_panic_cb(vu_dev, NULL);
446             break;
447         }
448 
449         vu_queue_push(vu_dev, vq, elem, 0);
450         vu_queue_notify(vu_dev, vq);
451 
452         free(elem);
453     }
454 }
455 
456 static void vus_queue_set_started(VuDev *vu_dev, int idx, bool started)
457 {
458     VuVirtq *vq;
459 
460     assert(vu_dev);
461 
462     if (idx < 0 || idx >= VHOST_MAX_NR_VIRTQUEUE) {
463         PERR("VQ Index out of range: %d", idx);
464         vus_panic_cb(vu_dev, NULL);
465         return;
466     }
467 
468     vq = vu_get_queue(vu_dev, idx);
469 
470     if (idx == 0 || idx == 1) {
471         PDBG("queue %d unimplemented", idx);
472     } else {
473         vu_set_queue_handler(vu_dev, vq, started ? vus_proc_req : NULL);
474     }
475 }
476 
477 static const VuDevIface vus_iface = {
478     .queue_set_started = vus_queue_set_started,
479 };
480 
481 static gboolean vus_vhost_cb(gpointer data)
482 {
483     VuDev *vu_dev = (VuDev *)data;
484 
485     assert(vu_dev);
486 
487     if (!vu_dispatch(vu_dev) != 0) {
488         PERR("Error processing vhost message");
489         vus_panic_cb(vu_dev, NULL);
490         return G_SOURCE_REMOVE;
491     }
492 
493     return G_SOURCE_CONTINUE;
494 }
495 
496 /** misc helpers **/
497 
498 static int unix_sock_new(char *unix_fn)
499 {
500     int sock;
501     struct sockaddr_un un;
502     size_t len;
503 
504     assert(unix_fn);
505 
506     sock = socket(AF_UNIX, SOCK_STREAM, 0);
507     if (sock <= 0) {
508         perror("socket");
509         return -1;
510     }
511 
512     un.sun_family = AF_UNIX;
513     (void)snprintf(un.sun_path, sizeof(un.sun_path), "%s", unix_fn);
514     len = sizeof(un.sun_family) + strlen(un.sun_path);
515 
516     (void)unlink(unix_fn);
517     if (bind(sock, (struct sockaddr *)&un, len) < 0) {
518         perror("bind");
519         goto fail;
520     }
521 
522     if (listen(sock, 1) < 0) {
523         perror("listen");
524         goto fail;
525     }
526 
527     return sock;
528 
529 fail:
530     (void)close(sock);
531 
532     return -1;
533 }
534 
535 /** vhost-user-scsi **/
536 
537 static void vdev_scsi_free(VusDev *vdev_scsi)
538 {
539     if (vdev_scsi->server_sock >= 0) {
540         close(vdev_scsi->server_sock);
541     }
542     g_main_loop_unref(vdev_scsi->loop);
543     g_tree_destroy(vdev_scsi->fdmap);
544     g_free(vdev_scsi);
545 }
546 
547 static VusDev *vdev_scsi_new(int server_sock)
548 {
549     VusDev *vdev_scsi;
550 
551     vdev_scsi = g_new0(VusDev, 1);
552     vdev_scsi->server_sock = server_sock;
553     vdev_scsi->loop = g_main_loop_new(NULL, FALSE);
554     vdev_scsi->fdmap = g_tree_new(vus_fdmap_compare);
555 
556     return vdev_scsi;
557 }
558 
559 static int vdev_scsi_run(VusDev *vdev_scsi)
560 {
561     int cli_sock;
562     int ret = 0;
563 
564     assert(vdev_scsi);
565     assert(vdev_scsi->server_sock >= 0);
566     assert(vdev_scsi->loop);
567 
568     cli_sock = accept(vdev_scsi->server_sock, NULL, NULL);
569     if (cli_sock < 0) {
570         perror("accept");
571         return -1;
572     }
573 
574     vu_init(&vdev_scsi->vu_dev,
575             cli_sock,
576             vus_panic_cb,
577             vus_add_watch_cb,
578             vus_del_watch_cb,
579             &vus_iface);
580 
581     vus_gsrc_new(vdev_scsi, cli_sock, G_IO_IN, NULL, vus_vhost_cb,
582                  &vdev_scsi->vu_dev);
583 
584     g_main_loop_run(vdev_scsi->loop);
585 
586     vu_deinit(&vdev_scsi->vu_dev);
587 
588     return ret;
589 }
590 
591 int main(int argc, char **argv)
592 {
593     VusDev *vdev_scsi = NULL;
594     char *unix_fn = NULL;
595     char *iscsi_uri = NULL;
596     int sock, opt, err = EXIT_SUCCESS;
597 
598     while ((opt = getopt(argc, argv, "u:i:")) != -1) {
599         switch (opt) {
600         case 'h':
601             goto help;
602         case 'u':
603             unix_fn = g_strdup(optarg);
604             break;
605         case 'i':
606             iscsi_uri = g_strdup(optarg);
607             break;
608         default:
609             goto help;
610         }
611     }
612     if (!unix_fn || !iscsi_uri) {
613         goto help;
614     }
615 
616     sock = unix_sock_new(unix_fn);
617     if (sock < 0) {
618         goto err;
619     }
620     vdev_scsi = vdev_scsi_new(sock);
621 
622     if (vus_iscsi_add_lun(&vdev_scsi->lun, iscsi_uri) != 0) {
623         goto err;
624     }
625 
626     if (vdev_scsi_run(vdev_scsi) != 0) {
627         goto err;
628     }
629 
630 out:
631     if (vdev_scsi) {
632         vdev_scsi_free(vdev_scsi);
633         unlink(unix_fn);
634     }
635     g_free(unix_fn);
636     g_free(iscsi_uri);
637 
638     return err;
639 
640 err:
641     err = EXIT_FAILURE;
642     goto out;
643 
644 help:
645     fprintf(stderr, "Usage: %s [ -u unix_sock_path -i iscsi_uri ] | [ -h ]\n",
646             argv[0]);
647     fprintf(stderr, "          -u path to unix socket\n");
648     fprintf(stderr, "          -i iscsi uri for lun 0\n");
649     fprintf(stderr, "          -h print help and quit\n");
650 
651     goto err;
652 }
653