xref: /kvmtool/builtin-run.c (revision 54f6802dee00520bd337034781e508b146bb933c)
1 #include <stdio.h>
2 #include <string.h>
3 #include <signal.h>
4 #include <unistd.h>
5 #include <stdlib.h>
6 #include <termios.h>
7 #include <sys/utsname.h>
8 #include <sys/types.h>
9 #include <sys/stat.h>
10 #include <ctype.h>
11 
12 /* user defined header files */
13 #include <linux/types.h>
14 #include <kvm/kvm.h>
15 #include <kvm/kvm-cpu.h>
16 #include <kvm/8250-serial.h>
17 #include <kvm/virtio-blk.h>
18 #include <kvm/virtio-net.h>
19 #include <kvm/virtio-console.h>
20 #include <kvm/virtio-rng.h>
21 #include <kvm/virtio-balloon.h>
22 #include <kvm/disk-image.h>
23 #include <kvm/util.h>
24 #include <kvm/pci.h>
25 #include <kvm/rtc.h>
26 #include <kvm/term.h>
27 #include <kvm/ioport.h>
28 #include <kvm/threadpool.h>
29 #include <kvm/barrier.h>
30 #include <kvm/symbol.h>
31 #include <kvm/virtio-9p.h>
32 #include <kvm/vesa.h>
33 #include <kvm/ioeventfd.h>
34 #include <kvm/i8042.h>
35 #include <kvm/vnc.h>
36 #include <kvm/sdl.h>
37 #include <kvm/framebuffer.h>
38 
39 /* header files for gitish interface  */
40 #include <kvm/builtin-run.h>
41 #include <kvm/parse-options.h>
42 #include <kvm/mutex.h>
43 
44 #define DEFAULT_KVM_DEV		"/dev/kvm"
45 #define DEFAULT_CONSOLE		"serial"
46 #define DEFAULT_NETWORK		"user"
47 #define DEFAULT_HOST_ADDR	"192.168.33.1"
48 #define DEFAULT_GUEST_ADDR	"192.168.33.15"
49 #define DEFAULT_GUEST_MAC	"02:15:15:15:15:15"
50 #define DEFAULT_HOST_MAC	"02:01:01:01:01:01"
51 #define DEFAULT_SCRIPT		"none"
52 
53 #define MB_SHIFT		(20)
54 #define MIN_RAM_SIZE_MB		(64ULL)
55 #define MIN_RAM_SIZE_BYTE	(MIN_RAM_SIZE_MB << MB_SHIFT)
56 
57 struct kvm *kvm;
58 struct kvm_cpu *kvm_cpus[KVM_NR_CPUS];
59 __thread struct kvm_cpu *current_kvm_cpu;
60 
61 static u64 ram_size;
62 static u8  image_count;
63 static int virtio_rng;
64 static const char *kernel_cmdline;
65 static const char *kernel_filename;
66 static const char *vmlinux_filename;
67 static const char *initrd_filename;
68 static const char *image_filename[MAX_DISK_IMAGES];
69 static const char *console;
70 static const char *kvm_dev;
71 static const char *network;
72 static const char *host_ip;
73 static const char *guest_ip;
74 static const char *guest_mac;
75 static const char *host_mac;
76 static const char *script;
77 static const char *guest_name;
78 static bool single_step;
79 static bool readonly_image[MAX_DISK_IMAGES];
80 static bool vnc;
81 static bool sdl;
82 static bool balloon;
83 extern bool ioport_debug;
84 extern int  active_console;
85 extern int  debug_iodelay;
86 
87 bool do_debug_print = false;
88 
89 static int nrcpus;
90 static int vidmode = -1;
91 
92 static const char * const run_usage[] = {
93 	"kvm run [<options>] [<kernel image>]",
94 	NULL
95 };
96 
97 static int img_name_parser(const struct option *opt, const char *arg, int unset)
98 {
99 	char *sep;
100 
101 	if (image_count >= MAX_DISK_IMAGES)
102 		die("Currently only 4 images are supported");
103 
104 	image_filename[image_count] = arg;
105 	sep = strstr(arg, ",");
106 	if (sep) {
107 		if (strcmp(sep + 1, "ro") == 0)
108 			readonly_image[image_count] = 1;
109 		*sep = 0;
110 	}
111 
112 	image_count++;
113 
114 	return 0;
115 }
116 
117 static int virtio_9p_rootdir_parser(const struct option *opt, const char *arg, int unset)
118 {
119 	char *tag_name;
120 	char tmp[PATH_MAX];
121 
122 	/*
123 	 * 9p dir can be of the form dirname,tag_name or
124 	 * just dirname. In the later case we use the
125 	 * default tag name
126 	 */
127 	tag_name = strstr(arg, ",");
128 	if (tag_name) {
129 		*tag_name = '\0';
130 		tag_name++;
131 	}
132 	if (realpath(arg, tmp)) {
133 		if (virtio_9p__init(kvm, tmp, tag_name) < 0)
134 			die("Unable to initialize virtio 9p");
135 	} else
136 		die("Failed resolving 9p path");
137 	return 0;
138 }
139 
140 
141 static const struct option options[] = {
142 	OPT_GROUP("Basic options:"),
143 	OPT_STRING('\0', "name", &guest_name, "guest name",
144 			"A name for the guest"),
145 	OPT_INTEGER('c', "cpus", &nrcpus, "Number of CPUs"),
146 	OPT_U64('m', "mem", &ram_size, "Virtual machine memory size in MiB."),
147 	OPT_CALLBACK('d', "disk", NULL, "image", "Disk image", img_name_parser),
148 	OPT_STRING('\0', "console", &console, "serial or virtio",
149 			"Console to use"),
150 	OPT_INCR('\0', "rng", &virtio_rng,
151 			"Enable virtio Random Number Generator"),
152 	OPT_STRING('\0', "kvm-dev", &kvm_dev, "kvm-dev", "KVM device file"),
153 	OPT_CALLBACK('\0', "virtio-9p", NULL, "dirname,tag_name",
154 		     "Enable 9p over virtio", virtio_9p_rootdir_parser),
155 	OPT_BOOLEAN('\0', "balloon", &balloon, "Enable virtio balloon"),
156 	OPT_BOOLEAN('\0', "vnc", &vnc, "Enable VNC framebuffer"),
157 	OPT_BOOLEAN('\0', "sdl", &sdl, "Enable SDL framebuffer"),
158 
159 	OPT_GROUP("Kernel options:"),
160 	OPT_STRING('k', "kernel", &kernel_filename, "kernel",
161 			"Kernel to boot in virtual machine"),
162 	OPT_STRING('i', "initrd", &initrd_filename, "initrd",
163 			"Initial RAM disk image"),
164 	OPT_STRING('p', "params", &kernel_cmdline, "params",
165 			"Kernel command line arguments"),
166 
167 	OPT_GROUP("Networking options:"),
168 	OPT_STRING('n', "network", &network, "user, tap, none",
169 			"Network to use"),
170 	OPT_STRING('\0', "host-ip", &host_ip, "a.b.c.d",
171 			"Assign this address to the host side networking"),
172 	OPT_STRING('\0', "guest-ip", &guest_ip, "a.b.c.d",
173 			"Assign this address to the guest side networking"),
174 	OPT_STRING('\0', "host-mac", &host_mac, "aa:bb:cc:dd:ee:ff",
175 			"Assign this address to the host side NIC"),
176 	OPT_STRING('\0', "guest-mac", &guest_mac, "aa:bb:cc:dd:ee:ff",
177 			"Assign this address to the guest side NIC"),
178 	OPT_STRING('\0', "tapscript", &script, "Script path",
179 			 "Assign a script to process created tap device"),
180 
181 	OPT_GROUP("BIOS options:"),
182 	OPT_INTEGER('\0', "vidmode", &vidmode,
183 		    "Video mode"),
184 
185 	OPT_GROUP("Debug options:"),
186 	OPT_BOOLEAN('\0', "debug", &do_debug_print,
187 			"Enable debug messages"),
188 	OPT_BOOLEAN('\0', "debug-single-step", &single_step,
189 			"Enable single stepping"),
190 	OPT_BOOLEAN('\0', "debug-ioport", &ioport_debug,
191 			"Enable ioport debugging"),
192 	OPT_INTEGER('\0', "debug-iodelay", &debug_iodelay,
193 			"Delay IO by millisecond"),
194 	OPT_END()
195 };
196 
197 /*
198  * Serialize debug printout so that the output of multiple vcpus does not
199  * get mixed up:
200  */
201 static int printout_done;
202 
203 static void handle_sigusr1(int sig)
204 {
205 	struct kvm_cpu *cpu = current_kvm_cpu;
206 
207 	if (!cpu)
208 		return;
209 
210 	printf("\n #\n # vCPU #%ld's dump:\n #\n", cpu->cpu_id);
211 	kvm_cpu__show_registers(cpu);
212 	kvm_cpu__show_code(cpu);
213 	kvm_cpu__show_page_tables(cpu);
214 	fflush(stdout);
215 	printout_done = 1;
216 	mb();
217 }
218 
219 /* Pause/resume the guest using SIGUSR2 */
220 static int is_paused;
221 
222 static void handle_sigusr2(int sig)
223 {
224 	if (is_paused)
225 		kvm__continue();
226 	else
227 		kvm__pause();
228 
229 	is_paused = !is_paused;
230 	pr_info("Guest %s\n", is_paused ? "paused" : "resumed");
231 }
232 
233 static void handle_sigquit(int sig)
234 {
235 	int i;
236 
237 	for (i = 0; i < nrcpus; i++) {
238 		struct kvm_cpu *cpu = kvm_cpus[i];
239 
240 		if (!cpu)
241 			continue;
242 
243 		printout_done = 0;
244 		pthread_kill(cpu->thread, SIGUSR1);
245 		/*
246 		 * Wait for the vCPU to dump state before signalling
247 		 * the next thread. Since this is debug code it does
248 		 * not matter that we are burning CPU time a bit:
249 		 */
250 		while (!printout_done)
251 			mb();
252 	}
253 
254 	serial8250__inject_sysrq(kvm);
255 }
256 
257 static void handle_sigalrm(int sig)
258 {
259 	serial8250__inject_interrupt(kvm);
260 	virtio_console__inject_interrupt(kvm);
261 }
262 
263 static void *kvm_cpu_thread(void *arg)
264 {
265 	current_kvm_cpu		= arg;
266 
267 	if (kvm_cpu__start(current_kvm_cpu))
268 		goto panic_kvm;
269 
270 	kvm_cpu__delete(current_kvm_cpu);
271 
272 	return (void *) (intptr_t) 0;
273 
274 panic_kvm:
275 	fprintf(stderr, "KVM exit reason: %u (\"%s\")\n",
276 		current_kvm_cpu->kvm_run->exit_reason,
277 		kvm_exit_reasons[current_kvm_cpu->kvm_run->exit_reason]);
278 	if (current_kvm_cpu->kvm_run->exit_reason == KVM_EXIT_UNKNOWN)
279 		fprintf(stderr, "KVM exit code: 0x%Lu\n",
280 			current_kvm_cpu->kvm_run->hw.hardware_exit_reason);
281 
282 	kvm_cpu__show_registers(current_kvm_cpu);
283 	kvm_cpu__show_code(current_kvm_cpu);
284 	kvm_cpu__show_page_tables(current_kvm_cpu);
285 
286 	kvm_cpu__delete(current_kvm_cpu);
287 
288 	return (void *) (intptr_t) 1;
289 }
290 
291 static char kernel[PATH_MAX];
292 
293 static const char *host_kernels[] = {
294 	"/boot/vmlinuz",
295 	"/boot/bzImage",
296 	NULL
297 };
298 
299 static const char *default_kernels[] = {
300 	"./bzImage",
301 	"../../arch/x86/boot/bzImage",
302 	NULL
303 };
304 
305 static const char *default_vmlinux[] = {
306 	"../../../vmlinux",
307 	"../../vmlinux",
308 	NULL
309 };
310 
311 static void kernel_usage_with_options(void)
312 {
313 	const char **k;
314 	struct utsname uts;
315 
316 	fprintf(stderr, "Fatal: could not find default kernel image in:\n");
317 	k = &default_kernels[0];
318 	while (*k) {
319 		fprintf(stderr, "\t%s\n", *k);
320 		k++;
321 	}
322 
323 	if (uname(&uts) < 0)
324 		return;
325 
326 	k = &host_kernels[0];
327 	while (*k) {
328 		if (snprintf(kernel, PATH_MAX, "%s-%s", *k, uts.release) < 0)
329 			return;
330 		fprintf(stderr, "\t%s\n", kernel);
331 		k++;
332 	}
333 	fprintf(stderr, "\nPlease see 'kvm run --help' for more options.\n\n");
334 }
335 
336 static u64 host_ram_size(void)
337 {
338 	long page_size;
339 	long nr_pages;
340 
341 	nr_pages	= sysconf(_SC_PHYS_PAGES);
342 	if (nr_pages < 0) {
343 		pr_warning("sysconf(_SC_PHYS_PAGES) failed");
344 		return 0;
345 	}
346 
347 	page_size	= sysconf(_SC_PAGE_SIZE);
348 	if (page_size < 0) {
349 		pr_warning("sysconf(_SC_PAGE_SIZE) failed");
350 		return 0;
351 	}
352 
353 	return (nr_pages * page_size) >> MB_SHIFT;
354 }
355 
356 /*
357  * If user didn't specify how much memory it wants to allocate for the guest,
358  * avoid filling the whole host RAM.
359  */
360 #define RAM_SIZE_RATIO		0.8
361 
362 static u64 get_ram_size(int nr_cpus)
363 {
364 	u64 available;
365 	u64 ram_size;
366 
367 	ram_size	= 64 * (nr_cpus + 3);
368 
369 	available	= host_ram_size() * RAM_SIZE_RATIO;
370 	if (!available)
371 		available = MIN_RAM_SIZE_MB;
372 
373 	if (ram_size > available)
374 		ram_size	= available;
375 
376 	return ram_size;
377 }
378 
379 static const char *find_kernel(void)
380 {
381 	const char **k;
382 	struct stat st;
383 	struct utsname uts;
384 
385 	k = &default_kernels[0];
386 	while (*k) {
387 		if (stat(*k, &st) < 0 || !S_ISREG(st.st_mode)) {
388 			k++;
389 			continue;
390 		}
391 		strncpy(kernel, *k, PATH_MAX);
392 		return kernel;
393 	}
394 
395 	if (uname(&uts) < 0)
396 		return NULL;
397 
398 	k = &host_kernels[0];
399 	while (*k) {
400 		if (snprintf(kernel, PATH_MAX, "%s-%s", *k, uts.release) < 0)
401 			return NULL;
402 
403 		if (stat(kernel, &st) < 0 || !S_ISREG(st.st_mode)) {
404 			k++;
405 			continue;
406 		}
407 		return kernel;
408 
409 	}
410 	return NULL;
411 }
412 
413 static const char *find_vmlinux(void)
414 {
415 	const char **vmlinux;
416 
417 	vmlinux = &default_vmlinux[0];
418 	while (*vmlinux) {
419 		struct stat st;
420 
421 		if (stat(*vmlinux, &st) < 0 || !S_ISREG(st.st_mode)) {
422 			vmlinux++;
423 			continue;
424 		}
425 		return *vmlinux;
426 	}
427 	return NULL;
428 }
429 
430 static int root_device(char *dev, long *part)
431 {
432 	struct stat st;
433 
434 	if (stat("/", &st) < 0)
435 		return -1;
436 
437 	*part = minor(st.st_dev);
438 
439 	sprintf(dev, "/dev/block/%u:0", major(st.st_dev));
440 	if (access(dev, R_OK) < 0)
441 		return -1;
442 
443 	return 0;
444 }
445 
446 static char *host_image(char *cmd_line, size_t size)
447 {
448 	char *t;
449 	char device[PATH_MAX];
450 	long part = 0;
451 
452 	t = malloc(PATH_MAX);
453 	if (!t)
454 		return NULL;
455 
456 	/* check for the root file system */
457 	if (root_device(device, &part) < 0) {
458 		free(t);
459 		return NULL;
460 	}
461 	strncpy(t, device, PATH_MAX);
462 	if (!strstr(cmd_line, "root=")) {
463 		char tmp[PATH_MAX];
464 		snprintf(tmp, sizeof(tmp), "root=/dev/vda%ld rw ", part);
465 		strlcat(cmd_line, tmp, size);
466 	}
467 	return t;
468 }
469 
470 void kvm_run_help(void)
471 {
472 	usage_with_options(run_usage, options);
473 }
474 
475 int kvm_cmd_run(int argc, const char **argv, const char *prefix)
476 {
477 	struct virtio_net_parameters net_params;
478 	static char real_cmdline[2048], default_name[20];
479 	struct framebuffer *fb = NULL;
480 	unsigned int nr_online_cpus;
481 	int exit_code = 0;
482 	int max_cpus, recommended_cpus;
483 	char *hi;
484 	int i;
485 	void *ret;
486 
487 	signal(SIGALRM, handle_sigalrm);
488 	signal(SIGQUIT, handle_sigquit);
489 	signal(SIGUSR1, handle_sigusr1);
490 	signal(SIGUSR2, handle_sigusr2);
491 
492 	nr_online_cpus = sysconf(_SC_NPROCESSORS_ONLN);
493 
494 	while (argc != 0) {
495 		argc = parse_options(argc, argv, options, run_usage,
496 				PARSE_OPT_STOP_AT_NON_OPTION);
497 		if (argc != 0) {
498 			if (kernel_filename) {
499 				fprintf(stderr, "Cannot handle parameter: "
500 						"%s\n", argv[0]);
501 				usage_with_options(run_usage, options);
502 				return EINVAL;
503 			}
504 			/* first unhandled parameter is treated as a kernel
505 			   image
506 			 */
507 			kernel_filename = argv[0];
508 			argv++;
509 			argc--;
510 		}
511 
512 	}
513 
514 	if (!kernel_filename)
515 		kernel_filename = find_kernel();
516 
517 	if (!kernel_filename) {
518 		kernel_usage_with_options();
519 		return EINVAL;
520 	}
521 
522 	vmlinux_filename = find_vmlinux();
523 
524 	if (nrcpus == 0)
525 		nrcpus = nr_online_cpus;
526 	else if (nrcpus < 1 || nrcpus > KVM_NR_CPUS)
527 		die("Number of CPUs %d is out of [1;%d] range", nrcpus, KVM_NR_CPUS);
528 
529 	if (!ram_size)
530 		ram_size	= get_ram_size(nrcpus);
531 
532 	if (ram_size < MIN_RAM_SIZE_MB)
533 		die("Not enough memory specified: %lluMB (min %lluMB)", ram_size, MIN_RAM_SIZE_MB);
534 
535 	if (ram_size > host_ram_size())
536 		pr_warning("Guest memory size %lluMB exceeds host physical RAM size %lluMB", ram_size, host_ram_size());
537 
538 	ram_size <<= MB_SHIFT;
539 
540 	if (!kvm_dev)
541 		kvm_dev = DEFAULT_KVM_DEV;
542 
543 	if (!console)
544 		console = DEFAULT_CONSOLE;
545 
546 	if (!strncmp(console, "virtio", 6))
547 		active_console  = CONSOLE_VIRTIO;
548 	else
549 		active_console  = CONSOLE_8250;
550 
551 	if (!host_ip)
552 		host_ip = DEFAULT_HOST_ADDR;
553 
554 	if (!guest_ip)
555 		guest_ip = DEFAULT_GUEST_ADDR;
556 
557 	if (!guest_mac)
558 		guest_mac = DEFAULT_GUEST_MAC;
559 
560 	if (!host_mac)
561 		host_mac = DEFAULT_HOST_MAC;
562 
563 	if (!script)
564 		script = DEFAULT_SCRIPT;
565 
566 	symbol__init(vmlinux_filename);
567 
568 	term_init();
569 
570 	if (!guest_name) {
571 		sprintf(default_name, "guest-%u", getpid());
572 		guest_name = default_name;
573 	}
574 
575 	kvm = kvm__init(kvm_dev, ram_size, guest_name);
576 
577 	kvm->single_step = single_step;
578 
579 	ioeventfd__init();
580 
581 	max_cpus = kvm__max_cpus(kvm);
582 	recommended_cpus = kvm__recommended_cpus(kvm);
583 
584 	if (nrcpus > max_cpus) {
585 		printf("  # Limit the number of CPUs to %d\n", max_cpus);
586 		kvm->nrcpus	= max_cpus;
587 	} else if (nrcpus > recommended_cpus) {
588 		printf("  # Warning: The maximum recommended amount of VCPUs"
589 			" is %d\n", recommended_cpus);
590 	}
591 
592 	kvm->nrcpus = nrcpus;
593 
594 	/*
595 	 * vidmode should be either specified
596 	 * either set by default
597 	 */
598 	if (vnc || sdl) {
599 		if (vidmode == -1)
600 			vidmode = 0x312;
601 	} else
602 		vidmode = 0;
603 
604 	memset(real_cmdline, 0, sizeof(real_cmdline));
605 	strcpy(real_cmdline, "notsc noapic noacpi pci=conf1 reboot=k panic=1");
606 	if (vnc || sdl) {
607 		strcat(real_cmdline, " video=vesafb console=tty0");
608 	} else
609 		strcat(real_cmdline, " console=ttyS0 earlyprintk=serial");
610 	strcat(real_cmdline, " ");
611 	if (kernel_cmdline)
612 		strlcat(real_cmdline, kernel_cmdline, sizeof(real_cmdline));
613 
614 	hi = NULL;
615 	if (!image_filename[0]) {
616 		hi = host_image(real_cmdline, sizeof(real_cmdline));
617 		if (hi) {
618 			image_filename[0] = hi;
619 			readonly_image[0] = true;
620 			image_count++;
621 		}
622 	}
623 
624 	if (!strstr(real_cmdline, "root="))
625 		strlcat(real_cmdline, " root=/dev/vda rw ", sizeof(real_cmdline));
626 
627 	if (image_count) {
628 		kvm->nr_disks = image_count;
629 		kvm->disks    = disk_image__open_all(image_filename, readonly_image, image_count);
630 		if (!kvm->disks)
631 			die("Unable to load all disk images.");
632 
633 		virtio_blk__init_all(kvm);
634 	}
635 
636 	free(hi);
637 
638 	printf("  # kvm run -k %s -m %Lu -c %d --name %s\n", kernel_filename, ram_size / 1024 / 1024, nrcpus, guest_name);
639 
640 	if (!kvm__load_kernel(kvm, kernel_filename, initrd_filename,
641 				real_cmdline, vidmode))
642 		die("unable to load kernel %s", kernel_filename);
643 
644 	kvm->vmlinux		= vmlinux_filename;
645 
646 	ioport__setup_legacy();
647 
648 	rtc__init();
649 
650 	serial8250__init(kvm);
651 
652 	pci__init();
653 
654 	if (active_console == CONSOLE_VIRTIO)
655 		virtio_console__init(kvm);
656 
657 	if (virtio_rng)
658 		while (virtio_rng--)
659 			virtio_rng__init(kvm);
660 
661 	if (balloon)
662 		virtio_bln__init(kvm);
663 
664 	if (!network)
665 		network = DEFAULT_NETWORK;
666 
667 	if (strncmp(network, "none", 4)) {
668 		net_params.guest_ip = guest_ip;
669 		net_params.host_ip = host_ip;
670 		net_params.kvm = kvm;
671 		net_params.script = script;
672 		sscanf(guest_mac, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx",
673 			net_params.guest_mac,
674 			net_params.guest_mac+1,
675 			net_params.guest_mac+2,
676 			net_params.guest_mac+3,
677 			net_params.guest_mac+4,
678 			net_params.guest_mac+5);
679 		sscanf(host_mac, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx",
680 			net_params.host_mac,
681 			net_params.host_mac+1,
682 			net_params.host_mac+2,
683 			net_params.host_mac+3,
684 			net_params.host_mac+4,
685 			net_params.host_mac+5);
686 
687 		if (!strncmp(network, "user", 4))
688 			net_params.mode = NET_MODE_USER;
689 		else if (!strncmp(network, "tap", 3))
690 			net_params.mode = NET_MODE_TAP;
691 		else
692 			die("Unkown network mode %s, please use -network user, tap, none", network);
693 		virtio_net__init(&net_params);
694 	}
695 
696 	kvm__start_timer(kvm);
697 
698 	kvm__setup_bios(kvm);
699 
700 	for (i = 0; i < nrcpus; i++) {
701 		kvm_cpus[i] = kvm_cpu__init(kvm, i);
702 		if (!kvm_cpus[i])
703 			die("unable to initialize KVM VCPU");
704 	}
705 
706 	kvm__init_ram(kvm);
707 
708 	kbd__init(kvm);
709 
710 	if (vnc || sdl)
711 		fb = vesa__init(kvm);
712 
713 	if (vnc) {
714 		if (fb)
715 			vnc__init(fb);
716 	}
717 
718 	if (sdl) {
719 		if (fb)
720 			sdl__init(fb);
721 	}
722 
723 	fb__start();
724 
725 	thread_pool__init(nr_online_cpus);
726 	ioeventfd__start();
727 
728 	for (i = 0; i < nrcpus; i++) {
729 		if (pthread_create(&kvm_cpus[i]->thread, NULL, kvm_cpu_thread, kvm_cpus[i]) != 0)
730 			die("unable to create KVM VCPU thread");
731 	}
732 
733 	/* Only VCPU #0 is going to exit by itself when shutting down */
734 	if (pthread_join(kvm_cpus[0]->thread, &ret) != 0)
735 		exit_code = 1;
736 
737 	for (i = 1; i < nrcpus; i++) {
738 		pthread_kill(kvm_cpus[i]->thread, SIGKVMEXIT);
739 		if (pthread_join(kvm_cpus[i]->thread, &ret) != 0)
740 			die("pthread_join");
741 
742 		if (ret != NULL)
743 			exit_code = 1;
744 	}
745 
746 	fb__stop();
747 
748 	virtio_blk__delete_all(kvm);
749 	virtio_rng__delete_all(kvm);
750 
751 	disk_image__close_all(kvm->disks, image_count);
752 	kvm__delete(kvm);
753 
754 	if (!exit_code)
755 		printf("\n  # KVM session ended normally.\n");
756 
757 	return exit_code;
758 }
759