xref: /kvmtool/builtin-run.c (revision 65182f37d251ff530c1eec3b58c636426a4e0e28)
1f967c427SPrasad Joshi #include <stdio.h>
2f967c427SPrasad Joshi #include <string.h>
3f967c427SPrasad Joshi #include <signal.h>
4f967c427SPrasad Joshi #include <stdint.h>
5f967c427SPrasad Joshi #include <unistd.h>
6f967c427SPrasad Joshi #include <inttypes.h>
7f967c427SPrasad Joshi #include <stdlib.h>
8f967c427SPrasad Joshi #include <termios.h>
98329f30bSPekka Enberg #include <sys/utsname.h>
108329f30bSPekka Enberg #include <sys/types.h>
118329f30bSPekka Enberg #include <sys/stat.h>
12f967c427SPrasad Joshi 
13f967c427SPrasad Joshi /* user defined header files */
14f967c427SPrasad Joshi #include <linux/types.h>
15f967c427SPrasad Joshi #include <kvm/kvm.h>
165c3d55faSPekka Enberg #include <kvm/kvm-cpu.h>
17f967c427SPrasad Joshi #include <kvm/8250-serial.h>
18f967c427SPrasad Joshi #include <kvm/virtio-blk.h>
194f56d42cSAsias He #include <kvm/virtio-net.h>
20f967c427SPrasad Joshi #include <kvm/virtio-console.h>
21f967c427SPrasad Joshi #include <kvm/disk-image.h>
22f967c427SPrasad Joshi #include <kvm/util.h>
23f967c427SPrasad Joshi #include <kvm/pci.h>
24f967c427SPrasad Joshi #include <kvm/term.h>
25ac38f433SPekka Enberg #include <kvm/ioport.h>
26f967c427SPrasad Joshi 
27f967c427SPrasad Joshi /* header files for gitish interface  */
28f967c427SPrasad Joshi #include <kvm/kvm-run.h>
29f967c427SPrasad Joshi #include <kvm/parse-options.h>
30f967c427SPrasad Joshi 
31f967c427SPrasad Joshi #define DEFAULT_KVM_DEV		"/dev/kvm"
329aa4a0ebSAsias He #define DEFAULT_CONSOLE		"serial"
333b02f580SSasha Levin #define DEFAULT_NETWORK		"virtio"
344d67c820SSasha Levin #define DEFAULT_HOST_ADDR	"192.168.33.2"
35a4e724ddSSasha Levin #define DEFAULT_GUEST_MAC	"00:11:22:33:44:55"
3673b7d038SAmos Kong #define DEFAULT_SCRIPT		"none"
37f967c427SPrasad Joshi 
38f967c427SPrasad Joshi #define MB_SHIFT		(20)
39a2a002f9SIngo Molnar #define MIN_RAM_SIZE_MB		(64ULL)
40f967c427SPrasad Joshi #define MIN_RAM_SIZE_BYTE	(MIN_RAM_SIZE_MB << MB_SHIFT)
41f967c427SPrasad Joshi 
42d77a9efaSCyrill Gorcunov #define KVM_NR_CPUS		(255)
435ee154d1SPekka Enberg 
44f967c427SPrasad Joshi static struct kvm *kvm;
45d77a9efaSCyrill Gorcunov static struct kvm_cpu *kvm_cpus[KVM_NR_CPUS];
46d77a9efaSCyrill Gorcunov static __thread struct kvm_cpu *current_kvm_cpu;
47f967c427SPrasad Joshi 
48f967c427SPrasad Joshi static void handle_sigquit(int sig)
49f967c427SPrasad Joshi {
505f457e0bSPekka Enberg 	serial8250__inject_sysrq(kvm);
51f967c427SPrasad Joshi }
52f967c427SPrasad Joshi 
535ee154d1SPekka Enberg static void handle_sigalrm(int sig)
545ee154d1SPekka Enberg {
555ee154d1SPekka Enberg 	serial8250__inject_interrupt(kvm);
565ee154d1SPekka Enberg 	virtio_console__inject_interrupt(kvm);
575ee154d1SPekka Enberg }
585ee154d1SPekka Enberg 
59ab261214SCyrill Gorcunov static u64 ram_size = MIN_RAM_SIZE_MB;
60f967c427SPrasad Joshi static const char *kernel_cmdline;
61f967c427SPrasad Joshi static const char *kernel_filename;
62f967c427SPrasad Joshi static const char *initrd_filename;
63f967c427SPrasad Joshi static const char *image_filename;
649aa4a0ebSAsias He static const char *console;
65f967c427SPrasad Joshi static const char *kvm_dev;
664f56d42cSAsias He static const char *network;
674d67c820SSasha Levin static const char *host_ip_addr;
68a4e724ddSSasha Levin static const char *guest_mac;
6973b7d038SAmos Kong static const char *script;
700ea58e5bSPekka Enberg static bool single_step;
719ac38fe1SSasha Levin static bool readonly_image;
72f967c427SPrasad Joshi extern bool ioport_debug;
73f967c427SPrasad Joshi extern int  active_console;
74f967c427SPrasad Joshi 
75d77a9efaSCyrill Gorcunov static int nrcpus = 1;
76d77a9efaSCyrill Gorcunov 
77f967c427SPrasad Joshi static const char * const run_usage[] = {
788329f30bSPekka Enberg 	"kvm run [<options>] [<kernel image>]",
79f967c427SPrasad Joshi 	NULL
80f967c427SPrasad Joshi };
81f967c427SPrasad Joshi 
82f967c427SPrasad Joshi static const struct option options[] = {
835e3af62dSSasha Levin 	OPT_GROUP("Basic options:"),
845e3af62dSSasha Levin 	OPT_INTEGER('\0', "cpus", &nrcpus, "Number of CPUs"),
85f967c427SPrasad Joshi 	OPT_U64('m', "mem", &ram_size, "Virtual machine memory size in MiB."),
86f967c427SPrasad Joshi 	OPT_STRING('i', "image", &image_filename, "image", "Disk image"),
879ac38fe1SSasha Levin 	OPT_BOOLEAN('\0', "readonly", &readonly_image,
889ac38fe1SSasha Levin 			"Don't write changes back to disk image"),
895e3af62dSSasha Levin 	OPT_STRING('c', "console", &console, "serial or virtio",
905e3af62dSSasha Levin 			"Console to use"),
915e3af62dSSasha Levin 
925e3af62dSSasha Levin 	OPT_GROUP("Kernel options:"),
935e3af62dSSasha Levin 	OPT_STRING('k', "kernel", &kernel_filename, "kernel",
945e3af62dSSasha Levin 			"Kernel to boot in virtual machine"),
955e3af62dSSasha Levin 	OPT_STRING('r', "initrd", &initrd_filename, "initrd",
965e3af62dSSasha Levin 			"Initial RAM disk image"),
975e3af62dSSasha Levin 	OPT_STRING('p', "params", &kernel_cmdline, "params",
985e3af62dSSasha Levin 			"Kernel command line arguments"),
995e3af62dSSasha Levin 
1004d67c820SSasha Levin 	OPT_GROUP("Networking options:"),
1014d67c820SSasha Levin 	OPT_STRING('n', "network", &network, "virtio",
1024d67c820SSasha Levin 			"Network to use"),
1034d67c820SSasha Levin 	OPT_STRING('\0', "host-ip-addr", &host_ip_addr, "a.b.c.d",
1044d67c820SSasha Levin 			"Assign this address to the host side networking"),
105a4e724ddSSasha Levin 	OPT_STRING('\0', "guest-mac", &guest_mac, "aa:bb:cc:dd:ee:ff",
106a4e724ddSSasha Levin 			"Assign this address to the guest side NIC"),
10773b7d038SAmos Kong 	OPT_STRING('\0', "tapscript", &script, "Script path",
10873b7d038SAmos Kong 			 "Assign a script to process created tap device"),
1095e3af62dSSasha Levin 	OPT_GROUP("Debug options:"),
110f967c427SPrasad Joshi 	OPT_STRING('d', "kvm-dev", &kvm_dev, "kvm-dev", "KVM device file"),
111f967c427SPrasad Joshi 	OPT_BOOLEAN('s', "single-step", &single_step,
112f967c427SPrasad Joshi 			"Enable single stepping"),
113f967c427SPrasad Joshi 	OPT_BOOLEAN('g', "ioport-debug", &ioport_debug,
114f967c427SPrasad Joshi 			"Enable ioport debugging"),
115f967c427SPrasad Joshi 	OPT_END()
116f967c427SPrasad Joshi };
117f967c427SPrasad Joshi 
1185ee154d1SPekka Enberg static void *kvm_cpu_thread(void *arg)
1195ee154d1SPekka Enberg {
120d77a9efaSCyrill Gorcunov 	current_kvm_cpu		= arg;
1215ee154d1SPekka Enberg 
122d77a9efaSCyrill Gorcunov 	if (kvm_cpu__start(current_kvm_cpu))
1235ee154d1SPekka Enberg 		goto panic_kvm;
1245ee154d1SPekka Enberg 
125d77a9efaSCyrill Gorcunov 	kvm_cpu__delete(current_kvm_cpu);
1265ee154d1SPekka Enberg 
1275ee154d1SPekka Enberg 	return (void *) (intptr_t) 0;
1285ee154d1SPekka Enberg 
1295ee154d1SPekka Enberg panic_kvm:
1305ee154d1SPekka Enberg 	fprintf(stderr, "KVM exit reason: %" PRIu32 " (\"%s\")\n",
131d77a9efaSCyrill Gorcunov 		current_kvm_cpu->kvm_run->exit_reason,
132d77a9efaSCyrill Gorcunov 		kvm_exit_reasons[current_kvm_cpu->kvm_run->exit_reason]);
133d77a9efaSCyrill Gorcunov 	if (current_kvm_cpu->kvm_run->exit_reason == KVM_EXIT_UNKNOWN)
1345ee154d1SPekka Enberg 		fprintf(stderr, "KVM exit code: 0x%Lu\n",
135d77a9efaSCyrill Gorcunov 			current_kvm_cpu->kvm_run->hw.hardware_exit_reason);
1365ee154d1SPekka Enberg 	disk_image__close(kvm->disk_image);
137d77a9efaSCyrill Gorcunov 	kvm_cpu__show_registers(current_kvm_cpu);
138d77a9efaSCyrill Gorcunov 	kvm_cpu__show_code(current_kvm_cpu);
139d77a9efaSCyrill Gorcunov 	kvm_cpu__show_page_tables(current_kvm_cpu);
1405ee154d1SPekka Enberg 
141d77a9efaSCyrill Gorcunov 	kvm_cpu__delete(current_kvm_cpu);
1425ee154d1SPekka Enberg 
1435ee154d1SPekka Enberg 	return (void *) (intptr_t) 1;
1445ee154d1SPekka Enberg }
1455ee154d1SPekka Enberg 
146e08c0896SPrasad Joshi static char kernel[PATH_MAX];
147e08c0896SPrasad Joshi const char *host_kernels[] = {
148e08c0896SPrasad Joshi 	"/boot/vmlinuz",
149e08c0896SPrasad Joshi 	"/boot/bzImage",
150e08c0896SPrasad Joshi 	NULL
151e08c0896SPrasad Joshi };
152*65182f37SPrasad Joshi const char *default_kernels[] = {
153e08c0896SPrasad Joshi 	"./bzImage",
154e08c0896SPrasad Joshi 	"../../arch/x86/boot/bzImage",
155e08c0896SPrasad Joshi 	NULL
156e08c0896SPrasad Joshi };
1578329f30bSPekka Enberg 
158e08c0896SPrasad Joshi static void kernel_usage_with_options(void)
1598329f30bSPekka Enberg {
160e08c0896SPrasad Joshi 	const char **k;
1618329f30bSPekka Enberg 	struct utsname uts;
162e08c0896SPrasad Joshi 
163e08c0896SPrasad Joshi 	fprintf(stderr, "Fatal: could not find default kernel image in:\n");
164*65182f37SPrasad Joshi 	k = &default_kernels[0];
165e08c0896SPrasad Joshi 	while (*k) {
166e08c0896SPrasad Joshi 		fprintf(stderr, "\t%s\n", *k);
167e08c0896SPrasad Joshi 		k++;
168e08c0896SPrasad Joshi 	}
169e08c0896SPrasad Joshi 
170e08c0896SPrasad Joshi 	if (uname(&uts) < 0)
171e08c0896SPrasad Joshi 		return;
172e08c0896SPrasad Joshi 
173e08c0896SPrasad Joshi 	k = &host_kernels[0];
174e08c0896SPrasad Joshi 	while (*k) {
175e08c0896SPrasad Joshi 		if (snprintf(kernel, PATH_MAX, "%s-%s", *k, uts.release) < 0)
176e08c0896SPrasad Joshi 			return;
177e08c0896SPrasad Joshi 		fprintf(stderr, "\t%s\n", kernel);
178e08c0896SPrasad Joshi 		k++;
179e08c0896SPrasad Joshi 	}
180e08c0896SPrasad Joshi 	usage_with_options(run_usage, options);
181e08c0896SPrasad Joshi }
182e08c0896SPrasad Joshi 
183e08c0896SPrasad Joshi static const char *find_kernel(void)
184e08c0896SPrasad Joshi {
185e08c0896SPrasad Joshi 	const char **k;
1868329f30bSPekka Enberg 	struct stat st;
187e08c0896SPrasad Joshi 	struct utsname uts;
188e08c0896SPrasad Joshi 
189*65182f37SPrasad Joshi 	k = &default_kernels[0];
190e08c0896SPrasad Joshi 	while (*k) {
191e08c0896SPrasad Joshi 		if (stat(*k, &st) < 0 || !S_ISREG(st.st_mode)) {
192e08c0896SPrasad Joshi 			k++;
193e08c0896SPrasad Joshi 			continue;
194e08c0896SPrasad Joshi 		}
195e08c0896SPrasad Joshi 		strncpy(kernel, *k, PATH_MAX);
196e08c0896SPrasad Joshi 		return kernel;
197e08c0896SPrasad Joshi 	}
1988329f30bSPekka Enberg 
1998329f30bSPekka Enberg 	if (uname(&uts) < 0)
2008329f30bSPekka Enberg 		return NULL;
2018329f30bSPekka Enberg 
202e08c0896SPrasad Joshi 	k = &host_kernels[0];
203e08c0896SPrasad Joshi 	while (*k) {
204e08c0896SPrasad Joshi 		if (snprintf(kernel, PATH_MAX, "%s-%s", *k, uts.release) < 0)
2058329f30bSPekka Enberg 			return NULL;
2068329f30bSPekka Enberg 
207e08c0896SPrasad Joshi 		if (stat(kernel, &st) < 0 || !S_ISREG(st.st_mode)) {
208e08c0896SPrasad Joshi 			k++;
209e08c0896SPrasad Joshi 			continue;
210e08c0896SPrasad Joshi 		}
211e08c0896SPrasad Joshi 		return kernel;
2128329f30bSPekka Enberg 
213e08c0896SPrasad Joshi 	}
2148329f30bSPekka Enberg 	return NULL;
2158329f30bSPekka Enberg }
2168329f30bSPekka Enberg 
217f967c427SPrasad Joshi int kvm_cmd_run(int argc, const char **argv, const char *prefix)
218f967c427SPrasad Joshi {
219f967c427SPrasad Joshi 	static char real_cmdline[2048];
2205ee154d1SPekka Enberg 	int exit_code = 0;
2215ee154d1SPekka Enberg 	int i;
222bdfcfca6SSasha Levin 	struct virtio_net_parameters net_params;
223f967c427SPrasad Joshi 
2245ee154d1SPekka Enberg 	signal(SIGALRM, handle_sigalrm);
225f967c427SPrasad Joshi 	signal(SIGQUIT, handle_sigquit);
226f967c427SPrasad Joshi 
227f967c427SPrasad Joshi 	while (argc != 0) {
228f967c427SPrasad Joshi 		argc = parse_options(argc, argv, options, run_usage,
229f967c427SPrasad Joshi 				PARSE_OPT_STOP_AT_NON_OPTION);
230f967c427SPrasad Joshi 		if (argc != 0) {
231f967c427SPrasad Joshi 			if (kernel_filename) {
232f967c427SPrasad Joshi 				fprintf(stderr, "Cannot handle parameter: "
233f967c427SPrasad Joshi 						"%s\n", argv[0]);
234f967c427SPrasad Joshi 				usage_with_options(run_usage, options);
235f967c427SPrasad Joshi 				return EINVAL;
236f967c427SPrasad Joshi 			}
237f967c427SPrasad Joshi 			/* first unhandled parameter is treated as a kernel
238f967c427SPrasad Joshi 			   image
239f967c427SPrasad Joshi 			 */
240f967c427SPrasad Joshi 			kernel_filename = argv[0];
241f967c427SPrasad Joshi 			argv++;
242f967c427SPrasad Joshi 			argc--;
243f967c427SPrasad Joshi 		}
244f967c427SPrasad Joshi 
245f967c427SPrasad Joshi 	}
246f967c427SPrasad Joshi 
24745bc2f02SPekka Enberg 	if (!kernel_filename)
248e08c0896SPrasad Joshi 		kernel_filename = find_kernel();
2498329f30bSPekka Enberg 
2508329f30bSPekka Enberg 	if (!kernel_filename) {
251e08c0896SPrasad Joshi 		kernel_usage_with_options();
2528329f30bSPekka Enberg 		return EINVAL;
2538329f30bSPekka Enberg 	}
2548329f30bSPekka Enberg 
255d77a9efaSCyrill Gorcunov 	if (nrcpus < 1 || nrcpus > KVM_NR_CPUS)
256d77a9efaSCyrill Gorcunov 		die("Number of CPUs %d is out of [1;%d] range", nrcpus, KVM_NR_CPUS);
257d77a9efaSCyrill Gorcunov 
258d77a9efaSCyrill Gorcunov 	/* FIXME: Remove as only SMP gets fully supported */
259d77a9efaSCyrill Gorcunov 	if (nrcpus > 1)
260d77a9efaSCyrill Gorcunov 		warning("Limiting CPUs to 1, true SMP is not yet implemented");
261d77a9efaSCyrill Gorcunov 
262a2a002f9SIngo Molnar 	if (ram_size < MIN_RAM_SIZE_MB)
263a2a002f9SIngo Molnar 		die("Not enough memory specified: %lluMB (min %lluMB)", ram_size, MIN_RAM_SIZE_MB);
264a2a002f9SIngo Molnar 
265f967c427SPrasad Joshi 	ram_size <<= MB_SHIFT;
266f967c427SPrasad Joshi 
267f967c427SPrasad Joshi 	if (!kvm_dev)
268f967c427SPrasad Joshi 		kvm_dev = DEFAULT_KVM_DEV;
269f967c427SPrasad Joshi 
2709aa4a0ebSAsias He 	if (!console)
2719aa4a0ebSAsias He 		console = DEFAULT_CONSOLE;
2729aa4a0ebSAsias He 
2739aa4a0ebSAsias He 	if (!strncmp(console, "virtio", 6))
274f967c427SPrasad Joshi 		active_console  = CONSOLE_VIRTIO;
2759aa4a0ebSAsias He 	else
2769aa4a0ebSAsias He 		active_console  = CONSOLE_8250;
277f967c427SPrasad Joshi 
2784d67c820SSasha Levin 	if (!host_ip_addr)
2794d67c820SSasha Levin 		host_ip_addr = DEFAULT_HOST_ADDR;
2804d67c820SSasha Levin 
281a4e724ddSSasha Levin 	if (!guest_mac)
282a4e724ddSSasha Levin 		guest_mac = DEFAULT_GUEST_MAC;
283a4e724ddSSasha Levin 
28473b7d038SAmos Kong 	if (!script)
28573b7d038SAmos Kong 		script = DEFAULT_SCRIPT;
28673b7d038SAmos Kong 
287f967c427SPrasad Joshi 	term_init();
288f967c427SPrasad Joshi 
289f967c427SPrasad Joshi 	kvm = kvm__init(kvm_dev, ram_size);
290f967c427SPrasad Joshi 
291f967c427SPrasad Joshi 	if (image_filename) {
2929ac38fe1SSasha Levin 		kvm->disk_image	= disk_image__open(image_filename, readonly_image);
293f967c427SPrasad Joshi 		if (!kvm->disk_image)
294f967c427SPrasad Joshi 			die("unable to load disk image %s", image_filename);
295f967c427SPrasad Joshi 	}
296f967c427SPrasad Joshi 
297f967c427SPrasad Joshi 	strcpy(real_cmdline, "notsc nolapic noacpi pci=conf1 console=ttyS0 ");
298f967c427SPrasad Joshi 	if (!kernel_cmdline || !strstr(kernel_cmdline, "root=")) {
299f967c427SPrasad Joshi 		strlcat(real_cmdline, "root=/dev/vda rw ",
300f967c427SPrasad Joshi 				sizeof(real_cmdline));
301f967c427SPrasad Joshi 	}
302f967c427SPrasad Joshi 
303f967c427SPrasad Joshi 	if (kernel_cmdline) {
304f967c427SPrasad Joshi 		strlcat(real_cmdline, kernel_cmdline, sizeof(real_cmdline));
305f967c427SPrasad Joshi 		real_cmdline[sizeof(real_cmdline)-1] = '\0';
306f967c427SPrasad Joshi 	}
307f967c427SPrasad Joshi 
308f967c427SPrasad Joshi 	if (!kvm__load_kernel(kvm, kernel_filename, initrd_filename,
309f967c427SPrasad Joshi 				real_cmdline))
310f967c427SPrasad Joshi 		die("unable to load kernel %s", kernel_filename);
311f967c427SPrasad Joshi 
312ac38f433SPekka Enberg 	ioport__setup_legacy();
313ac38f433SPekka Enberg 
314f967c427SPrasad Joshi 	kvm__setup_bios(kvm);
315f967c427SPrasad Joshi 
316f967c427SPrasad Joshi 	serial8250__init(kvm);
317f967c427SPrasad Joshi 
318f967c427SPrasad Joshi 	pci__init();
319f967c427SPrasad Joshi 
320f967c427SPrasad Joshi 	virtio_blk__init(kvm);
321f967c427SPrasad Joshi 
322f967c427SPrasad Joshi 	virtio_console__init(kvm);
323f967c427SPrasad Joshi 
3244f56d42cSAsias He 	if (!network)
3254f56d42cSAsias He 		network = DEFAULT_NETWORK;
3264f56d42cSAsias He 
327bdfcfca6SSasha Levin 	if (!strncmp(network, "virtio", 6)) {
328bdfcfca6SSasha Levin 		net_params = (struct virtio_net_parameters) {
329bdfcfca6SSasha Levin 			.host_ip = host_ip_addr,
33073b7d038SAmos Kong 			.self = kvm,
33173b7d038SAmos Kong 			.script = script
332bdfcfca6SSasha Levin 		};
333a4e724ddSSasha Levin 		sscanf(guest_mac,	"%hhx:%hhx:%hhx:%hhx:%hhx:%hhx",
334a4e724ddSSasha Levin 							net_params.guest_mac,
335a4e724ddSSasha Levin 							net_params.guest_mac+1,
336a4e724ddSSasha Levin 							net_params.guest_mac+2,
337a4e724ddSSasha Levin 							net_params.guest_mac+3,
338a4e724ddSSasha Levin 							net_params.guest_mac+4,
339a4e724ddSSasha Levin 							net_params.guest_mac+5);
340a4e724ddSSasha Levin 
341bdfcfca6SSasha Levin 		virtio_net__init(&net_params);
342bdfcfca6SSasha Levin 	}
3434f56d42cSAsias He 
344f967c427SPrasad Joshi 	kvm__start_timer(kvm);
345f967c427SPrasad Joshi 
346d77a9efaSCyrill Gorcunov 	for (i = 0; i < nrcpus; i++) {
347d77a9efaSCyrill Gorcunov 		kvm_cpus[i] = kvm_cpu__init(kvm, i);
348d77a9efaSCyrill Gorcunov 		if (!kvm_cpus[i])
3495ee154d1SPekka Enberg 			die("unable to initialize KVM VCPU");
3505d1a249cSPekka Enberg 
3515ee154d1SPekka Enberg 		if (single_step)
352d77a9efaSCyrill Gorcunov 			kvm_cpu__enable_singlestep(kvm_cpus[i]);
3535ee154d1SPekka Enberg 
354d77a9efaSCyrill Gorcunov 		if (pthread_create(&kvm_cpus[i]->thread, NULL, kvm_cpu_thread, kvm_cpus[i]) != 0)
3555ee154d1SPekka Enberg 			die("unable to create KVM VCPU thread");
3565ee154d1SPekka Enberg 	}
3575ee154d1SPekka Enberg 
358d77a9efaSCyrill Gorcunov 	for (i = 0; i < nrcpus; i++) {
3595ee154d1SPekka Enberg 		void *ret;
3605ee154d1SPekka Enberg 
361d77a9efaSCyrill Gorcunov 		if (pthread_join(kvm_cpus[i]->thread, &ret) != 0)
3625ee154d1SPekka Enberg 			die("pthread_join");
3635ee154d1SPekka Enberg 
3645ee154d1SPekka Enberg 		if (ret != NULL)
3655ee154d1SPekka Enberg 			exit_code	= 1;
3665ee154d1SPekka Enberg 	}
367f967c427SPrasad Joshi 
368f967c427SPrasad Joshi 	disk_image__close(kvm->disk_image);
369f967c427SPrasad Joshi 	kvm__delete(kvm);
370f967c427SPrasad Joshi 
3715ee154d1SPekka Enberg 	if (!exit_code)
372f967c427SPrasad Joshi 		printf("\n  # KVM session ended normally.\n");
373f967c427SPrasad Joshi 
3745ee154d1SPekka Enberg 	return exit_code;
375f967c427SPrasad Joshi }
376