1 #include <stdio.h> 2 #include <string.h> 3 #include <signal.h> 4 #include <unistd.h> 5 #include <stdlib.h> 6 #include <termios.h> 7 #include <sys/utsname.h> 8 #include <sys/types.h> 9 #include <sys/stat.h> 10 #include <ctype.h> 11 12 /* user defined header files */ 13 #include <linux/types.h> 14 #include <kvm/kvm.h> 15 #include <kvm/kvm-cpu.h> 16 #include <kvm/8250-serial.h> 17 #include <kvm/virtio-blk.h> 18 #include <kvm/virtio-net.h> 19 #include <kvm/virtio-console.h> 20 #include <kvm/virtio-rng.h> 21 #include <kvm/virtio-balloon.h> 22 #include <kvm/disk-image.h> 23 #include <kvm/util.h> 24 #include <kvm/pci.h> 25 #include <kvm/rtc.h> 26 #include <kvm/term.h> 27 #include <kvm/ioport.h> 28 #include <kvm/threadpool.h> 29 #include <kvm/barrier.h> 30 #include <kvm/symbol.h> 31 #include <kvm/virtio-9p.h> 32 #include <kvm/vesa.h> 33 #include <kvm/ioeventfd.h> 34 #include <kvm/i8042.h> 35 #include <kvm/vnc.h> 36 #include <kvm/sdl.h> 37 #include <kvm/framebuffer.h> 38 #include <kvm/irq.h> 39 40 /* header files for gitish interface */ 41 #include <kvm/builtin-run.h> 42 #include <kvm/parse-options.h> 43 #include <kvm/mutex.h> 44 45 #define DEFAULT_KVM_DEV "/dev/kvm" 46 #define DEFAULT_CONSOLE "serial" 47 #define DEFAULT_NETWORK "user" 48 #define DEFAULT_HOST_ADDR "192.168.33.1" 49 #define DEFAULT_GUEST_ADDR "192.168.33.15" 50 #define DEFAULT_GUEST_MAC "02:15:15:15:15:15" 51 #define DEFAULT_HOST_MAC "02:01:01:01:01:01" 52 #define DEFAULT_SCRIPT "none" 53 54 #define MB_SHIFT (20) 55 #define MIN_RAM_SIZE_MB (64ULL) 56 #define MIN_RAM_SIZE_BYTE (MIN_RAM_SIZE_MB << MB_SHIFT) 57 58 struct kvm *kvm; 59 struct kvm_cpu *kvm_cpus[KVM_NR_CPUS]; 60 __thread struct kvm_cpu *current_kvm_cpu; 61 62 static u64 ram_size; 63 static u8 image_count; 64 static int virtio_rng; 65 static const char *kernel_cmdline; 66 static const char *kernel_filename; 67 static const char *vmlinux_filename; 68 static const char *initrd_filename; 69 static const char *image_filename[MAX_DISK_IMAGES]; 70 static const char *console; 71 static const char *dev; 72 static const char *network; 73 static const char *host_ip; 74 static const char *guest_ip; 75 static const char *guest_mac; 76 static const char *host_mac; 77 static const char *script; 78 static const char *guest_name; 79 static bool single_step; 80 static bool readonly_image[MAX_DISK_IMAGES]; 81 static bool vnc; 82 static bool sdl; 83 static bool balloon; 84 static bool using_rootfs; 85 extern bool ioport_debug; 86 extern int active_console; 87 extern int debug_iodelay; 88 89 bool do_debug_print = false; 90 91 static int nrcpus; 92 static int vidmode = -1; 93 94 static const char * const run_usage[] = { 95 "kvm run [<options>] [<kernel image>]", 96 NULL 97 }; 98 99 static int img_name_parser(const struct option *opt, const char *arg, int unset) 100 { 101 char *sep; 102 struct stat st; 103 104 if (stat(arg, &st) == 0 && 105 S_ISDIR(st.st_mode)) { 106 char tmp[PATH_MAX]; 107 108 if (realpath(arg, tmp) == 0 || 109 virtio_9p__init(kvm, tmp, "/dev/root") < 0) 110 die("Unable to initialize virtio 9p"); 111 using_rootfs = 1; 112 return 0; 113 } 114 115 if (image_count >= MAX_DISK_IMAGES) 116 die("Currently only 4 images are supported"); 117 118 image_filename[image_count] = arg; 119 sep = strstr(arg, ","); 120 if (sep) { 121 if (strcmp(sep + 1, "ro") == 0) 122 readonly_image[image_count] = 1; 123 *sep = 0; 124 } 125 126 image_count++; 127 128 return 0; 129 } 130 131 static int virtio_9p_rootdir_parser(const struct option *opt, const char *arg, int unset) 132 { 133 char *tag_name; 134 char tmp[PATH_MAX]; 135 136 /* 137 * 9p dir can be of the form dirname,tag_name or 138 * just dirname. In the later case we use the 139 * default tag name 140 */ 141 tag_name = strstr(arg, ","); 142 if (tag_name) { 143 *tag_name = '\0'; 144 tag_name++; 145 } 146 if (realpath(arg, tmp)) { 147 if (virtio_9p__init(kvm, tmp, tag_name) < 0) 148 die("Unable to initialize virtio 9p"); 149 } else 150 die("Failed resolving 9p path"); 151 return 0; 152 } 153 154 155 static const struct option options[] = { 156 OPT_GROUP("Basic options:"), 157 OPT_STRING('\0', "name", &guest_name, "guest name", 158 "A name for the guest"), 159 OPT_INTEGER('c', "cpus", &nrcpus, "Number of CPUs"), 160 OPT_U64('m', "mem", &ram_size, "Virtual machine memory size in MiB."), 161 OPT_CALLBACK('d', "disk", NULL, "image", "Disk image", img_name_parser), 162 OPT_STRING('\0', "console", &console, "serial or virtio", 163 "Console to use"), 164 OPT_INCR('\0', "rng", &virtio_rng, 165 "Enable virtio Random Number Generator"), 166 OPT_STRING('\0', "dev", &dev, "device_file", "KVM device file"), 167 OPT_CALLBACK('\0', "9p", NULL, "dir_to_share,tag_name", 168 "Enable virtio 9p to share files between host and guest", virtio_9p_rootdir_parser), 169 OPT_BOOLEAN('\0', "balloon", &balloon, "Enable virtio balloon"), 170 OPT_BOOLEAN('\0', "vnc", &vnc, "Enable VNC framebuffer"), 171 OPT_BOOLEAN('\0', "sdl", &sdl, "Enable SDL framebuffer"), 172 173 OPT_GROUP("Kernel options:"), 174 OPT_STRING('k', "kernel", &kernel_filename, "kernel", 175 "Kernel to boot in virtual machine"), 176 OPT_STRING('i', "initrd", &initrd_filename, "initrd", 177 "Initial RAM disk image"), 178 OPT_STRING('p', "params", &kernel_cmdline, "params", 179 "Kernel command line arguments"), 180 181 OPT_GROUP("Networking options:"), 182 OPT_STRING('n', "network", &network, "user, tap, none", 183 "Network to use"), 184 OPT_STRING('\0', "host-ip", &host_ip, "a.b.c.d", 185 "Assign this address to the host side networking"), 186 OPT_STRING('\0', "guest-ip", &guest_ip, "a.b.c.d", 187 "Assign this address to the guest side networking"), 188 OPT_STRING('\0', "host-mac", &host_mac, "aa:bb:cc:dd:ee:ff", 189 "Assign this address to the host side NIC"), 190 OPT_STRING('\0', "guest-mac", &guest_mac, "aa:bb:cc:dd:ee:ff", 191 "Assign this address to the guest side NIC"), 192 OPT_STRING('\0', "tapscript", &script, "Script path", 193 "Assign a script to process created tap device"), 194 195 OPT_GROUP("BIOS options:"), 196 OPT_INTEGER('\0', "vidmode", &vidmode, 197 "Video mode"), 198 199 OPT_GROUP("Debug options:"), 200 OPT_BOOLEAN('\0', "debug", &do_debug_print, 201 "Enable debug messages"), 202 OPT_BOOLEAN('\0', "debug-single-step", &single_step, 203 "Enable single stepping"), 204 OPT_BOOLEAN('\0', "debug-ioport", &ioport_debug, 205 "Enable ioport debugging"), 206 OPT_INTEGER('\0', "debug-iodelay", &debug_iodelay, 207 "Delay IO by millisecond"), 208 OPT_END() 209 }; 210 211 /* 212 * Serialize debug printout so that the output of multiple vcpus does not 213 * get mixed up: 214 */ 215 static int printout_done; 216 217 static void handle_sigusr1(int sig) 218 { 219 struct kvm_cpu *cpu = current_kvm_cpu; 220 221 if (!cpu) 222 return; 223 224 printf("\n #\n # vCPU #%ld's dump:\n #\n", cpu->cpu_id); 225 kvm_cpu__show_registers(cpu); 226 kvm_cpu__show_code(cpu); 227 kvm_cpu__show_page_tables(cpu); 228 fflush(stdout); 229 printout_done = 1; 230 mb(); 231 } 232 233 /* Pause/resume the guest using SIGUSR2 */ 234 static int is_paused; 235 236 static void handle_sigusr2(int sig) 237 { 238 if (sig == SIGKVMRESUME && is_paused) 239 kvm__continue(); 240 else if (sig == SIGUSR2 && !is_paused) 241 kvm__pause(); 242 else 243 return; 244 245 is_paused = !is_paused; 246 pr_info("Guest %s\n", is_paused ? "paused" : "resumed"); 247 } 248 249 static void handle_sigquit(int sig) 250 { 251 int i; 252 253 for (i = 0; i < nrcpus; i++) { 254 struct kvm_cpu *cpu = kvm_cpus[i]; 255 256 if (!cpu) 257 continue; 258 259 printout_done = 0; 260 pthread_kill(cpu->thread, SIGUSR1); 261 /* 262 * Wait for the vCPU to dump state before signalling 263 * the next thread. Since this is debug code it does 264 * not matter that we are burning CPU time a bit: 265 */ 266 while (!printout_done) 267 mb(); 268 } 269 270 serial8250__inject_sysrq(kvm); 271 } 272 273 static void handle_sigalrm(int sig) 274 { 275 serial8250__inject_interrupt(kvm); 276 virtio_console__inject_interrupt(kvm); 277 } 278 279 static void handle_sigstop(int sig) 280 { 281 kvm_cpu__reboot(); 282 } 283 284 static void *kvm_cpu_thread(void *arg) 285 { 286 current_kvm_cpu = arg; 287 288 if (kvm_cpu__start(current_kvm_cpu)) 289 goto panic_kvm; 290 291 kvm_cpu__delete(current_kvm_cpu); 292 293 return (void *) (intptr_t) 0; 294 295 panic_kvm: 296 fprintf(stderr, "KVM exit reason: %u (\"%s\")\n", 297 current_kvm_cpu->kvm_run->exit_reason, 298 kvm_exit_reasons[current_kvm_cpu->kvm_run->exit_reason]); 299 if (current_kvm_cpu->kvm_run->exit_reason == KVM_EXIT_UNKNOWN) 300 fprintf(stderr, "KVM exit code: 0x%Lu\n", 301 current_kvm_cpu->kvm_run->hw.hardware_exit_reason); 302 303 kvm_cpu__show_registers(current_kvm_cpu); 304 kvm_cpu__show_code(current_kvm_cpu); 305 kvm_cpu__show_page_tables(current_kvm_cpu); 306 307 kvm_cpu__delete(current_kvm_cpu); 308 309 return (void *) (intptr_t) 1; 310 } 311 312 static char kernel[PATH_MAX]; 313 314 static const char *host_kernels[] = { 315 "/boot/vmlinuz", 316 "/boot/bzImage", 317 NULL 318 }; 319 320 static const char *default_kernels[] = { 321 "./bzImage", 322 "../../arch/x86/boot/bzImage", 323 NULL 324 }; 325 326 static const char *default_vmlinux[] = { 327 "../../../vmlinux", 328 "../../vmlinux", 329 NULL 330 }; 331 332 static void kernel_usage_with_options(void) 333 { 334 const char **k; 335 struct utsname uts; 336 337 fprintf(stderr, "Fatal: could not find default kernel image in:\n"); 338 k = &default_kernels[0]; 339 while (*k) { 340 fprintf(stderr, "\t%s\n", *k); 341 k++; 342 } 343 344 if (uname(&uts) < 0) 345 return; 346 347 k = &host_kernels[0]; 348 while (*k) { 349 if (snprintf(kernel, PATH_MAX, "%s-%s", *k, uts.release) < 0) 350 return; 351 fprintf(stderr, "\t%s\n", kernel); 352 k++; 353 } 354 fprintf(stderr, "\nPlease see 'kvm run --help' for more options.\n\n"); 355 } 356 357 static u64 host_ram_size(void) 358 { 359 long page_size; 360 long nr_pages; 361 362 nr_pages = sysconf(_SC_PHYS_PAGES); 363 if (nr_pages < 0) { 364 pr_warning("sysconf(_SC_PHYS_PAGES) failed"); 365 return 0; 366 } 367 368 page_size = sysconf(_SC_PAGE_SIZE); 369 if (page_size < 0) { 370 pr_warning("sysconf(_SC_PAGE_SIZE) failed"); 371 return 0; 372 } 373 374 return (nr_pages * page_size) >> MB_SHIFT; 375 } 376 377 /* 378 * If user didn't specify how much memory it wants to allocate for the guest, 379 * avoid filling the whole host RAM. 380 */ 381 #define RAM_SIZE_RATIO 0.8 382 383 static u64 get_ram_size(int nr_cpus) 384 { 385 u64 available; 386 u64 ram_size; 387 388 ram_size = 64 * (nr_cpus + 3); 389 390 available = host_ram_size() * RAM_SIZE_RATIO; 391 if (!available) 392 available = MIN_RAM_SIZE_MB; 393 394 if (ram_size > available) 395 ram_size = available; 396 397 return ram_size; 398 } 399 400 static const char *find_kernel(void) 401 { 402 const char **k; 403 struct stat st; 404 struct utsname uts; 405 406 k = &default_kernels[0]; 407 while (*k) { 408 if (stat(*k, &st) < 0 || !S_ISREG(st.st_mode)) { 409 k++; 410 continue; 411 } 412 strncpy(kernel, *k, PATH_MAX); 413 return kernel; 414 } 415 416 if (uname(&uts) < 0) 417 return NULL; 418 419 k = &host_kernels[0]; 420 while (*k) { 421 if (snprintf(kernel, PATH_MAX, "%s-%s", *k, uts.release) < 0) 422 return NULL; 423 424 if (stat(kernel, &st) < 0 || !S_ISREG(st.st_mode)) { 425 k++; 426 continue; 427 } 428 return kernel; 429 430 } 431 return NULL; 432 } 433 434 static const char *find_vmlinux(void) 435 { 436 const char **vmlinux; 437 438 vmlinux = &default_vmlinux[0]; 439 while (*vmlinux) { 440 struct stat st; 441 442 if (stat(*vmlinux, &st) < 0 || !S_ISREG(st.st_mode)) { 443 vmlinux++; 444 continue; 445 } 446 return *vmlinux; 447 } 448 return NULL; 449 } 450 451 static int root_device(char *dev, long *part) 452 { 453 struct stat st; 454 455 if (stat("/", &st) < 0) 456 return -1; 457 458 *part = minor(st.st_dev); 459 460 sprintf(dev, "/dev/block/%u:0", major(st.st_dev)); 461 if (access(dev, R_OK) < 0) 462 return -1; 463 464 return 0; 465 } 466 467 static char *host_image(char *cmd_line, size_t size) 468 { 469 char *t; 470 char device[PATH_MAX]; 471 long part = 0; 472 473 t = malloc(PATH_MAX); 474 if (!t) 475 return NULL; 476 477 /* check for the root file system */ 478 if (root_device(device, &part) < 0) { 479 free(t); 480 return NULL; 481 } 482 strncpy(t, device, PATH_MAX); 483 if (!strstr(cmd_line, "root=")) { 484 char tmp[PATH_MAX]; 485 snprintf(tmp, sizeof(tmp), "root=/dev/vda%ld rw ", part); 486 strlcat(cmd_line, tmp, size); 487 } 488 return t; 489 } 490 491 void kvm_run_help(void) 492 { 493 usage_with_options(run_usage, options); 494 } 495 496 int kvm_cmd_run(int argc, const char **argv, const char *prefix) 497 { 498 struct virtio_net_parameters net_params; 499 static char real_cmdline[2048], default_name[20]; 500 struct framebuffer *fb = NULL; 501 unsigned int nr_online_cpus; 502 int exit_code = 0; 503 int max_cpus, recommended_cpus; 504 char *hi; 505 int i; 506 void *ret; 507 508 signal(SIGALRM, handle_sigalrm); 509 signal(SIGQUIT, handle_sigquit); 510 signal(SIGUSR1, handle_sigusr1); 511 signal(SIGUSR2, handle_sigusr2); 512 signal(SIGKVMSTOP, handle_sigstop); 513 signal(SIGKVMRESUME, handle_sigusr2); 514 515 nr_online_cpus = sysconf(_SC_NPROCESSORS_ONLN); 516 517 while (argc != 0) { 518 argc = parse_options(argc, argv, options, run_usage, 519 PARSE_OPT_STOP_AT_NON_OPTION); 520 if (argc != 0) { 521 if (kernel_filename) { 522 fprintf(stderr, "Cannot handle parameter: " 523 "%s\n", argv[0]); 524 usage_with_options(run_usage, options); 525 return EINVAL; 526 } 527 /* first unhandled parameter is treated as a kernel 528 image 529 */ 530 kernel_filename = argv[0]; 531 argv++; 532 argc--; 533 } 534 535 } 536 537 if (!kernel_filename) 538 kernel_filename = find_kernel(); 539 540 if (!kernel_filename) { 541 kernel_usage_with_options(); 542 return EINVAL; 543 } 544 545 vmlinux_filename = find_vmlinux(); 546 547 if (nrcpus == 0) 548 nrcpus = nr_online_cpus; 549 else if (nrcpus < 1 || nrcpus > KVM_NR_CPUS) 550 die("Number of CPUs %d is out of [1;%d] range", nrcpus, KVM_NR_CPUS); 551 552 if (!ram_size) 553 ram_size = get_ram_size(nrcpus); 554 555 if (ram_size < MIN_RAM_SIZE_MB) 556 die("Not enough memory specified: %lluMB (min %lluMB)", ram_size, MIN_RAM_SIZE_MB); 557 558 if (ram_size > host_ram_size()) 559 pr_warning("Guest memory size %lluMB exceeds host physical RAM size %lluMB", ram_size, host_ram_size()); 560 561 ram_size <<= MB_SHIFT; 562 563 if (!dev) 564 dev = DEFAULT_KVM_DEV; 565 566 if (!console) 567 console = DEFAULT_CONSOLE; 568 569 if (!strncmp(console, "virtio", 6)) 570 active_console = CONSOLE_VIRTIO; 571 else 572 active_console = CONSOLE_8250; 573 574 if (!host_ip) 575 host_ip = DEFAULT_HOST_ADDR; 576 577 if (!guest_ip) 578 guest_ip = DEFAULT_GUEST_ADDR; 579 580 if (!guest_mac) 581 guest_mac = DEFAULT_GUEST_MAC; 582 583 if (!host_mac) 584 host_mac = DEFAULT_HOST_MAC; 585 586 if (!script) 587 script = DEFAULT_SCRIPT; 588 589 symbol__init(vmlinux_filename); 590 591 term_init(); 592 593 if (!guest_name) { 594 sprintf(default_name, "guest-%u", getpid()); 595 guest_name = default_name; 596 } 597 598 kvm = kvm__init(dev, ram_size, guest_name); 599 600 irq__init(kvm); 601 602 kvm->single_step = single_step; 603 604 ioeventfd__init(); 605 606 max_cpus = kvm__max_cpus(kvm); 607 recommended_cpus = kvm__recommended_cpus(kvm); 608 609 if (nrcpus > max_cpus) { 610 printf(" # Limit the number of CPUs to %d\n", max_cpus); 611 kvm->nrcpus = max_cpus; 612 } else if (nrcpus > recommended_cpus) { 613 printf(" # Warning: The maximum recommended amount of VCPUs" 614 " is %d\n", recommended_cpus); 615 } 616 617 kvm->nrcpus = nrcpus; 618 619 /* 620 * vidmode should be either specified 621 * either set by default 622 */ 623 if (vnc || sdl) { 624 if (vidmode == -1) 625 vidmode = 0x312; 626 } else 627 vidmode = 0; 628 629 memset(real_cmdline, 0, sizeof(real_cmdline)); 630 strcpy(real_cmdline, "notsc noapic noacpi pci=conf1 reboot=k panic=1"); 631 if (vnc || sdl) { 632 strcat(real_cmdline, " video=vesafb console=tty0"); 633 } else 634 strcat(real_cmdline, " console=ttyS0 earlyprintk=serial"); 635 strcat(real_cmdline, " "); 636 if (kernel_cmdline) 637 strlcat(real_cmdline, kernel_cmdline, sizeof(real_cmdline)); 638 639 hi = NULL; 640 if (!using_rootfs && !image_filename[0]) { 641 hi = host_image(real_cmdline, sizeof(real_cmdline)); 642 if (hi) { 643 image_filename[0] = hi; 644 readonly_image[0] = true; 645 image_count++; 646 } 647 } 648 649 if (!strstr(real_cmdline, "root=")) 650 strlcat(real_cmdline, " root=/dev/vda rw ", sizeof(real_cmdline)); 651 652 if (using_rootfs) 653 strcat(real_cmdline, " root=/dev/root rootflags=rw,trans=virtio,version=9p2000.u rootfstype=9p"); 654 655 if (image_count) { 656 kvm->nr_disks = image_count; 657 kvm->disks = disk_image__open_all(image_filename, readonly_image, image_count); 658 if (!kvm->disks) 659 die("Unable to load all disk images."); 660 661 virtio_blk__init_all(kvm); 662 } 663 664 free(hi); 665 666 printf(" # kvm run -k %s -m %Lu -c %d --name %s\n", kernel_filename, ram_size / 1024 / 1024, nrcpus, guest_name); 667 668 if (!kvm__load_kernel(kvm, kernel_filename, initrd_filename, 669 real_cmdline, vidmode)) 670 die("unable to load kernel %s", kernel_filename); 671 672 kvm->vmlinux = vmlinux_filename; 673 674 ioport__setup_legacy(); 675 676 rtc__init(); 677 678 serial8250__init(kvm); 679 680 pci__init(); 681 682 if (active_console == CONSOLE_VIRTIO) 683 virtio_console__init(kvm); 684 685 if (virtio_rng) 686 while (virtio_rng--) 687 virtio_rng__init(kvm); 688 689 if (balloon) 690 virtio_bln__init(kvm); 691 692 if (!network) 693 network = DEFAULT_NETWORK; 694 695 if (strncmp(network, "none", 4)) { 696 net_params.guest_ip = guest_ip; 697 net_params.host_ip = host_ip; 698 net_params.kvm = kvm; 699 net_params.script = script; 700 sscanf(guest_mac, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", 701 net_params.guest_mac, 702 net_params.guest_mac+1, 703 net_params.guest_mac+2, 704 net_params.guest_mac+3, 705 net_params.guest_mac+4, 706 net_params.guest_mac+5); 707 sscanf(host_mac, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", 708 net_params.host_mac, 709 net_params.host_mac+1, 710 net_params.host_mac+2, 711 net_params.host_mac+3, 712 net_params.host_mac+4, 713 net_params.host_mac+5); 714 715 if (!strncmp(network, "user", 4)) 716 net_params.mode = NET_MODE_USER; 717 else if (!strncmp(network, "tap", 3)) 718 net_params.mode = NET_MODE_TAP; 719 else 720 die("Unkown network mode %s, please use -network user, tap, none", network); 721 virtio_net__init(&net_params); 722 } 723 724 kvm__start_timer(kvm); 725 726 kvm__setup_bios(kvm); 727 728 for (i = 0; i < nrcpus; i++) { 729 kvm_cpus[i] = kvm_cpu__init(kvm, i); 730 if (!kvm_cpus[i]) 731 die("unable to initialize KVM VCPU"); 732 } 733 734 kvm__init_ram(kvm); 735 736 kbd__init(kvm); 737 738 if (vnc || sdl) 739 fb = vesa__init(kvm); 740 741 if (vnc) { 742 if (fb) 743 vnc__init(fb); 744 } 745 746 if (sdl) { 747 if (fb) 748 sdl__init(fb); 749 } 750 751 fb__start(); 752 753 thread_pool__init(nr_online_cpus); 754 ioeventfd__start(); 755 756 for (i = 0; i < nrcpus; i++) { 757 if (pthread_create(&kvm_cpus[i]->thread, NULL, kvm_cpu_thread, kvm_cpus[i]) != 0) 758 die("unable to create KVM VCPU thread"); 759 } 760 761 /* Only VCPU #0 is going to exit by itself when shutting down */ 762 if (pthread_join(kvm_cpus[0]->thread, &ret) != 0) 763 exit_code = 1; 764 765 for (i = 1; i < nrcpus; i++) { 766 pthread_kill(kvm_cpus[i]->thread, SIGKVMEXIT); 767 if (pthread_join(kvm_cpus[i]->thread, &ret) != 0) 768 die("pthread_join"); 769 770 if (ret != NULL) 771 exit_code = 1; 772 } 773 774 fb__stop(); 775 776 virtio_blk__delete_all(kvm); 777 virtio_rng__delete_all(kvm); 778 779 disk_image__close_all(kvm->disks, image_count); 780 kvm__delete(kvm); 781 782 if (!exit_code) 783 printf("\n # KVM session ended normally.\n"); 784 785 return exit_code; 786 } 787