1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 #[macro_use] 12 extern crate lazy_static; 13 14 extern crate test_infra; 15 16 use net_util::MacAddr; 17 use std::collections::HashMap; 18 use std::fs; 19 use std::io; 20 use std::io::BufRead; 21 use std::io::Read; 22 use std::io::Seek; 23 use std::io::Write; 24 use std::os::unix::io::AsRawFd; 25 use std::path::PathBuf; 26 use std::process::{Child, Command, Stdio}; 27 use std::string::String; 28 use std::sync::mpsc; 29 use std::sync::mpsc::Receiver; 30 #[cfg(target_arch = "x86_64")] 31 use std::sync::Mutex; 32 use std::thread; 33 use test_infra::*; 34 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 35 use wait_timeout::ChildExt; 36 37 #[cfg(target_arch = "x86_64")] 38 mod x86_64 { 39 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw"; 40 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 41 pub const FOCAL_SGX_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-sgx.raw"; 42 pub const HIRSUTE_NVIDIA_IMAGE_NAME: &str = "hirsute-server-cloudimg-amd64-nvidia.raw"; 43 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 44 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 45 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 46 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20220329-0.raw"; 47 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2019.raw"; 48 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 49 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 50 } 51 52 #[cfg(target_arch = "x86_64")] 53 use x86_64::*; 54 55 #[cfg(target_arch = "aarch64")] 56 mod aarch64 { 57 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 58 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 59 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 60 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 61 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 62 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 63 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 64 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 65 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 66 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 67 } 68 69 #[cfg(target_arch = "aarch64")] 70 use aarch64::*; 71 72 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 73 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 74 75 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 76 77 fn prepare_virtiofsd( 78 tmp_dir: &TempDir, 79 shared_dir: &str, 80 cache: &str, 81 ) -> (std::process::Child, String) { 82 let mut workload_path = dirs::home_dir().unwrap(); 83 workload_path.push("workloads"); 84 85 let mut virtiofsd_path = workload_path; 86 virtiofsd_path.push("virtiofsd"); 87 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 88 89 let virtiofsd_socket_path = 90 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 91 92 // Start the daemon 93 let child = Command::new(virtiofsd_path.as_str()) 94 .args(&["--shared-dir", shared_dir]) 95 .args(&["--socket-path", virtiofsd_socket_path.as_str()]) 96 .args(&["--cache", cache]) 97 .spawn() 98 .unwrap(); 99 100 thread::sleep(std::time::Duration::new(10, 0)); 101 102 (child, virtiofsd_socket_path) 103 } 104 105 fn prepare_vubd( 106 tmp_dir: &TempDir, 107 blk_img: &str, 108 num_queues: usize, 109 rdonly: bool, 110 direct: bool, 111 ) -> (std::process::Child, String) { 112 let mut workload_path = dirs::home_dir().unwrap(); 113 workload_path.push("workloads"); 114 115 let mut blk_file_path = workload_path; 116 blk_file_path.push(blk_img); 117 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 118 119 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 120 121 // Start the daemon 122 let child = Command::new(clh_command("vhost_user_block")) 123 .args(&[ 124 "--block-backend", 125 format!( 126 "path={},socket={},num_queues={},readonly={},direct={}", 127 blk_file_path, vubd_socket_path, num_queues, rdonly, direct 128 ) 129 .as_str(), 130 ]) 131 .spawn() 132 .unwrap(); 133 134 thread::sleep(std::time::Duration::new(10, 0)); 135 136 (child, vubd_socket_path) 137 } 138 139 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 140 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 141 } 142 143 fn temp_api_path(tmp_dir: &TempDir) -> String { 144 String::from( 145 tmp_dir 146 .as_path() 147 .join("cloud-hypervisor.sock") 148 .to_str() 149 .unwrap(), 150 ) 151 } 152 153 // Creates the directory and returns the path. 154 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 155 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 156 std::fs::create_dir(&snapshot_dir).unwrap(); 157 snapshot_dir 158 } 159 160 // Creates the path for direct kernel boot and return the path. 161 // For x86_64, this function returns the vmlinux kernel path. 162 // For AArch64, this function returns the PE kernel path. 163 fn direct_kernel_boot_path() -> PathBuf { 164 let mut workload_path = dirs::home_dir().unwrap(); 165 workload_path.push("workloads"); 166 167 let mut kernel_path = workload_path; 168 #[cfg(target_arch = "x86_64")] 169 kernel_path.push("vmlinux"); 170 #[cfg(target_arch = "aarch64")] 171 kernel_path.push("Image"); 172 173 kernel_path 174 } 175 176 #[cfg(target_arch = "aarch64")] 177 fn edk2_path() -> PathBuf { 178 let mut workload_path = dirs::home_dir().unwrap(); 179 workload_path.push("workloads"); 180 let mut edk2_path = workload_path; 181 edk2_path.push("CLOUDHV_EFI.fd"); 182 183 edk2_path 184 } 185 186 fn prepare_vhost_user_net_daemon( 187 tmp_dir: &TempDir, 188 ip: &str, 189 tap: Option<&str>, 190 num_queues: usize, 191 client_mode: bool, 192 ) -> (std::process::Command, String) { 193 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 194 195 // Start the daemon 196 let net_params = if let Some(tap_str) = tap { 197 format!( 198 "tap={},ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 199 tap_str, ip, vunet_socket_path, num_queues, client_mode 200 ) 201 } else { 202 format!( 203 "ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 204 ip, vunet_socket_path, num_queues, client_mode 205 ) 206 }; 207 208 let mut command = Command::new(clh_command("vhost_user_net")); 209 command.args(&["--net-backend", net_params.as_str()]); 210 211 (command, vunet_socket_path) 212 } 213 214 fn curl_command(api_socket: &str, method: &str, url: &str, http_body: Option<&str>) { 215 let mut curl_args: Vec<&str> = ["--unix-socket", api_socket, "-i", "-X", method, url].to_vec(); 216 217 if let Some(body) = http_body { 218 curl_args.push("-H"); 219 curl_args.push("Accept: application/json"); 220 curl_args.push("-H"); 221 curl_args.push("Content-Type: application/json"); 222 curl_args.push("-d"); 223 curl_args.push(body); 224 } 225 226 let status = Command::new("curl") 227 .args(curl_args) 228 .status() 229 .expect("Failed to launch curl command"); 230 231 assert!(status.success()); 232 } 233 234 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 235 let mut cmd = Command::new(clh_command("ch-remote")); 236 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 237 238 if let Some(arg) = arg { 239 cmd.arg(arg); 240 } 241 242 cmd.status().expect("Failed to launch ch-remote").success() 243 } 244 245 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 246 let mut cmd = Command::new(clh_command("ch-remote")); 247 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 248 249 if let Some(arg) = arg { 250 cmd.arg(arg); 251 } 252 253 let output = cmd.output().expect("Failed to launch ch-remote"); 254 255 (output.status.success(), output.stdout) 256 } 257 258 fn resize_command( 259 api_socket: &str, 260 desired_vcpus: Option<u8>, 261 desired_ram: Option<usize>, 262 desired_balloon: Option<usize>, 263 ) -> bool { 264 let mut cmd = Command::new(clh_command("ch-remote")); 265 cmd.args(&[&format!("--api-socket={}", api_socket), "resize"]); 266 267 if let Some(desired_vcpus) = desired_vcpus { 268 cmd.arg(format!("--cpus={}", desired_vcpus)); 269 } 270 271 if let Some(desired_ram) = desired_ram { 272 cmd.arg(format!("--memory={}", desired_ram)); 273 } 274 275 if let Some(desired_balloon) = desired_balloon { 276 cmd.arg(format!("--balloon={}", desired_balloon)); 277 } 278 279 cmd.status().expect("Failed to launch ch-remote").success() 280 } 281 282 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 283 let mut cmd = Command::new(clh_command("ch-remote")); 284 cmd.args(&[ 285 &format!("--api-socket={}", api_socket), 286 "resize-zone", 287 &format!("--id={}", id), 288 &format!("--size={}", desired_size), 289 ]); 290 291 cmd.status().expect("Failed to launch ch-remote").success() 292 } 293 294 // setup OVS-DPDK bridge and ports 295 fn setup_ovs_dpdk() { 296 // setup OVS-DPDK 297 assert!(exec_host_command_status("service openvswitch-switch start").success()); 298 assert!(exec_host_command_status("ovs-vsctl init").success()); 299 assert!( 300 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 301 .success() 302 ); 303 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 304 305 // Create OVS-DPDK bridge and ports 306 assert!(exec_host_command_status( 307 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 308 ) 309 .success()); 310 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 311 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 312 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 313 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 314 } 315 fn cleanup_ovs_dpdk() { 316 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 317 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 318 } 319 // Setup two guests and ensure they are connected through ovs-dpdk 320 fn setup_ovs_dpdk_guests(guest1: &Guest, guest2: &Guest, api_socket: &str) -> (Child, Child) { 321 setup_ovs_dpdk(); 322 323 let mut child1 = GuestCommand::new(guest1) 324 .args(&["--cpus", "boot=2"]) 325 .args(&["--memory", "size=0,shared=on"]) 326 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 327 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 328 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 329 .default_disks() 330 .args(&["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 331 .capture_output() 332 .spawn() 333 .unwrap(); 334 335 #[cfg(target_arch = "x86_64")] 336 let guest_net_iface = "ens5"; 337 #[cfg(target_arch = "aarch64")] 338 let guest_net_iface = "enp0s5"; 339 340 let r = std::panic::catch_unwind(|| { 341 guest1.wait_vm_boot(None).unwrap(); 342 343 guest1 344 .ssh_command(&format!( 345 "sudo ip addr add 172.100.0.1/24 dev {}", 346 guest_net_iface 347 )) 348 .unwrap(); 349 guest1 350 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 351 .unwrap(); 352 353 let guest_ip = guest1.network.guest_ip.clone(); 354 thread::spawn(move || { 355 ssh_command_ip( 356 "nc -l 12345", 357 &guest_ip, 358 DEFAULT_SSH_RETRIES, 359 DEFAULT_SSH_TIMEOUT, 360 ) 361 .unwrap(); 362 }); 363 }); 364 if r.is_err() { 365 cleanup_ovs_dpdk(); 366 367 let _ = child1.kill(); 368 let output = child1.wait_with_output().unwrap(); 369 handle_child_output(r, &output); 370 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 371 } 372 373 let mut child2 = GuestCommand::new(guest2) 374 .args(&["--api-socket", api_socket]) 375 .args(&["--cpus", "boot=2"]) 376 .args(&["--memory", "size=0,shared=on"]) 377 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 378 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 379 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 380 .default_disks() 381 .args(&["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 382 .capture_output() 383 .spawn() 384 .unwrap(); 385 386 let r = std::panic::catch_unwind(|| { 387 guest2.wait_vm_boot(None).unwrap(); 388 389 guest2 390 .ssh_command(&format!( 391 "sudo ip addr add 172.100.0.2/24 dev {}", 392 guest_net_iface 393 )) 394 .unwrap(); 395 guest2 396 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 397 .unwrap(); 398 399 // Check the connection works properly between the two VMs 400 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 401 }); 402 if r.is_err() { 403 cleanup_ovs_dpdk(); 404 405 let _ = child1.kill(); 406 let _ = child2.kill(); 407 let output = child2.wait_with_output().unwrap(); 408 handle_child_output(r, &output); 409 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 410 } 411 412 (child1, child2) 413 } 414 415 enum FwType { 416 Ovmf, 417 RustHypervisorFirmware, 418 } 419 420 fn fw_path(fw_type: FwType) -> String { 421 let mut workload_path = dirs::home_dir().unwrap(); 422 workload_path.push("workloads"); 423 424 let mut fw_path = workload_path; 425 #[cfg(target_arch = "aarch64")] 426 fw_path.push("CLOUDHV_EFI.fd"); 427 #[cfg(target_arch = "x86_64")] 428 { 429 match fw_type { 430 FwType::Ovmf => fw_path.push(OVMF_NAME), 431 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 432 } 433 } 434 435 fw_path.to_str().unwrap().to_string() 436 } 437 438 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 439 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 440 let guest = Guest::new(Box::new(focal)); 441 let total_vcpus = threads_per_core * cores_per_package * packages; 442 let direct_kernel_boot_path = direct_kernel_boot_path(); 443 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 444 let fw_path = fw_path(FwType::RustHypervisorFirmware); 445 if use_fw { 446 kernel_path = fw_path.as_str(); 447 } 448 449 let mut child = GuestCommand::new(&guest) 450 .args(&[ 451 "--cpus", 452 &format!( 453 "boot={},topology={}:{}:1:{}", 454 total_vcpus, threads_per_core, cores_per_package, packages 455 ), 456 ]) 457 .args(&["--memory", "size=512M"]) 458 .args(&["--kernel", kernel_path]) 459 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 460 .default_disks() 461 .default_net() 462 .capture_output() 463 .spawn() 464 .unwrap(); 465 466 let r = std::panic::catch_unwind(|| { 467 guest.wait_vm_boot(None).unwrap(); 468 assert_eq!( 469 guest.get_cpu_count().unwrap_or_default(), 470 u32::from(total_vcpus) 471 ); 472 assert_eq!( 473 guest 474 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 475 .unwrap() 476 .trim() 477 .parse::<u8>() 478 .unwrap_or(0), 479 threads_per_core 480 ); 481 482 assert_eq!( 483 guest 484 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 485 .unwrap() 486 .trim() 487 .parse::<u8>() 488 .unwrap_or(0), 489 cores_per_package 490 ); 491 492 assert_eq!( 493 guest 494 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 495 .unwrap() 496 .trim() 497 .parse::<u8>() 498 .unwrap_or(0), 499 packages 500 ); 501 }); 502 503 let _ = child.kill(); 504 let output = child.wait_with_output().unwrap(); 505 506 handle_child_output(r, &output); 507 } 508 509 #[allow(unused_variables)] 510 fn _test_guest_numa_nodes(acpi: bool) { 511 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 512 let guest = Guest::new(Box::new(focal)); 513 let api_socket = temp_api_path(&guest.tmp_dir); 514 #[cfg(target_arch = "x86_64")] 515 let kernel_path = direct_kernel_boot_path(); 516 #[cfg(target_arch = "aarch64")] 517 let kernel_path = if acpi { 518 edk2_path() 519 } else { 520 direct_kernel_boot_path() 521 }; 522 523 let mut child = GuestCommand::new(&guest) 524 .args(&["--cpus", "boot=6,max=12"]) 525 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 526 .args(&[ 527 "--memory-zone", 528 "id=mem0,size=1G,hotplug_size=3G", 529 "id=mem1,size=2G,hotplug_size=3G", 530 "id=mem2,size=3G,hotplug_size=3G", 531 ]) 532 .args(&[ 533 "--numa", 534 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 535 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 536 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 537 ]) 538 .args(&["--kernel", kernel_path.to_str().unwrap()]) 539 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 540 .args(&["--api-socket", &api_socket]) 541 .capture_output() 542 .default_disks() 543 .default_net() 544 .spawn() 545 .unwrap(); 546 547 let r = std::panic::catch_unwind(|| { 548 guest.wait_vm_boot(None).unwrap(); 549 550 guest.check_numa_common( 551 Some(&[960_000, 1_920_000, 2_880_000]), 552 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 553 Some(&["10 15 20", "20 10 25", "25 30 10"]), 554 ); 555 556 // AArch64 currently does not support hotplug, and therefore we only 557 // test hotplug-related function on x86_64 here. 558 #[cfg(target_arch = "x86_64")] 559 { 560 guest.enable_memory_hotplug(); 561 562 // Resize every memory zone and check each associated NUMA node 563 // has been assigned the right amount of memory. 564 resize_zone_command(&api_socket, "mem0", "4G"); 565 resize_zone_command(&api_socket, "mem1", "4G"); 566 resize_zone_command(&api_socket, "mem2", "4G"); 567 // Resize to the maximum amount of CPUs and check each NUMA 568 // node has been assigned the right CPUs set. 569 resize_command(&api_socket, Some(12), None, None); 570 thread::sleep(std::time::Duration::new(5, 0)); 571 572 guest.check_numa_common( 573 Some(&[3_840_000, 3_840_000, 3_840_000]), 574 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 575 None, 576 ); 577 } 578 }); 579 580 let _ = child.kill(); 581 let output = child.wait_with_output().unwrap(); 582 583 handle_child_output(r, &output); 584 } 585 586 #[allow(unused_variables)] 587 fn _test_power_button(acpi: bool) { 588 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 589 let guest = Guest::new(Box::new(focal)); 590 let mut cmd = GuestCommand::new(&guest); 591 let api_socket = temp_api_path(&guest.tmp_dir); 592 593 #[cfg(target_arch = "x86_64")] 594 let kernel_path = direct_kernel_boot_path(); 595 #[cfg(target_arch = "aarch64")] 596 let kernel_path = if acpi { 597 edk2_path() 598 } else { 599 direct_kernel_boot_path() 600 }; 601 602 cmd.args(&["--cpus", "boot=1"]) 603 .args(&["--memory", "size=512M"]) 604 .args(&["--kernel", kernel_path.to_str().unwrap()]) 605 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 606 .capture_output() 607 .default_disks() 608 .default_net() 609 .args(&["--api-socket", &api_socket]); 610 611 let child = cmd.spawn().unwrap(); 612 613 let r = std::panic::catch_unwind(|| { 614 guest.wait_vm_boot(None).unwrap(); 615 assert!(remote_command(&api_socket, "power-button", None)); 616 }); 617 618 let output = child.wait_with_output().unwrap(); 619 assert!(output.status.success()); 620 handle_child_output(r, &output); 621 } 622 623 type PrepareNetDaemon = 624 dyn Fn(&TempDir, &str, Option<&str>, usize, bool) -> (std::process::Command, String); 625 626 fn test_vhost_user_net( 627 tap: Option<&str>, 628 num_queues: usize, 629 prepare_daemon: &PrepareNetDaemon, 630 generate_host_mac: bool, 631 client_mode_daemon: bool, 632 ) { 633 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 634 let guest = Guest::new(Box::new(focal)); 635 let api_socket = temp_api_path(&guest.tmp_dir); 636 637 let kernel_path = direct_kernel_boot_path(); 638 639 let host_mac = if generate_host_mac { 640 Some(MacAddr::local_random()) 641 } else { 642 None 643 }; 644 645 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 646 &guest.tmp_dir, 647 &guest.network.host_ip, 648 tap, 649 num_queues, 650 client_mode_daemon, 651 ); 652 653 let net_params = format!( 654 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={}", 655 guest.network.guest_mac, 656 vunet_socket_path, 657 num_queues, 658 if let Some(host_mac) = host_mac { 659 format!(",host_mac={}", host_mac) 660 } else { 661 "".to_owned() 662 }, 663 if client_mode_daemon { 664 "server" 665 } else { 666 "client" 667 }, 668 ); 669 670 let mut ch_command = GuestCommand::new(&guest); 671 ch_command 672 .args(&["--cpus", format!("boot={}", num_queues / 2).as_str()]) 673 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 674 .args(&["--kernel", kernel_path.to_str().unwrap()]) 675 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 676 .default_disks() 677 .args(&["--net", net_params.as_str()]) 678 .args(&["--api-socket", &api_socket]) 679 .capture_output(); 680 681 let mut daemon_child: std::process::Child; 682 let mut child: std::process::Child; 683 684 if client_mode_daemon { 685 child = ch_command.spawn().unwrap(); 686 // Make sure the VMM is waiting for the backend to connect 687 thread::sleep(std::time::Duration::new(10, 0)); 688 daemon_child = daemon_command.spawn().unwrap(); 689 } else { 690 daemon_child = daemon_command.spawn().unwrap(); 691 // Make sure the backend is waiting for the VMM to connect 692 thread::sleep(std::time::Duration::new(10, 0)); 693 child = ch_command.spawn().unwrap(); 694 } 695 696 let r = std::panic::catch_unwind(|| { 697 guest.wait_vm_boot(None).unwrap(); 698 699 if let Some(tap_name) = tap { 700 let tap_count = exec_host_command_output(&format!("ip link | grep -c {}", tap_name)); 701 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 702 } 703 704 if let Some(host_mac) = tap { 705 let mac_count = exec_host_command_output(&format!("ip link | grep -c {}", host_mac)); 706 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 707 } 708 709 // 1 network interface + default localhost ==> 2 interfaces 710 // It's important to note that this test is fully exercising the 711 // vhost-user-net implementation and the associated backend since 712 // it does not define any --net network interface. That means all 713 // the ssh communication in that test happens through the network 714 // interface backed by vhost-user-net. 715 assert_eq!( 716 guest 717 .ssh_command("ip -o link | wc -l") 718 .unwrap() 719 .trim() 720 .parse::<u32>() 721 .unwrap_or_default(), 722 2 723 ); 724 725 // The following pci devices will appear on guest with PCI-MSI 726 // interrupt vectors assigned. 727 // 1 virtio-console with 3 vectors: config, Rx, Tx 728 // 1 virtio-blk with 2 vectors: config, Request 729 // 1 virtio-blk with 2 vectors: config, Request 730 // 1 virtio-rng with 2 vectors: config, Request 731 // Since virtio-net has 2 queue pairs, its vectors is as follows: 732 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 733 // Based on the above, the total vectors should 14. 734 #[cfg(target_arch = "x86_64")] 735 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 736 #[cfg(target_arch = "aarch64")] 737 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 738 assert_eq!( 739 guest 740 .ssh_command(grep_cmd) 741 .unwrap() 742 .trim() 743 .parse::<u32>() 744 .unwrap_or_default(), 745 10 + (num_queues as u32) 746 ); 747 748 // ACPI feature is needed. 749 #[cfg(target_arch = "x86_64")] 750 { 751 guest.enable_memory_hotplug(); 752 753 // Add RAM to the VM 754 let desired_ram = 1024 << 20; 755 resize_command(&api_socket, None, Some(desired_ram), None); 756 757 thread::sleep(std::time::Duration::new(10, 0)); 758 759 // Here by simply checking the size (through ssh), we validate 760 // the connection is still working, which means vhost-user-net 761 // keeps working after the resize. 762 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 763 } 764 }); 765 766 let _ = child.kill(); 767 let output = child.wait_with_output().unwrap(); 768 769 thread::sleep(std::time::Duration::new(5, 0)); 770 let _ = daemon_child.kill(); 771 let _ = daemon_child.wait(); 772 773 handle_child_output(r, &output); 774 } 775 776 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 777 778 fn test_vhost_user_blk( 779 num_queues: usize, 780 readonly: bool, 781 direct: bool, 782 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 783 ) { 784 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 785 let guest = Guest::new(Box::new(focal)); 786 let api_socket = temp_api_path(&guest.tmp_dir); 787 788 let kernel_path = direct_kernel_boot_path(); 789 790 let (blk_params, daemon_child) = { 791 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 792 // Start the daemon 793 let (daemon_child, vubd_socket_path) = 794 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 795 796 ( 797 format!( 798 "vhost_user=true,socket={},num_queues={},queue_size=128", 799 vubd_socket_path, num_queues, 800 ), 801 Some(daemon_child), 802 ) 803 }; 804 805 let mut child = GuestCommand::new(&guest) 806 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 807 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 808 .args(&["--kernel", kernel_path.to_str().unwrap()]) 809 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 810 .args(&[ 811 "--disk", 812 format!( 813 "path={}", 814 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 815 ) 816 .as_str(), 817 format!( 818 "path={}", 819 guest.disk_config.disk(DiskType::CloudInit).unwrap() 820 ) 821 .as_str(), 822 blk_params.as_str(), 823 ]) 824 .default_net() 825 .args(&["--api-socket", &api_socket]) 826 .capture_output() 827 .spawn() 828 .unwrap(); 829 830 let r = std::panic::catch_unwind(|| { 831 guest.wait_vm_boot(None).unwrap(); 832 833 // Check both if /dev/vdc exists and if the block size is 16M. 834 assert_eq!( 835 guest 836 .ssh_command("lsblk | grep vdc | grep -c 16M") 837 .unwrap() 838 .trim() 839 .parse::<u32>() 840 .unwrap_or_default(), 841 1 842 ); 843 844 // Check if this block is RO or RW. 845 assert_eq!( 846 guest 847 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 848 .unwrap() 849 .trim() 850 .parse::<u32>() 851 .unwrap_or_default(), 852 readonly as u32 853 ); 854 855 // Check if the number of queues in /sys/block/vdc/mq matches the 856 // expected num_queues. 857 assert_eq!( 858 guest 859 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 860 .unwrap() 861 .trim() 862 .parse::<u32>() 863 .unwrap_or_default(), 864 num_queues as u32 865 ); 866 867 // Mount the device 868 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 869 guest.ssh_command("mkdir mount_image").unwrap(); 870 guest 871 .ssh_command( 872 format!( 873 "sudo mount -o {} -t ext4 /dev/vdc mount_image/", 874 mount_ro_rw_flag 875 ) 876 .as_str(), 877 ) 878 .unwrap(); 879 880 // Check the content of the block device. The file "foo" should 881 // contain "bar". 882 assert_eq!( 883 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 884 "bar" 885 ); 886 887 // ACPI feature is needed. 888 #[cfg(target_arch = "x86_64")] 889 { 890 guest.enable_memory_hotplug(); 891 892 // Add RAM to the VM 893 let desired_ram = 1024 << 20; 894 resize_command(&api_socket, None, Some(desired_ram), None); 895 896 thread::sleep(std::time::Duration::new(10, 0)); 897 898 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 899 900 // Check again the content of the block device after the resize 901 // has been performed. 902 assert_eq!( 903 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 904 "bar" 905 ); 906 } 907 908 // Unmount the device 909 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 910 guest.ssh_command("rm -r mount_image").unwrap(); 911 }); 912 913 let _ = child.kill(); 914 let output = child.wait_with_output().unwrap(); 915 916 if let Some(mut daemon_child) = daemon_child { 917 thread::sleep(std::time::Duration::new(5, 0)); 918 let _ = daemon_child.kill(); 919 let _ = daemon_child.wait(); 920 } 921 922 handle_child_output(r, &output); 923 } 924 925 fn test_boot_from_vhost_user_blk( 926 num_queues: usize, 927 readonly: bool, 928 direct: bool, 929 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 930 ) { 931 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 932 let guest = Guest::new(Box::new(focal)); 933 934 let kernel_path = direct_kernel_boot_path(); 935 936 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 937 938 let (blk_boot_params, daemon_child) = { 939 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 940 // Start the daemon 941 let (daemon_child, vubd_socket_path) = prepare_daemon( 942 &guest.tmp_dir, 943 disk_path.as_str(), 944 num_queues, 945 readonly, 946 direct, 947 ); 948 949 ( 950 format!( 951 "vhost_user=true,socket={},num_queues={},queue_size=128", 952 vubd_socket_path, num_queues, 953 ), 954 Some(daemon_child), 955 ) 956 }; 957 958 let mut child = GuestCommand::new(&guest) 959 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 960 .args(&["--memory", "size=512M,shared=on"]) 961 .args(&["--kernel", kernel_path.to_str().unwrap()]) 962 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 963 .args(&[ 964 "--disk", 965 blk_boot_params.as_str(), 966 format!( 967 "path={}", 968 guest.disk_config.disk(DiskType::CloudInit).unwrap() 969 ) 970 .as_str(), 971 ]) 972 .default_net() 973 .capture_output() 974 .spawn() 975 .unwrap(); 976 977 let r = std::panic::catch_unwind(|| { 978 guest.wait_vm_boot(None).unwrap(); 979 980 // Just check the VM booted correctly. 981 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 982 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 983 }); 984 let _ = child.kill(); 985 let output = child.wait_with_output().unwrap(); 986 987 if let Some(mut daemon_child) = daemon_child { 988 thread::sleep(std::time::Duration::new(5, 0)); 989 let _ = daemon_child.kill(); 990 let _ = daemon_child.wait(); 991 } 992 993 handle_child_output(r, &output); 994 } 995 996 fn test_virtio_fs( 997 dax: bool, 998 cache_size: Option<u64>, 999 virtiofsd_cache: &str, 1000 prepare_daemon: &dyn Fn(&TempDir, &str, &str) -> (std::process::Child, String), 1001 hotplug: bool, 1002 pci_segment: Option<u16>, 1003 ) { 1004 #[cfg(target_arch = "aarch64")] 1005 let focal_image = if hotplug { 1006 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1007 } else { 1008 FOCAL_IMAGE_NAME.to_string() 1009 }; 1010 #[cfg(target_arch = "x86_64")] 1011 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1012 let focal = UbuntuDiskConfig::new(focal_image); 1013 let guest = Guest::new(Box::new(focal)); 1014 let api_socket = temp_api_path(&guest.tmp_dir); 1015 1016 let mut workload_path = dirs::home_dir().unwrap(); 1017 workload_path.push("workloads"); 1018 1019 let mut shared_dir = workload_path; 1020 shared_dir.push("shared_dir"); 1021 1022 #[cfg(target_arch = "x86_64")] 1023 let kernel_path = direct_kernel_boot_path(); 1024 #[cfg(target_arch = "aarch64")] 1025 let kernel_path = if hotplug { 1026 edk2_path() 1027 } else { 1028 direct_kernel_boot_path() 1029 }; 1030 1031 let (dax_vmm_param, dax_mount_param) = if dax { ("on", "-o dax") } else { ("off", "") }; 1032 let cache_size_vmm_param = if let Some(cache) = cache_size { 1033 format!(",cache_size={}", cache) 1034 } else { 1035 "".to_string() 1036 }; 1037 1038 let (mut daemon_child, virtiofsd_socket_path) = prepare_daemon( 1039 &guest.tmp_dir, 1040 shared_dir.to_str().unwrap(), 1041 virtiofsd_cache, 1042 ); 1043 1044 let mut guest_command = GuestCommand::new(&guest); 1045 guest_command 1046 .args(&["--cpus", "boot=1"]) 1047 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1048 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1049 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1050 .default_disks() 1051 .default_net() 1052 .args(&["--api-socket", &api_socket]); 1053 if pci_segment.is_some() { 1054 guest_command.args(&["--platform", "num_pci_segments=16"]); 1055 } 1056 1057 let fs_params = format!( 1058 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024,dax={}{}{}", 1059 virtiofsd_socket_path, 1060 dax_vmm_param, 1061 cache_size_vmm_param, 1062 if let Some(pci_segment) = pci_segment { 1063 format!(",pci_segment={}", pci_segment) 1064 } else { 1065 "".to_owned() 1066 } 1067 ); 1068 1069 if !hotplug { 1070 guest_command.args(&["--fs", fs_params.as_str()]); 1071 } 1072 1073 let mut child = guest_command.capture_output().spawn().unwrap(); 1074 1075 let r = std::panic::catch_unwind(|| { 1076 guest.wait_vm_boot(None).unwrap(); 1077 1078 if hotplug { 1079 // Add fs to the VM 1080 let (cmd_success, cmd_output) = 1081 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1082 assert!(cmd_success); 1083 1084 if let Some(pci_segment) = pci_segment { 1085 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1086 "{{\"id\":\"myfs0\",\"bdf\":\"{:04x}:00:01.0\"}}", 1087 pci_segment 1088 ))); 1089 } else { 1090 assert!(String::from_utf8_lossy(&cmd_output) 1091 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1092 } 1093 1094 thread::sleep(std::time::Duration::new(10, 0)); 1095 } 1096 1097 // Mount shared directory through virtio_fs filesystem 1098 let mount_cmd = format!( 1099 "mkdir -p mount_dir && \ 1100 sudo mount -t virtiofs {} myfs mount_dir/", 1101 dax_mount_param 1102 ); 1103 guest.ssh_command(&mount_cmd).unwrap(); 1104 1105 assert!(guest 1106 .valid_virtio_fs_cache_size(dax, cache_size) 1107 .unwrap_or_default()); 1108 1109 // Check file1 exists and its content is "foo" 1110 assert_eq!( 1111 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1112 "foo" 1113 ); 1114 // Check file2 does not exist 1115 guest 1116 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1117 .unwrap(); 1118 1119 // Check file3 exists and its content is "bar" 1120 assert_eq!( 1121 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1122 "bar" 1123 ); 1124 1125 // ACPI feature is needed. 1126 #[cfg(target_arch = "x86_64")] 1127 { 1128 guest.enable_memory_hotplug(); 1129 1130 // Add RAM to the VM 1131 let desired_ram = 1024 << 20; 1132 resize_command(&api_socket, None, Some(desired_ram), None); 1133 1134 thread::sleep(std::time::Duration::new(30, 0)); 1135 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1136 1137 // After the resize, check again that file1 exists and its 1138 // content is "foo". 1139 assert_eq!( 1140 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1141 "foo" 1142 ); 1143 } 1144 1145 if hotplug { 1146 // Remove from VM 1147 guest.ssh_command("sudo umount mount_dir").unwrap(); 1148 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1149 } 1150 }); 1151 1152 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1153 thread::sleep(std::time::Duration::new(10, 0)); 1154 let (daemon_child, virtiofsd_socket_path) = prepare_daemon( 1155 &guest.tmp_dir, 1156 shared_dir.to_str().unwrap(), 1157 virtiofsd_cache, 1158 ); 1159 1160 let r = std::panic::catch_unwind(|| { 1161 thread::sleep(std::time::Duration::new(10, 0)); 1162 let fs_params = format!( 1163 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024,dax={}{}{}", 1164 virtiofsd_socket_path, 1165 dax_vmm_param, 1166 cache_size_vmm_param, 1167 if let Some(pci_segment) = pci_segment { 1168 format!(",pci_segment={}", pci_segment) 1169 } else { 1170 "".to_owned() 1171 } 1172 ); 1173 1174 // Add back and check it works 1175 let (cmd_success, cmd_output) = 1176 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1177 assert!(cmd_success); 1178 if let Some(pci_segment) = pci_segment { 1179 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1180 "{{\"id\":\"myfs0\",\"bdf\":\"{:04x}:00:01.0\"}}", 1181 pci_segment 1182 ))); 1183 } else { 1184 assert!(String::from_utf8_lossy(&cmd_output) 1185 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1186 } 1187 1188 thread::sleep(std::time::Duration::new(10, 0)); 1189 // Mount shared directory through virtio_fs filesystem 1190 let mount_cmd = format!( 1191 "mkdir -p mount_dir && \ 1192 sudo mount -t virtiofs {} myfs mount_dir/", 1193 dax_mount_param 1194 ); 1195 guest.ssh_command(&mount_cmd).unwrap(); 1196 // Check file1 exists and its content is "foo" 1197 assert_eq!( 1198 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1199 "foo" 1200 ); 1201 }); 1202 1203 (r, Some(daemon_child)) 1204 } else { 1205 (r, None) 1206 }; 1207 1208 let _ = child.kill(); 1209 let output = child.wait_with_output().unwrap(); 1210 1211 let _ = daemon_child.kill(); 1212 let _ = daemon_child.wait(); 1213 1214 if let Some(mut daemon_child) = hotplug_daemon_child { 1215 let _ = daemon_child.kill(); 1216 let _ = daemon_child.wait(); 1217 } 1218 1219 handle_child_output(r, &output); 1220 } 1221 1222 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1223 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1224 let guest = Guest::new(Box::new(focal)); 1225 1226 let kernel_path = direct_kernel_boot_path(); 1227 1228 let pmem_temp_file = TempFile::new().unwrap(); 1229 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1230 1231 std::process::Command::new("mkfs.ext4") 1232 .arg(pmem_temp_file.as_path()) 1233 .output() 1234 .expect("Expect creating disk image to succeed"); 1235 1236 let mut child = GuestCommand::new(&guest) 1237 .args(&["--cpus", "boot=1"]) 1238 .args(&["--memory", "size=512M"]) 1239 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1240 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1241 .default_disks() 1242 .default_net() 1243 .args(&[ 1244 "--pmem", 1245 format!( 1246 "file={}{}{}", 1247 pmem_temp_file.as_path().to_str().unwrap(), 1248 if specify_size { ",size=128M" } else { "" }, 1249 if discard_writes { 1250 ",discard_writes=on" 1251 } else { 1252 "" 1253 } 1254 ) 1255 .as_str(), 1256 ]) 1257 .capture_output() 1258 .spawn() 1259 .unwrap(); 1260 1261 let r = std::panic::catch_unwind(|| { 1262 guest.wait_vm_boot(None).unwrap(); 1263 1264 // Check for the presence of /dev/pmem0 1265 assert_eq!( 1266 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1267 "/dev/pmem0" 1268 ); 1269 1270 // Check changes persist after reboot 1271 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1272 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1273 guest 1274 .ssh_command("echo test123 | sudo tee /mnt/test") 1275 .unwrap(); 1276 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1277 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1278 1279 guest.reboot_linux(0, None); 1280 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1281 assert_eq!( 1282 guest 1283 .ssh_command("sudo cat /mnt/test || true") 1284 .unwrap() 1285 .trim(), 1286 if discard_writes { "" } else { "test123" } 1287 ); 1288 }); 1289 1290 let _ = child.kill(); 1291 let output = child.wait_with_output().unwrap(); 1292 1293 handle_child_output(r, &output); 1294 } 1295 1296 fn get_fd_count(pid: u32) -> usize { 1297 fs::read_dir(format!("/proc/{}/fd", pid)).unwrap().count() 1298 } 1299 1300 fn _test_virtio_vsock(hotplug: bool) { 1301 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1302 let guest = Guest::new(Box::new(focal)); 1303 1304 #[cfg(target_arch = "x86_64")] 1305 let kernel_path = direct_kernel_boot_path(); 1306 #[cfg(target_arch = "aarch64")] 1307 let kernel_path = if hotplug { 1308 edk2_path() 1309 } else { 1310 direct_kernel_boot_path() 1311 }; 1312 1313 let socket = temp_vsock_path(&guest.tmp_dir); 1314 let api_socket = temp_api_path(&guest.tmp_dir); 1315 1316 let mut cmd = GuestCommand::new(&guest); 1317 cmd.args(&["--api-socket", &api_socket]); 1318 cmd.args(&["--cpus", "boot=1"]); 1319 cmd.args(&["--memory", "size=512M"]); 1320 cmd.args(&["--kernel", kernel_path.to_str().unwrap()]); 1321 cmd.args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1322 cmd.default_disks(); 1323 cmd.default_net(); 1324 1325 if !hotplug { 1326 cmd.args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]); 1327 } 1328 1329 let mut child = cmd.capture_output().spawn().unwrap(); 1330 1331 let r = std::panic::catch_unwind(|| { 1332 guest.wait_vm_boot(None).unwrap(); 1333 1334 if hotplug { 1335 let (cmd_success, cmd_output) = remote_command_w_output( 1336 &api_socket, 1337 "add-vsock", 1338 Some(format!("cid=3,socket={},id=test0", socket).as_str()), 1339 ); 1340 assert!(cmd_success); 1341 assert!(String::from_utf8_lossy(&cmd_output) 1342 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1343 thread::sleep(std::time::Duration::new(10, 0)); 1344 // Check adding a second one fails 1345 assert!(!remote_command( 1346 &api_socket, 1347 "add-vsock", 1348 Some("cid=1234,socket=/tmp/fail") 1349 )); 1350 } 1351 1352 // Validate vsock works as expected. 1353 guest.check_vsock(socket.as_str()); 1354 guest.reboot_linux(0, None); 1355 // Validate vsock still works after a reboot. 1356 guest.check_vsock(socket.as_str()); 1357 1358 if hotplug { 1359 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1360 } 1361 }); 1362 1363 let _ = child.kill(); 1364 let output = child.wait_with_output().unwrap(); 1365 1366 handle_child_output(r, &output); 1367 } 1368 1369 fn get_ksm_pages_shared() -> u32 { 1370 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1371 .unwrap() 1372 .trim() 1373 .parse::<u32>() 1374 .unwrap() 1375 } 1376 1377 fn test_memory_mergeable(mergeable: bool) { 1378 let memory_param = if mergeable { 1379 "mergeable=on" 1380 } else { 1381 "mergeable=off" 1382 }; 1383 1384 // We are assuming the rest of the system in our CI is not using mergeable memeory 1385 let ksm_ps_init = get_ksm_pages_shared(); 1386 assert!(ksm_ps_init == 0); 1387 1388 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1389 let guest1 = Guest::new(Box::new(focal1)); 1390 let mut child1 = GuestCommand::new(&guest1) 1391 .args(&["--cpus", "boot=1"]) 1392 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1393 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1394 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1395 .default_disks() 1396 .args(&["--net", guest1.default_net_string().as_str()]) 1397 .args(&["--serial", "tty", "--console", "off"]) 1398 .capture_output() 1399 .spawn() 1400 .unwrap(); 1401 1402 let r = std::panic::catch_unwind(|| { 1403 guest1.wait_vm_boot(None).unwrap(); 1404 }); 1405 if r.is_err() { 1406 let _ = child1.kill(); 1407 let output = child1.wait_with_output().unwrap(); 1408 handle_child_output(r, &output); 1409 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1410 } 1411 1412 let ksm_ps_guest1 = get_ksm_pages_shared(); 1413 1414 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1415 let guest2 = Guest::new(Box::new(focal2)); 1416 let mut child2 = GuestCommand::new(&guest2) 1417 .args(&["--cpus", "boot=1"]) 1418 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1419 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1420 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1421 .default_disks() 1422 .args(&["--net", guest2.default_net_string().as_str()]) 1423 .args(&["--serial", "tty", "--console", "off"]) 1424 .capture_output() 1425 .spawn() 1426 .unwrap(); 1427 1428 let r = std::panic::catch_unwind(|| { 1429 guest2.wait_vm_boot(None).unwrap(); 1430 let ksm_ps_guest2 = get_ksm_pages_shared(); 1431 1432 if mergeable { 1433 println!( 1434 "ksm pages_shared after vm1 booted '{}', ksm pages_shared after vm2 booted '{}'", 1435 ksm_ps_guest1, ksm_ps_guest2 1436 ); 1437 // We are expecting the number of shared pages to increase as the number of VM increases 1438 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1439 } else { 1440 assert!(ksm_ps_guest1 == 0); 1441 assert!(ksm_ps_guest2 == 0); 1442 } 1443 }); 1444 1445 let _ = child1.kill(); 1446 let _ = child2.kill(); 1447 1448 let output = child1.wait_with_output().unwrap(); 1449 child2.wait().unwrap(); 1450 1451 handle_child_output(r, &output); 1452 } 1453 1454 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 1455 let smaps = fs::File::open(format!("/proc/{}/smaps", pid)).unwrap(); 1456 let reader = io::BufReader::new(smaps); 1457 1458 let mut skip_map: bool = false; 1459 let mut region_name: String = "".to_string(); 1460 let mut region_maps = HashMap::new(); 1461 for line in reader.lines() { 1462 let l = line.unwrap(); 1463 1464 if l.contains('-') { 1465 let values: Vec<&str> = l.split_whitespace().collect(); 1466 region_name = values.last().unwrap().trim().to_string(); 1467 if region_name == "0" { 1468 region_name = "anonymous".to_string() 1469 } 1470 } 1471 1472 // Each section begins with something that looks like: 1473 // Size: 2184 kB 1474 if l.starts_with("Size:") { 1475 let values: Vec<&str> = l.split_whitespace().collect(); 1476 let map_size = values[1].parse::<u32>().unwrap(); 1477 // We skip the assigned guest RAM map, its RSS is only 1478 // dependent on the guest actual memory usage. 1479 // Everything else can be added to the VMM overhead. 1480 skip_map = map_size >= guest_memory_size; 1481 continue; 1482 } 1483 1484 // If this is a map we're taking into account, then we only 1485 // count the RSS. The sum of all counted RSS is the VMM overhead. 1486 if !skip_map && l.starts_with("Rss:") { 1487 let values: Vec<&str> = l.split_whitespace().collect(); 1488 let value = values[1].trim().parse::<u32>().unwrap(); 1489 *region_maps.entry(region_name.clone()).or_insert(0) += value; 1490 } 1491 } 1492 1493 region_maps 1494 } 1495 1496 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 1497 let mut total = 0; 1498 1499 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 1500 eprintln!("{}: {}", region_name, value); 1501 total += value; 1502 } 1503 1504 total 1505 } 1506 1507 fn process_rss_kib(pid: u32) -> usize { 1508 let command = format!("ps -q {} -o rss=", pid); 1509 let rss = exec_host_command_output(&command); 1510 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 1511 } 1512 1513 // 10MB is our maximum accepted overhead. 1514 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 1515 1516 #[derive(PartialEq, PartialOrd)] 1517 struct Counters { 1518 rx_bytes: u64, 1519 rx_frames: u64, 1520 tx_bytes: u64, 1521 tx_frames: u64, 1522 read_bytes: u64, 1523 write_bytes: u64, 1524 read_ops: u64, 1525 write_ops: u64, 1526 } 1527 1528 fn get_counters(api_socket: &str) -> Counters { 1529 // Get counters 1530 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 1531 assert!(cmd_success); 1532 1533 let counters: HashMap<&str, HashMap<&str, u64>> = 1534 serde_json::from_slice(&cmd_output).unwrap_or_default(); 1535 1536 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 1537 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 1538 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 1539 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 1540 1541 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 1542 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 1543 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 1544 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 1545 1546 Counters { 1547 rx_bytes, 1548 rx_frames, 1549 tx_bytes, 1550 tx_frames, 1551 read_bytes, 1552 write_bytes, 1553 read_ops, 1554 write_ops, 1555 } 1556 } 1557 1558 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 1559 let (tx, rx) = mpsc::channel::<String>(); 1560 thread::spawn(move || loop { 1561 thread::sleep(std::time::Duration::new(1, 0)); 1562 let mut buf = [0; 512]; 1563 match pty.read(&mut buf) { 1564 Ok(_) => { 1565 let output = std::str::from_utf8(&buf).unwrap().to_string(); 1566 match tx.send(output) { 1567 Ok(_) => (), 1568 Err(_) => break, 1569 } 1570 } 1571 Err(_) => break, 1572 } 1573 }); 1574 rx 1575 } 1576 1577 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 1578 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1579 assert!(cmd_success); 1580 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1581 assert_eq!("Pty", info["config"][pty_type]["mode"]); 1582 PathBuf::from( 1583 info["config"][pty_type]["file"] 1584 .as_str() 1585 .expect("Missing pty path"), 1586 ) 1587 } 1588 1589 // VFIO test network setup. 1590 // We reserve a different IP class for it: 172.18.0.0/24. 1591 #[cfg(target_arch = "x86_64")] 1592 fn setup_vfio_network_interfaces() { 1593 // 'vfio-br0' 1594 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 1595 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 1596 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 1597 // 'vfio-tap0' 1598 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 1599 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 1600 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 1601 // 'vfio-tap1' 1602 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 1603 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 1604 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 1605 // 'vfio-tap2' 1606 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 1607 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 1608 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 1609 // 'vfio-tap3' 1610 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 1611 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 1612 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 1613 } 1614 1615 // Tear VFIO test network down 1616 #[cfg(target_arch = "x86_64")] 1617 fn cleanup_vfio_network_interfaces() { 1618 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 1619 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 1620 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 1621 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 1622 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 1623 } 1624 1625 fn balloon_size(api_socket: &str) -> u64 { 1626 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1627 assert!(cmd_success); 1628 1629 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1630 let total_mem = &info["config"]["memory"]["size"] 1631 .to_string() 1632 .parse::<u64>() 1633 .unwrap(); 1634 let actual_mem = &info["memory_actual_size"] 1635 .to_string() 1636 .parse::<u64>() 1637 .unwrap(); 1638 total_mem - actual_mem 1639 } 1640 1641 // This test validates that it can find the virtio-iommu device at first. 1642 // It also verifies that both disks and the network card are attached to 1643 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 1644 // The last interesting part of this test is that it exercises the network 1645 // interface attached to the virtual IOMMU since this is the one used to 1646 // send all commands through SSH. 1647 fn _test_virtio_iommu(acpi: bool) { 1648 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 1649 // Focal image is still old. 1650 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 1651 // the kernel binary has been updated. 1652 #[cfg(target_arch = "aarch64")] 1653 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 1654 #[cfg(target_arch = "x86_64")] 1655 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1656 let focal = UbuntuDiskConfig::new(focal_image); 1657 let guest = Guest::new(Box::new(focal)); 1658 1659 #[cfg(target_arch = "x86_64")] 1660 let kernel_path = direct_kernel_boot_path(); 1661 #[cfg(target_arch = "aarch64")] 1662 let kernel_path = if acpi { 1663 edk2_path() 1664 } else { 1665 direct_kernel_boot_path() 1666 }; 1667 1668 let mut child = GuestCommand::new(&guest) 1669 .args(&["--cpus", "boot=1"]) 1670 .args(&["--memory", "size=512M"]) 1671 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1672 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1673 .args(&[ 1674 "--disk", 1675 format!( 1676 "path={},iommu=on", 1677 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1678 ) 1679 .as_str(), 1680 format!( 1681 "path={},iommu=on", 1682 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1683 ) 1684 .as_str(), 1685 ]) 1686 .args(&["--net", guest.default_net_string_w_iommu().as_str()]) 1687 .capture_output() 1688 .spawn() 1689 .unwrap(); 1690 1691 let r = std::panic::catch_unwind(|| { 1692 guest.wait_vm_boot(None).unwrap(); 1693 1694 // Verify the virtio-iommu device is present. 1695 assert!(guest 1696 .does_device_vendor_pair_match("0x1057", "0x1af4") 1697 .unwrap_or_default()); 1698 1699 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 1700 // different with ACPI. 1701 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 1702 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 1703 // and appear under folder '/sys/kernel/iommu_groups/'. 1704 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 1705 // which is the console. The first disk "0000:00:02.0" is in group '1'. 1706 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 1707 // contains "0000:00:02.0" which is the first disk. 1708 // 1709 // Verify the iommu group of the first disk. 1710 let iommu_group = if acpi { 0 } else { 1 }; 1711 assert_eq!( 1712 guest 1713 .ssh_command( 1714 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1715 ) 1716 .unwrap() 1717 .trim(), 1718 "0000:00:02.0" 1719 ); 1720 1721 // Verify the iommu group of the second disk. 1722 let iommu_group = if acpi { 1 } else { 2 }; 1723 assert_eq!( 1724 guest 1725 .ssh_command( 1726 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1727 ) 1728 .unwrap() 1729 .trim(), 1730 "0000:00:03.0" 1731 ); 1732 1733 // Verify the iommu group of the network card. 1734 let iommu_group = if acpi { 2 } else { 3 }; 1735 assert_eq!( 1736 guest 1737 .ssh_command( 1738 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1739 ) 1740 .unwrap() 1741 .trim(), 1742 "0000:00:04.0" 1743 ); 1744 }); 1745 1746 let _ = child.kill(); 1747 let output = child.wait_with_output().unwrap(); 1748 1749 handle_child_output(r, &output); 1750 } 1751 1752 mod parallel { 1753 use std::io::SeekFrom; 1754 1755 use crate::*; 1756 1757 #[test] 1758 #[cfg(target_arch = "x86_64")] 1759 fn test_bionic_hypervisor_fw() { 1760 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), BIONIC_IMAGE_NAME) 1761 } 1762 1763 #[test] 1764 #[cfg(target_arch = "x86_64")] 1765 fn test_focal_hypervisor_fw() { 1766 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 1767 } 1768 1769 #[test] 1770 #[cfg(target_arch = "x86_64")] 1771 fn test_bionic_ovmf() { 1772 test_simple_launch(fw_path(FwType::Ovmf), BIONIC_IMAGE_NAME) 1773 } 1774 1775 #[test] 1776 #[cfg(target_arch = "x86_64")] 1777 fn test_focal_ovmf() { 1778 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 1779 } 1780 1781 #[cfg(target_arch = "x86_64")] 1782 fn test_simple_launch(fw_path: String, disk_path: &str) { 1783 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 1784 let guest = Guest::new(disk_config); 1785 1786 let mut child = GuestCommand::new(&guest) 1787 .args(&["--cpus", "boot=1"]) 1788 .args(&["--memory", "size=512M"]) 1789 .args(&["--kernel", fw_path.as_str()]) 1790 .default_disks() 1791 .default_net() 1792 .args(&["--serial", "tty", "--console", "off"]) 1793 .capture_output() 1794 .spawn() 1795 .unwrap(); 1796 1797 let r = std::panic::catch_unwind(|| { 1798 guest.wait_vm_boot(Some(120)).unwrap(); 1799 1800 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 1801 assert_eq!(guest.get_initial_apicid().unwrap_or(1), 0); 1802 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1803 assert!(guest.get_entropy().unwrap_or_default() >= 900); 1804 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 1805 }); 1806 1807 let _ = child.kill(); 1808 let output = child.wait_with_output().unwrap(); 1809 1810 handle_child_output(r, &output); 1811 } 1812 1813 #[test] 1814 fn test_multi_cpu() { 1815 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 1816 let guest = Guest::new(Box::new(bionic)); 1817 let mut cmd = GuestCommand::new(&guest); 1818 cmd.args(&["--cpus", "boot=2,max=4"]) 1819 .args(&["--memory", "size=512M"]) 1820 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1821 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1822 .capture_output() 1823 .default_disks() 1824 .default_net(); 1825 1826 let mut child = cmd.spawn().unwrap(); 1827 1828 let r = std::panic::catch_unwind(|| { 1829 guest.wait_vm_boot(Some(120)).unwrap(); 1830 1831 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 1832 1833 #[cfg(target_arch = "x86_64")] 1834 assert_eq!( 1835 guest 1836 .ssh_command(r#"dmesg | grep "smpboot: Allowing" | sed "s/\[\ *[0-9.]*\] //""#) 1837 .unwrap() 1838 .trim(), 1839 "smpboot: Allowing 4 CPUs, 2 hotplug CPUs" 1840 ); 1841 #[cfg(target_arch = "aarch64")] 1842 assert_eq!( 1843 guest 1844 .ssh_command(r#"dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""#) 1845 .unwrap() 1846 .trim(), 1847 "smp: Brought up 1 node, 2 CPUs" 1848 ); 1849 }); 1850 1851 let _ = child.kill(); 1852 let output = child.wait_with_output().unwrap(); 1853 1854 handle_child_output(r, &output); 1855 } 1856 1857 #[test] 1858 #[cfg(not(feature = "mshv"))] 1859 fn test_cpu_topology_421() { 1860 test_cpu_topology(4, 2, 1, false); 1861 } 1862 1863 #[test] 1864 #[cfg(not(feature = "mshv"))] 1865 fn test_cpu_topology_142() { 1866 test_cpu_topology(1, 4, 2, false); 1867 } 1868 1869 #[test] 1870 #[cfg(not(feature = "mshv"))] 1871 fn test_cpu_topology_262() { 1872 test_cpu_topology(2, 6, 2, false); 1873 } 1874 1875 #[test] 1876 #[cfg(target_arch = "x86_64")] 1877 #[cfg(not(feature = "mshv"))] 1878 fn test_cpu_physical_bits() { 1879 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1880 let guest = Guest::new(Box::new(focal)); 1881 let max_phys_bits: u8 = 36; 1882 let mut child = GuestCommand::new(&guest) 1883 .args(&["--cpus", &format!("max_phys_bits={}", max_phys_bits)]) 1884 .args(&["--memory", "size=512M"]) 1885 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1886 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1887 .default_disks() 1888 .default_net() 1889 .capture_output() 1890 .spawn() 1891 .unwrap(); 1892 1893 let r = std::panic::catch_unwind(|| { 1894 guest.wait_vm_boot(None).unwrap(); 1895 1896 assert!( 1897 guest 1898 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 1899 .unwrap() 1900 .trim() 1901 .parse::<u8>() 1902 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 1903 ); 1904 }); 1905 1906 let _ = child.kill(); 1907 let output = child.wait_with_output().unwrap(); 1908 1909 handle_child_output(r, &output); 1910 } 1911 1912 #[test] 1913 fn test_cpu_affinity() { 1914 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1915 let guest = Guest::new(Box::new(focal)); 1916 1917 // We need the host to have at least 4 CPUs if we want to be able 1918 // to run this test. 1919 let host_cpus_count = exec_host_command_output("nproc"); 1920 assert!( 1921 String::from_utf8_lossy(&host_cpus_count.stdout) 1922 .trim() 1923 .parse::<u8>() 1924 .unwrap_or(0) 1925 >= 4 1926 ); 1927 1928 let mut child = GuestCommand::new(&guest) 1929 .args(&["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 1930 .args(&["--memory", "size=512M"]) 1931 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1932 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1933 .default_disks() 1934 .default_net() 1935 .capture_output() 1936 .spawn() 1937 .unwrap(); 1938 1939 let r = std::panic::catch_unwind(|| { 1940 guest.wait_vm_boot(None).unwrap(); 1941 let pid = child.id(); 1942 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"", pid).as_str()); 1943 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 1944 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"", pid).as_str()); 1945 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 1946 }); 1947 1948 let _ = child.kill(); 1949 let output = child.wait_with_output().unwrap(); 1950 1951 handle_child_output(r, &output); 1952 } 1953 1954 #[test] 1955 #[cfg(not(feature = "mshv"))] 1956 fn test_large_vm() { 1957 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1958 let guest = Guest::new(Box::new(focal)); 1959 let mut cmd = GuestCommand::new(&guest); 1960 cmd.args(&["--cpus", "boot=48"]) 1961 .args(&["--memory", "size=5120M"]) 1962 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1963 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1964 .args(&["--serial", "tty"]) 1965 .args(&["--console", "off"]) 1966 .capture_output() 1967 .default_disks() 1968 .default_net(); 1969 1970 let mut child = cmd.spawn().unwrap(); 1971 1972 guest.wait_vm_boot(None).unwrap(); 1973 1974 let r = std::panic::catch_unwind(|| { 1975 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 1976 assert_eq!( 1977 guest 1978 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1979 .unwrap() 1980 .trim(), 1981 "0-47" 1982 ); 1983 1984 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 1985 }); 1986 1987 let _ = child.kill(); 1988 let output = child.wait_with_output().unwrap(); 1989 1990 handle_child_output(r, &output); 1991 } 1992 1993 #[test] 1994 #[cfg(not(feature = "mshv"))] 1995 fn test_huge_memory() { 1996 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1997 let guest = Guest::new(Box::new(focal)); 1998 let mut cmd = GuestCommand::new(&guest); 1999 cmd.args(&["--cpus", "boot=1"]) 2000 .args(&["--memory", "size=128G"]) 2001 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2002 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2003 .capture_output() 2004 .default_disks() 2005 .default_net(); 2006 2007 let mut child = cmd.spawn().unwrap(); 2008 2009 guest.wait_vm_boot(Some(120)).unwrap(); 2010 2011 let r = std::panic::catch_unwind(|| { 2012 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2013 }); 2014 2015 let _ = child.kill(); 2016 let output = child.wait_with_output().unwrap(); 2017 2018 handle_child_output(r, &output); 2019 } 2020 2021 #[test] 2022 fn test_power_button() { 2023 _test_power_button(false); 2024 } 2025 2026 #[test] 2027 #[cfg(not(feature = "mshv"))] 2028 fn test_user_defined_memory_regions() { 2029 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2030 let guest = Guest::new(Box::new(focal)); 2031 let api_socket = temp_api_path(&guest.tmp_dir); 2032 2033 let kernel_path = direct_kernel_boot_path(); 2034 2035 let mut child = GuestCommand::new(&guest) 2036 .args(&["--cpus", "boot=1"]) 2037 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 2038 .args(&[ 2039 "--memory-zone", 2040 "id=mem0,size=1G,hotplug_size=2G", 2041 "id=mem1,size=1G,file=/dev/shm", 2042 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2043 ]) 2044 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2045 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2046 .args(&["--api-socket", &api_socket]) 2047 .capture_output() 2048 .default_disks() 2049 .default_net() 2050 .spawn() 2051 .unwrap(); 2052 2053 let r = std::panic::catch_unwind(|| { 2054 guest.wait_vm_boot(None).unwrap(); 2055 2056 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2057 2058 guest.enable_memory_hotplug(); 2059 2060 resize_zone_command(&api_socket, "mem0", "3G"); 2061 thread::sleep(std::time::Duration::new(5, 0)); 2062 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2063 resize_zone_command(&api_socket, "mem2", "3G"); 2064 thread::sleep(std::time::Duration::new(5, 0)); 2065 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2066 resize_zone_command(&api_socket, "mem0", "2G"); 2067 thread::sleep(std::time::Duration::new(5, 0)); 2068 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2069 resize_zone_command(&api_socket, "mem2", "2G"); 2070 thread::sleep(std::time::Duration::new(5, 0)); 2071 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2072 2073 guest.reboot_linux(0, None); 2074 2075 // Check the amount of RAM after reboot 2076 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2077 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2078 2079 // Check if we can still resize down to the initial 'boot'size 2080 resize_zone_command(&api_socket, "mem0", "1G"); 2081 thread::sleep(std::time::Duration::new(5, 0)); 2082 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2083 resize_zone_command(&api_socket, "mem2", "1G"); 2084 thread::sleep(std::time::Duration::new(5, 0)); 2085 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2086 }); 2087 2088 let _ = child.kill(); 2089 let output = child.wait_with_output().unwrap(); 2090 2091 handle_child_output(r, &output); 2092 } 2093 2094 #[test] 2095 #[cfg(not(feature = "mshv"))] 2096 fn test_guest_numa_nodes() { 2097 _test_guest_numa_nodes(false); 2098 } 2099 2100 #[test] 2101 #[cfg(target_arch = "x86_64")] 2102 fn test_iommu_segments() { 2103 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2104 let focal = UbuntuDiskConfig::new(focal_image); 2105 let guest = Guest::new(Box::new(focal)); 2106 2107 // Prepare another disk file for the virtio-disk device 2108 let test_disk_path = String::from( 2109 guest 2110 .tmp_dir 2111 .as_path() 2112 .join("test-disk.raw") 2113 .to_str() 2114 .unwrap(), 2115 ); 2116 assert!( 2117 exec_host_command_status(format!("truncate {} -s 4M", test_disk_path).as_str()) 2118 .success() 2119 ); 2120 assert!( 2121 exec_host_command_status(format!("mkfs.ext4 {}", test_disk_path).as_str()).success() 2122 ); 2123 2124 let api_socket = temp_api_path(&guest.tmp_dir); 2125 let mut cmd = GuestCommand::new(&guest); 2126 2127 cmd.args(&["--cpus", "boot=1"]) 2128 .args(&["--api-socket", &api_socket]) 2129 .args(&["--memory", "size=512M"]) 2130 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2131 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2132 .args(&["--platform", "num_pci_segments=16,iommu_segments=[1]"]) 2133 .default_disks() 2134 .capture_output() 2135 .default_net(); 2136 2137 let mut child = cmd.spawn().unwrap(); 2138 2139 guest.wait_vm_boot(None).unwrap(); 2140 2141 let r = std::panic::catch_unwind(|| { 2142 let (cmd_success, cmd_output) = remote_command_w_output( 2143 &api_socket, 2144 "add-disk", 2145 Some( 2146 format!( 2147 "path={},id=test0,pci_segment=1,iommu=on", 2148 test_disk_path.as_str() 2149 ) 2150 .as_str(), 2151 ), 2152 ); 2153 assert!(cmd_success); 2154 assert!(String::from_utf8_lossy(&cmd_output) 2155 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2156 2157 // Check IOMMU setup 2158 assert!(guest 2159 .does_device_vendor_pair_match("0x1057", "0x1af4") 2160 .unwrap_or_default()); 2161 assert_eq!( 2162 guest 2163 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2164 .unwrap() 2165 .trim(), 2166 "0001:00:01.0" 2167 ); 2168 }); 2169 2170 let _ = child.kill(); 2171 let output = child.wait_with_output().unwrap(); 2172 2173 handle_child_output(r, &output); 2174 } 2175 2176 #[test] 2177 fn test_pci_msi() { 2178 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2179 let guest = Guest::new(Box::new(focal)); 2180 let mut cmd = GuestCommand::new(&guest); 2181 cmd.args(&["--cpus", "boot=1"]) 2182 .args(&["--memory", "size=512M"]) 2183 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2184 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2185 .capture_output() 2186 .default_disks() 2187 .default_net(); 2188 2189 let mut child = cmd.spawn().unwrap(); 2190 2191 guest.wait_vm_boot(None).unwrap(); 2192 2193 #[cfg(target_arch = "x86_64")] 2194 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2195 #[cfg(target_arch = "aarch64")] 2196 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2197 2198 let r = std::panic::catch_unwind(|| { 2199 assert_eq!( 2200 guest 2201 .ssh_command(grep_cmd) 2202 .unwrap() 2203 .trim() 2204 .parse::<u32>() 2205 .unwrap_or_default(), 2206 12 2207 ); 2208 }); 2209 2210 let _ = child.kill(); 2211 let output = child.wait_with_output().unwrap(); 2212 2213 handle_child_output(r, &output); 2214 } 2215 2216 #[test] 2217 fn test_virtio_net_ctrl_queue() { 2218 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2219 let guest = Guest::new(Box::new(focal)); 2220 let mut cmd = GuestCommand::new(&guest); 2221 cmd.args(&["--cpus", "boot=1"]) 2222 .args(&["--memory", "size=512M"]) 2223 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2224 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2225 .capture_output() 2226 .default_disks() 2227 .default_net(); 2228 2229 let mut child = cmd.spawn().unwrap(); 2230 2231 guest.wait_vm_boot(None).unwrap(); 2232 2233 #[cfg(target_arch = "aarch64")] 2234 let iface = "enp0s4"; 2235 #[cfg(target_arch = "x86_64")] 2236 let iface = "ens4"; 2237 2238 let r = std::panic::catch_unwind(|| { 2239 assert_eq!( 2240 guest 2241 .ssh_command( 2242 format!("sudo ethtool -K {} rx-gro-hw off && echo success", iface).as_str() 2243 ) 2244 .unwrap() 2245 .trim(), 2246 "success" 2247 ); 2248 }); 2249 2250 let _ = child.kill(); 2251 let output = child.wait_with_output().unwrap(); 2252 2253 handle_child_output(r, &output); 2254 } 2255 2256 #[test] 2257 #[cfg(not(feature = "mshv"))] 2258 fn test_pci_multiple_segments() { 2259 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2260 let guest = Guest::new(Box::new(focal)); 2261 2262 // Prepare another disk file for the virtio-disk device 2263 let test_disk_path = String::from( 2264 guest 2265 .tmp_dir 2266 .as_path() 2267 .join("test-disk.raw") 2268 .to_str() 2269 .unwrap(), 2270 ); 2271 assert!( 2272 exec_host_command_status(format!("truncate {} -s 4M", test_disk_path).as_str()) 2273 .success() 2274 ); 2275 assert!( 2276 exec_host_command_status(format!("mkfs.ext4 {}", test_disk_path).as_str()).success() 2277 ); 2278 2279 let mut cmd = GuestCommand::new(&guest); 2280 cmd.args(&["--cpus", "boot=1"]) 2281 .args(&["--memory", "size=512M"]) 2282 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2283 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2284 .args(&["--platform", "num_pci_segments=16"]) 2285 .args(&[ 2286 "--disk", 2287 format!( 2288 "path={}", 2289 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2290 ) 2291 .as_str(), 2292 format!( 2293 "path={}", 2294 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2295 ) 2296 .as_str(), 2297 format!("path={},pci_segment=15", test_disk_path).as_str(), 2298 ]) 2299 .capture_output() 2300 .default_net(); 2301 2302 let mut child = cmd.spawn().unwrap(); 2303 2304 guest.wait_vm_boot(None).unwrap(); 2305 2306 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2307 2308 let r = std::panic::catch_unwind(|| { 2309 // There should be 16 PCI host bridges in the guest. 2310 assert_eq!( 2311 guest 2312 .ssh_command(grep_cmd) 2313 .unwrap() 2314 .trim() 2315 .parse::<u32>() 2316 .unwrap_or_default(), 2317 16 2318 ); 2319 2320 // Check both if /dev/vdc exists and if the block size is 4M. 2321 assert_eq!( 2322 guest 2323 .ssh_command("lsblk | grep vdc | grep -c 4M") 2324 .unwrap() 2325 .trim() 2326 .parse::<u32>() 2327 .unwrap_or_default(), 2328 1 2329 ); 2330 2331 // Mount the device. 2332 guest.ssh_command("mkdir mount_image").unwrap(); 2333 guest 2334 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 2335 .unwrap(); 2336 // Grant all users with write permission. 2337 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 2338 2339 // Write something to the device. 2340 guest 2341 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 2342 .unwrap(); 2343 2344 // Check the content of the block device. The file "foo" should 2345 // contain "bar". 2346 assert_eq!( 2347 guest 2348 .ssh_command("sudo cat mount_image/foo") 2349 .unwrap() 2350 .trim(), 2351 "bar" 2352 ); 2353 }); 2354 2355 let _ = child.kill(); 2356 let output = child.wait_with_output().unwrap(); 2357 2358 handle_child_output(r, &output); 2359 } 2360 2361 #[test] 2362 fn test_direct_kernel_boot() { 2363 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2364 let guest = Guest::new(Box::new(focal)); 2365 2366 let kernel_path = direct_kernel_boot_path(); 2367 2368 let mut child = GuestCommand::new(&guest) 2369 .args(&["--cpus", "boot=1"]) 2370 .args(&["--memory", "size=512M"]) 2371 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2372 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2373 .default_disks() 2374 .default_net() 2375 .capture_output() 2376 .spawn() 2377 .unwrap(); 2378 2379 let r = std::panic::catch_unwind(|| { 2380 guest.wait_vm_boot(None).unwrap(); 2381 2382 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2383 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2384 assert!(guest.get_entropy().unwrap_or_default() >= 900); 2385 2386 let grep_cmd = if cfg!(target_arch = "x86_64") { 2387 "grep -c PCI-MSI /proc/interrupts" 2388 } else { 2389 "grep -c ITS-MSI /proc/interrupts" 2390 }; 2391 assert_eq!( 2392 guest 2393 .ssh_command(grep_cmd) 2394 .unwrap() 2395 .trim() 2396 .parse::<u32>() 2397 .unwrap_or_default(), 2398 12 2399 ); 2400 }); 2401 2402 let _ = child.kill(); 2403 let output = child.wait_with_output().unwrap(); 2404 2405 handle_child_output(r, &output); 2406 } 2407 2408 fn _test_virtio_block(image_name: &str, disable_io_uring: bool) { 2409 let focal = UbuntuDiskConfig::new(image_name.to_string()); 2410 let guest = Guest::new(Box::new(focal)); 2411 2412 let mut workload_path = dirs::home_dir().unwrap(); 2413 workload_path.push("workloads"); 2414 2415 let mut blk_file_path = workload_path; 2416 blk_file_path.push("blk.img"); 2417 2418 let kernel_path = direct_kernel_boot_path(); 2419 2420 let mut cloud_child = GuestCommand::new(&guest) 2421 .args(&["--cpus", "boot=4"]) 2422 .args(&["--memory", "size=512M,shared=on"]) 2423 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2424 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2425 .args(&[ 2426 "--disk", 2427 format!( 2428 "path={}", 2429 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2430 ) 2431 .as_str(), 2432 format!( 2433 "path={}", 2434 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2435 ) 2436 .as_str(), 2437 format!( 2438 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={}", 2439 blk_file_path.to_str().unwrap(), 2440 disable_io_uring 2441 ) 2442 .as_str(), 2443 ]) 2444 .default_net() 2445 .capture_output() 2446 .spawn() 2447 .unwrap(); 2448 2449 let r = std::panic::catch_unwind(|| { 2450 guest.wait_vm_boot(None).unwrap(); 2451 2452 // Check both if /dev/vdc exists and if the block size is 16M. 2453 assert_eq!( 2454 guest 2455 .ssh_command("lsblk | grep vdc | grep -c 16M") 2456 .unwrap() 2457 .trim() 2458 .parse::<u32>() 2459 .unwrap_or_default(), 2460 1 2461 ); 2462 2463 // Check both if /dev/vdc exists and if this block is RO. 2464 assert_eq!( 2465 guest 2466 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 2467 .unwrap() 2468 .trim() 2469 .parse::<u32>() 2470 .unwrap_or_default(), 2471 1 2472 ); 2473 2474 // Check if the number of queues is 4. 2475 assert_eq!( 2476 guest 2477 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 2478 .unwrap() 2479 .trim() 2480 .parse::<u32>() 2481 .unwrap_or_default(), 2482 4 2483 ); 2484 }); 2485 2486 let _ = cloud_child.kill(); 2487 let output = cloud_child.wait_with_output().unwrap(); 2488 2489 handle_child_output(r, &output); 2490 } 2491 2492 #[test] 2493 fn test_virtio_block() { 2494 _test_virtio_block(FOCAL_IMAGE_NAME, false) 2495 } 2496 2497 #[test] 2498 fn test_virtio_block_disable_io_uring() { 2499 _test_virtio_block(FOCAL_IMAGE_NAME, true) 2500 } 2501 2502 #[test] 2503 fn test_virtio_block_qcow2() { 2504 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false) 2505 } 2506 2507 #[test] 2508 fn test_virtio_block_vhd() { 2509 let mut workload_path = dirs::home_dir().unwrap(); 2510 workload_path.push("workloads"); 2511 2512 let mut raw_file_path = workload_path.clone(); 2513 let mut vhd_file_path = workload_path; 2514 raw_file_path.push(FOCAL_IMAGE_NAME); 2515 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 2516 2517 // Generate VHD file from RAW file 2518 std::process::Command::new("qemu-img") 2519 .arg("convert") 2520 .arg("-p") 2521 .args(&["-f", "raw"]) 2522 .args(&["-O", "vpc"]) 2523 .args(&["-o", "subformat=fixed"]) 2524 .arg(raw_file_path.to_str().unwrap()) 2525 .arg(vhd_file_path.to_str().unwrap()) 2526 .output() 2527 .expect("Expect generating VHD image from RAW image"); 2528 2529 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false) 2530 } 2531 2532 #[test] 2533 fn test_virtio_block_vhdx() { 2534 let mut workload_path = dirs::home_dir().unwrap(); 2535 workload_path.push("workloads"); 2536 2537 let mut raw_file_path = workload_path.clone(); 2538 let mut vhdx_file_path = workload_path; 2539 raw_file_path.push(FOCAL_IMAGE_NAME); 2540 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 2541 2542 // Generate dynamic VHDX file from RAW file 2543 std::process::Command::new("qemu-img") 2544 .arg("convert") 2545 .arg("-p") 2546 .args(&["-f", "raw"]) 2547 .args(&["-O", "vhdx"]) 2548 .arg(raw_file_path.to_str().unwrap()) 2549 .arg(vhdx_file_path.to_str().unwrap()) 2550 .output() 2551 .expect("Expect generating dynamic VHDx image from RAW image"); 2552 2553 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false) 2554 } 2555 2556 #[test] 2557 fn test_virtio_block_dynamic_vhdx_expand() { 2558 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 2559 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 2560 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 2561 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 2562 2563 let mut workload_path = dirs::home_dir().unwrap(); 2564 workload_path.push("workloads"); 2565 2566 let mut vhdx_file_path = workload_path; 2567 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 2568 let vhdx_path = vhdx_file_path.to_str().unwrap(); 2569 2570 // Generate a 100 MiB dynamic VHDX file 2571 std::process::Command::new("qemu-img") 2572 .arg("create") 2573 .args(&["-f", "vhdx"]) 2574 .arg(vhdx_path) 2575 .arg(VIRTUAL_DISK_SIZE.to_string()) 2576 .output() 2577 .expect("Expect generating dynamic VHDx image from RAW image"); 2578 2579 // Check if the size matches with empty VHDx file size 2580 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 2581 2582 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2583 let guest = Guest::new(Box::new(focal)); 2584 let kernel_path = direct_kernel_boot_path(); 2585 2586 let mut cloud_child = GuestCommand::new(&guest) 2587 .args(&["--cpus", "boot=1"]) 2588 .args(&["--memory", "size=512M"]) 2589 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2590 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2591 .args(&[ 2592 "--disk", 2593 format!( 2594 "path={}", 2595 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2596 ) 2597 .as_str(), 2598 format!( 2599 "path={}", 2600 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2601 ) 2602 .as_str(), 2603 format!("path={}", vhdx_path).as_str(), 2604 ]) 2605 .default_net() 2606 .capture_output() 2607 .spawn() 2608 .unwrap(); 2609 2610 let r = std::panic::catch_unwind(|| { 2611 guest.wait_vm_boot(None).unwrap(); 2612 2613 // Check both if /dev/vdc exists and if the block size is 100 MiB. 2614 assert_eq!( 2615 guest 2616 .ssh_command("lsblk | grep vdc | grep -c 100M") 2617 .unwrap() 2618 .trim() 2619 .parse::<u32>() 2620 .unwrap_or_default(), 2621 1 2622 ); 2623 2624 // Write 100 MB of data to the VHDx disk 2625 guest 2626 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 2627 .unwrap(); 2628 }); 2629 2630 // Check if the size matches with expected expanded VHDx file size 2631 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 2632 2633 let _ = cloud_child.kill(); 2634 let output = cloud_child.wait_with_output().unwrap(); 2635 2636 handle_child_output(r, &output); 2637 } 2638 2639 fn vhdx_image_size(disk_name: &str) -> u64 { 2640 std::fs::File::open(disk_name) 2641 .unwrap() 2642 .seek(SeekFrom::End(0)) 2643 .unwrap() 2644 } 2645 2646 #[test] 2647 fn test_virtio_block_direct_and_firmware() { 2648 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2649 let guest = Guest::new(Box::new(focal)); 2650 2651 // The OS disk must be copied to a location that is not backed by 2652 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 2653 // with EINVAL because tmpfs doesn't support this flag. 2654 let mut workloads_path = dirs::home_dir().unwrap(); 2655 workloads_path.push("workloads"); 2656 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 2657 let mut os_path = os_dir.as_path().to_path_buf(); 2658 os_path.push("osdisk.img"); 2659 rate_limited_copy( 2660 &guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 2661 os_path.as_path(), 2662 ) 2663 .expect("copying of OS disk failed"); 2664 2665 let mut child = GuestCommand::new(&guest) 2666 .args(&["--cpus", "boot=1"]) 2667 .args(&["--memory", "size=512M"]) 2668 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 2669 .args(&[ 2670 "--disk", 2671 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 2672 format!( 2673 "path={}", 2674 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2675 ) 2676 .as_str(), 2677 ]) 2678 .default_net() 2679 .capture_output() 2680 .spawn() 2681 .unwrap(); 2682 2683 let r = std::panic::catch_unwind(|| { 2684 guest.wait_vm_boot(Some(120)).unwrap(); 2685 }); 2686 2687 let _ = child.kill(); 2688 let output = child.wait_with_output().unwrap(); 2689 2690 handle_child_output(r, &output); 2691 } 2692 2693 #[test] 2694 fn test_vhost_user_net_default() { 2695 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 2696 } 2697 2698 #[test] 2699 fn test_vhost_user_net_named_tap() { 2700 test_vhost_user_net( 2701 Some("mytap0"), 2702 2, 2703 &prepare_vhost_user_net_daemon, 2704 false, 2705 false, 2706 ) 2707 } 2708 2709 #[test] 2710 fn test_vhost_user_net_existing_tap() { 2711 test_vhost_user_net( 2712 Some("vunet-tap0"), 2713 2, 2714 &prepare_vhost_user_net_daemon, 2715 false, 2716 false, 2717 ) 2718 } 2719 2720 #[test] 2721 fn test_vhost_user_net_multiple_queues() { 2722 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 2723 } 2724 2725 #[test] 2726 fn test_vhost_user_net_tap_multiple_queues() { 2727 test_vhost_user_net( 2728 Some("vunet-tap1"), 2729 4, 2730 &prepare_vhost_user_net_daemon, 2731 false, 2732 false, 2733 ) 2734 } 2735 2736 #[test] 2737 fn test_vhost_user_net_host_mac() { 2738 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 2739 } 2740 2741 #[test] 2742 fn test_vhost_user_net_client_mode() { 2743 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 2744 } 2745 2746 #[test] 2747 fn test_vhost_user_blk_default() { 2748 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 2749 } 2750 2751 #[test] 2752 fn test_vhost_user_blk_readonly() { 2753 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 2754 } 2755 2756 #[test] 2757 fn test_vhost_user_blk_direct() { 2758 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 2759 } 2760 2761 #[test] 2762 fn test_boot_from_vhost_user_blk_default() { 2763 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 2764 } 2765 2766 #[test] 2767 #[cfg(target_arch = "x86_64")] 2768 fn test_split_irqchip() { 2769 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2770 let guest = Guest::new(Box::new(focal)); 2771 2772 let mut child = GuestCommand::new(&guest) 2773 .args(&["--cpus", "boot=1"]) 2774 .args(&["--memory", "size=512M"]) 2775 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2776 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2777 .default_disks() 2778 .default_net() 2779 .capture_output() 2780 .spawn() 2781 .unwrap(); 2782 2783 let r = std::panic::catch_unwind(|| { 2784 guest.wait_vm_boot(None).unwrap(); 2785 2786 assert_eq!( 2787 guest 2788 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 2789 .unwrap() 2790 .trim() 2791 .parse::<u32>() 2792 .unwrap_or(1), 2793 0 2794 ); 2795 assert_eq!( 2796 guest 2797 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 2798 .unwrap() 2799 .trim() 2800 .parse::<u32>() 2801 .unwrap_or(1), 2802 0 2803 ); 2804 }); 2805 2806 let _ = child.kill(); 2807 let output = child.wait_with_output().unwrap(); 2808 2809 handle_child_output(r, &output); 2810 } 2811 2812 #[test] 2813 fn test_virtio_fs_dax_off() { 2814 test_virtio_fs(false, None, "never", &prepare_virtiofsd, false, None) 2815 } 2816 2817 #[test] 2818 fn test_virtio_fs_hotplug_dax_off() { 2819 test_virtio_fs(false, None, "never", &prepare_virtiofsd, true, None) 2820 } 2821 2822 #[test] 2823 #[cfg(not(feature = "mshv"))] 2824 fn test_virtio_fs_multi_segment_hotplug() { 2825 test_virtio_fs(false, None, "never", &prepare_virtiofsd, true, Some(15)) 2826 } 2827 2828 #[test] 2829 #[cfg(not(feature = "mshv"))] 2830 fn test_virtio_fs_multi_segment() { 2831 test_virtio_fs(false, None, "never", &prepare_virtiofsd, false, Some(15)) 2832 } 2833 2834 #[test] 2835 fn test_virtio_pmem_persist_writes() { 2836 test_virtio_pmem(false, false) 2837 } 2838 2839 #[test] 2840 fn test_virtio_pmem_discard_writes() { 2841 test_virtio_pmem(true, false) 2842 } 2843 2844 #[test] 2845 fn test_virtio_pmem_with_size() { 2846 test_virtio_pmem(true, true) 2847 } 2848 2849 #[test] 2850 fn test_boot_from_virtio_pmem() { 2851 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2852 let guest = Guest::new(Box::new(focal)); 2853 2854 let kernel_path = direct_kernel_boot_path(); 2855 2856 let mut child = GuestCommand::new(&guest) 2857 .args(&["--cpus", "boot=1"]) 2858 .args(&["--memory", "size=512M"]) 2859 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2860 .args(&[ 2861 "--disk", 2862 format!( 2863 "path={}", 2864 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2865 ) 2866 .as_str(), 2867 ]) 2868 .default_net() 2869 .args(&[ 2870 "--pmem", 2871 format!( 2872 "file={},size={}", 2873 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 2874 fs::metadata(&guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 2875 .unwrap() 2876 .len() 2877 ) 2878 .as_str(), 2879 ]) 2880 .args(&[ 2881 "--cmdline", 2882 DIRECT_KERNEL_BOOT_CMDLINE 2883 .replace("vda1", "pmem0p1") 2884 .as_str(), 2885 ]) 2886 .capture_output() 2887 .spawn() 2888 .unwrap(); 2889 2890 let r = std::panic::catch_unwind(|| { 2891 guest.wait_vm_boot(None).unwrap(); 2892 2893 // Simple checks to validate the VM booted properly 2894 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2895 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2896 }); 2897 2898 let _ = child.kill(); 2899 let output = child.wait_with_output().unwrap(); 2900 2901 handle_child_output(r, &output); 2902 } 2903 2904 #[test] 2905 fn test_multiple_network_interfaces() { 2906 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2907 let guest = Guest::new(Box::new(focal)); 2908 2909 let kernel_path = direct_kernel_boot_path(); 2910 2911 let mut child = GuestCommand::new(&guest) 2912 .args(&["--cpus", "boot=1"]) 2913 .args(&["--memory", "size=512M"]) 2914 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2915 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2916 .default_disks() 2917 .args(&[ 2918 "--net", 2919 guest.default_net_string().as_str(), 2920 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 2921 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 2922 ]) 2923 .capture_output() 2924 .spawn() 2925 .unwrap(); 2926 2927 let r = std::panic::catch_unwind(|| { 2928 guest.wait_vm_boot(None).unwrap(); 2929 2930 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 2931 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 2932 2933 // 3 network interfaces + default localhost ==> 4 interfaces 2934 assert_eq!( 2935 guest 2936 .ssh_command("ip -o link | wc -l") 2937 .unwrap() 2938 .trim() 2939 .parse::<u32>() 2940 .unwrap_or_default(), 2941 4 2942 ); 2943 }); 2944 2945 let _ = child.kill(); 2946 let output = child.wait_with_output().unwrap(); 2947 2948 handle_child_output(r, &output); 2949 } 2950 2951 #[test] 2952 #[cfg(target_arch = "aarch64")] 2953 fn test_pmu_on() { 2954 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2955 let guest = Guest::new(Box::new(focal)); 2956 let mut child = GuestCommand::new(&guest) 2957 .args(&["--cpus", "boot=1"]) 2958 .args(&["--memory", "size=512M"]) 2959 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2960 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2961 .default_disks() 2962 .default_net() 2963 .capture_output() 2964 .spawn() 2965 .unwrap(); 2966 2967 let r = std::panic::catch_unwind(|| { 2968 guest.wait_vm_boot(None).unwrap(); 2969 2970 // Test that PMU exists. 2971 assert_eq!( 2972 guest 2973 .ssh_command(GREP_PMU_IRQ_CMD) 2974 .unwrap() 2975 .trim() 2976 .parse::<u32>() 2977 .unwrap_or_default(), 2978 1 2979 ); 2980 }); 2981 2982 let _ = child.kill(); 2983 let output = child.wait_with_output().unwrap(); 2984 2985 handle_child_output(r, &output); 2986 } 2987 2988 #[test] 2989 fn test_serial_off() { 2990 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2991 let guest = Guest::new(Box::new(focal)); 2992 let mut child = GuestCommand::new(&guest) 2993 .args(&["--cpus", "boot=1"]) 2994 .args(&["--memory", "size=512M"]) 2995 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2996 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2997 .default_disks() 2998 .default_net() 2999 .args(&["--serial", "off"]) 3000 .capture_output() 3001 .spawn() 3002 .unwrap(); 3003 3004 let r = std::panic::catch_unwind(|| { 3005 guest.wait_vm_boot(None).unwrap(); 3006 3007 // Test that there is no ttyS0 3008 assert_eq!( 3009 guest 3010 .ssh_command(GREP_SERIAL_IRQ_CMD) 3011 .unwrap() 3012 .trim() 3013 .parse::<u32>() 3014 .unwrap_or(1), 3015 0 3016 ); 3017 }); 3018 3019 let _ = child.kill(); 3020 let output = child.wait_with_output().unwrap(); 3021 3022 handle_child_output(r, &output); 3023 } 3024 3025 #[test] 3026 fn test_serial_null() { 3027 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3028 let guest = Guest::new(Box::new(focal)); 3029 let mut cmd = GuestCommand::new(&guest); 3030 #[cfg(target_arch = "x86_64")] 3031 let console_str: &str = "console=ttyS0"; 3032 #[cfg(target_arch = "aarch64")] 3033 let console_str: &str = "console=ttyAMA0"; 3034 3035 cmd.args(&["--cpus", "boot=1"]) 3036 .args(&["--memory", "size=512M"]) 3037 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3038 .args(&[ 3039 "--cmdline", 3040 DIRECT_KERNEL_BOOT_CMDLINE 3041 .replace("console=hvc0 ", console_str) 3042 .as_str(), 3043 ]) 3044 .default_disks() 3045 .default_net() 3046 .args(&["--serial", "null"]) 3047 .args(&["--console", "off"]) 3048 .capture_output(); 3049 3050 let mut child = cmd.spawn().unwrap(); 3051 3052 let r = std::panic::catch_unwind(|| { 3053 guest.wait_vm_boot(None).unwrap(); 3054 3055 // Test that there is a ttyS0 3056 assert_eq!( 3057 guest 3058 .ssh_command(GREP_SERIAL_IRQ_CMD) 3059 .unwrap() 3060 .trim() 3061 .parse::<u32>() 3062 .unwrap_or_default(), 3063 1 3064 ); 3065 }); 3066 3067 let _ = child.kill(); 3068 let output = child.wait_with_output().unwrap(); 3069 handle_child_output(r, &output); 3070 3071 let r = std::panic::catch_unwind(|| { 3072 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3073 }); 3074 3075 handle_child_output(r, &output); 3076 } 3077 3078 #[test] 3079 fn test_serial_tty() { 3080 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3081 let guest = Guest::new(Box::new(focal)); 3082 3083 let kernel_path = direct_kernel_boot_path(); 3084 3085 #[cfg(target_arch = "x86_64")] 3086 let console_str: &str = "console=ttyS0"; 3087 #[cfg(target_arch = "aarch64")] 3088 let console_str: &str = "console=ttyAMA0"; 3089 3090 let mut child = GuestCommand::new(&guest) 3091 .args(&["--cpus", "boot=1"]) 3092 .args(&["--memory", "size=512M"]) 3093 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3094 .args(&[ 3095 "--cmdline", 3096 DIRECT_KERNEL_BOOT_CMDLINE 3097 .replace("console=hvc0 ", console_str) 3098 .as_str(), 3099 ]) 3100 .default_disks() 3101 .default_net() 3102 .args(&["--serial", "tty"]) 3103 .args(&["--console", "off"]) 3104 .capture_output() 3105 .spawn() 3106 .unwrap(); 3107 3108 let r = std::panic::catch_unwind(|| { 3109 guest.wait_vm_boot(None).unwrap(); 3110 3111 // Test that there is a ttyS0 3112 assert_eq!( 3113 guest 3114 .ssh_command(GREP_SERIAL_IRQ_CMD) 3115 .unwrap() 3116 .trim() 3117 .parse::<u32>() 3118 .unwrap_or_default(), 3119 1 3120 ); 3121 }); 3122 3123 // This sleep is needed to wait for the login prompt 3124 thread::sleep(std::time::Duration::new(2, 0)); 3125 3126 let _ = child.kill(); 3127 let output = child.wait_with_output().unwrap(); 3128 handle_child_output(r, &output); 3129 3130 let r = std::panic::catch_unwind(|| { 3131 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3132 }); 3133 3134 handle_child_output(r, &output); 3135 } 3136 3137 #[test] 3138 fn test_serial_file() { 3139 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3140 let guest = Guest::new(Box::new(focal)); 3141 3142 let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output"); 3143 #[cfg(target_arch = "x86_64")] 3144 let console_str: &str = "console=ttyS0"; 3145 #[cfg(target_arch = "aarch64")] 3146 let console_str: &str = "console=ttyAMA0"; 3147 3148 let mut child = GuestCommand::new(&guest) 3149 .args(&["--cpus", "boot=1"]) 3150 .args(&["--memory", "size=512M"]) 3151 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3152 .args(&[ 3153 "--cmdline", 3154 DIRECT_KERNEL_BOOT_CMDLINE 3155 .replace("console=hvc0 ", console_str) 3156 .as_str(), 3157 ]) 3158 .default_disks() 3159 .default_net() 3160 .args(&[ 3161 "--serial", 3162 format!("file={}", serial_path.to_str().unwrap()).as_str(), 3163 ]) 3164 .capture_output() 3165 .spawn() 3166 .unwrap(); 3167 3168 let r = std::panic::catch_unwind(|| { 3169 guest.wait_vm_boot(None).unwrap(); 3170 3171 // Test that there is a ttyS0 3172 assert_eq!( 3173 guest 3174 .ssh_command(GREP_SERIAL_IRQ_CMD) 3175 .unwrap() 3176 .trim() 3177 .parse::<u32>() 3178 .unwrap_or_default(), 3179 1 3180 ); 3181 3182 guest.ssh_command("sudo shutdown -h now").unwrap(); 3183 }); 3184 3185 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3186 let _ = child.kill(); 3187 let output = child.wait_with_output().unwrap(); 3188 handle_child_output(r, &output); 3189 3190 let r = std::panic::catch_unwind(|| { 3191 // Check that the cloud-hypervisor binary actually terminated 3192 assert!(output.status.success()); 3193 3194 // Do this check after shutdown of the VM as an easy way to ensure 3195 // all writes are flushed to disk 3196 let mut f = std::fs::File::open(serial_path).unwrap(); 3197 let mut buf = String::new(); 3198 f.read_to_string(&mut buf).unwrap(); 3199 assert!(buf.contains(CONSOLE_TEST_STRING)); 3200 }); 3201 3202 handle_child_output(r, &output); 3203 } 3204 3205 #[test] 3206 fn test_pty_interaction() { 3207 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3208 let guest = Guest::new(Box::new(focal)); 3209 let api_socket = temp_api_path(&guest.tmp_dir); 3210 let serial_option = if cfg!(target_arch = "x86_64") { 3211 " console=ttyS0" 3212 } else { 3213 " console=ttyAMA0" 3214 }; 3215 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 3216 3217 let mut child = GuestCommand::new(&guest) 3218 .args(&["--cpus", "boot=1"]) 3219 .args(&["--memory", "size=512M"]) 3220 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3221 .args(&["--cmdline", &cmdline]) 3222 .default_disks() 3223 .default_net() 3224 .args(&["--serial", "null"]) 3225 .args(&["--console", "pty"]) 3226 .args(&["--api-socket", &api_socket]) 3227 .spawn() 3228 .unwrap(); 3229 3230 let r = std::panic::catch_unwind(|| { 3231 guest.wait_vm_boot(None).unwrap(); 3232 // Get pty fd for console 3233 let console_path = get_pty_path(&api_socket, "console"); 3234 // TODO: Get serial pty test working 3235 let mut cf = std::fs::OpenOptions::new() 3236 .write(true) 3237 .read(true) 3238 .open(console_path) 3239 .unwrap(); 3240 3241 // Some dumb sleeps but we don't want to write 3242 // before the console is up and we don't want 3243 // to try and write the next line before the 3244 // login process is ready. 3245 thread::sleep(std::time::Duration::new(5, 0)); 3246 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 3247 thread::sleep(std::time::Duration::new(2, 0)); 3248 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 3249 thread::sleep(std::time::Duration::new(2, 0)); 3250 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 3251 thread::sleep(std::time::Duration::new(2, 0)); 3252 3253 // read pty and ensure they have a login shell 3254 // some fairly hacky workarounds to avoid looping 3255 // forever in case the channel is blocked getting output 3256 let ptyc = pty_read(cf); 3257 let mut empty = 0; 3258 let mut prev = String::new(); 3259 loop { 3260 thread::sleep(std::time::Duration::new(2, 0)); 3261 match ptyc.try_recv() { 3262 Ok(line) => { 3263 empty = 0; 3264 prev = prev + &line; 3265 if prev.contains("test_pty_console") { 3266 break; 3267 } 3268 } 3269 Err(mpsc::TryRecvError::Empty) => { 3270 empty += 1; 3271 assert!(empty <= 5, "No login on pty"); 3272 } 3273 _ => panic!("No login on pty"), 3274 } 3275 } 3276 3277 guest.ssh_command("sudo shutdown -h now").unwrap(); 3278 }); 3279 3280 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3281 let _ = child.kill(); 3282 let output = child.wait_with_output().unwrap(); 3283 handle_child_output(r, &output); 3284 3285 let r = std::panic::catch_unwind(|| { 3286 // Check that the cloud-hypervisor binary actually terminated 3287 assert!(output.status.success()) 3288 }); 3289 handle_child_output(r, &output); 3290 } 3291 3292 #[test] 3293 fn test_virtio_console() { 3294 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3295 let guest = Guest::new(Box::new(focal)); 3296 3297 let kernel_path = direct_kernel_boot_path(); 3298 3299 let mut child = GuestCommand::new(&guest) 3300 .args(&["--cpus", "boot=1"]) 3301 .args(&["--memory", "size=512M"]) 3302 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3303 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3304 .default_disks() 3305 .default_net() 3306 .args(&["--console", "tty"]) 3307 .args(&["--serial", "null"]) 3308 .capture_output() 3309 .spawn() 3310 .unwrap(); 3311 3312 let text = String::from("On a branch floating down river a cricket, singing."); 3313 let cmd = format!("echo {} | sudo tee /dev/hvc0", text); 3314 3315 let r = std::panic::catch_unwind(|| { 3316 guest.wait_vm_boot(None).unwrap(); 3317 3318 assert!(guest 3319 .does_device_vendor_pair_match("0x1043", "0x1af4") 3320 .unwrap_or_default()); 3321 3322 guest.ssh_command(&cmd).unwrap(); 3323 }); 3324 3325 let _ = child.kill(); 3326 let output = child.wait_with_output().unwrap(); 3327 handle_child_output(r, &output); 3328 3329 let r = std::panic::catch_unwind(|| { 3330 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 3331 }); 3332 3333 handle_child_output(r, &output); 3334 } 3335 3336 #[test] 3337 fn test_console_file() { 3338 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3339 let guest = Guest::new(Box::new(focal)); 3340 3341 let console_path = guest.tmp_dir.as_path().join("/tmp/console-output"); 3342 let mut child = GuestCommand::new(&guest) 3343 .args(&["--cpus", "boot=1"]) 3344 .args(&["--memory", "size=512M"]) 3345 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3346 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3347 .default_disks() 3348 .default_net() 3349 .args(&[ 3350 "--console", 3351 format!("file={}", console_path.to_str().unwrap()).as_str(), 3352 ]) 3353 .capture_output() 3354 .spawn() 3355 .unwrap(); 3356 3357 guest.wait_vm_boot(None).unwrap(); 3358 3359 guest.ssh_command("sudo shutdown -h now").unwrap(); 3360 3361 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3362 let _ = child.kill(); 3363 let output = child.wait_with_output().unwrap(); 3364 3365 let r = std::panic::catch_unwind(|| { 3366 // Check that the cloud-hypervisor binary actually terminated 3367 assert!(output.status.success()); 3368 3369 // Do this check after shutdown of the VM as an easy way to ensure 3370 // all writes are flushed to disk 3371 let mut f = std::fs::File::open(console_path).unwrap(); 3372 let mut buf = String::new(); 3373 f.read_to_string(&mut buf).unwrap(); 3374 3375 if !buf.contains(CONSOLE_TEST_STRING) { 3376 eprintln!( 3377 "\n\n==== Console file output ====\n\n{}\n\n==== End console file output ====", 3378 buf 3379 ); 3380 } 3381 assert!(buf.contains(CONSOLE_TEST_STRING)); 3382 }); 3383 3384 handle_child_output(r, &output); 3385 } 3386 3387 #[test] 3388 #[cfg(target_arch = "x86_64")] 3389 #[cfg(not(feature = "mshv"))] 3390 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 3391 // backed networking interfaces, bound through a simple bridge on the host. 3392 // So if the nested cloud-hypervisor succeeds in getting a directly 3393 // assigned interface from its cloud-hypervisor host, we should be able to 3394 // ssh into it, and verify that it's running with the right kernel command 3395 // line (We tag the command line from cloud-hypervisor for that purpose). 3396 // The third device is added to validate that hotplug works correctly since 3397 // it is being added to the L2 VM through hotplugging mechanism. 3398 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 3399 // vfio device support 3400 fn test_vfio() { 3401 setup_vfio_network_interfaces(); 3402 3403 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3404 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 3405 3406 let mut workload_path = dirs::home_dir().unwrap(); 3407 workload_path.push("workloads"); 3408 3409 let kernel_path = direct_kernel_boot_path(); 3410 3411 let mut vfio_path = workload_path.clone(); 3412 vfio_path.push("vfio"); 3413 3414 let mut cloud_init_vfio_base_path = vfio_path.clone(); 3415 cloud_init_vfio_base_path.push("cloudinit.img"); 3416 3417 // We copy our cloudinit into the vfio mount point, for the nested 3418 // cloud-hypervisor guest to use. 3419 rate_limited_copy( 3420 &guest.disk_config.disk(DiskType::CloudInit).unwrap(), 3421 &cloud_init_vfio_base_path, 3422 ) 3423 .expect("copying of cloud-init disk failed"); 3424 3425 let mut vfio_disk_path = workload_path.clone(); 3426 vfio_disk_path.push("vfio.img"); 3427 3428 // Create the vfio disk image 3429 let output = Command::new("mkfs.ext4") 3430 .arg("-d") 3431 .arg(vfio_path.to_str().unwrap()) 3432 .arg(vfio_disk_path.to_str().unwrap()) 3433 .arg("2g") 3434 .output() 3435 .unwrap(); 3436 if !output.status.success() { 3437 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 3438 panic!("mkfs.ext4 command generated an error"); 3439 } 3440 3441 let mut blk_file_path = workload_path; 3442 blk_file_path.push("blk.img"); 3443 3444 let vfio_tap0 = "vfio-tap0"; 3445 let vfio_tap1 = "vfio-tap1"; 3446 let vfio_tap2 = "vfio-tap2"; 3447 let vfio_tap3 = "vfio-tap3"; 3448 3449 let mut child = GuestCommand::new(&guest) 3450 .args(&["--cpus", "boot=4"]) 3451 .args(&["--memory", "size=2G,hugepages=on,shared=on"]) 3452 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3453 .args(&[ 3454 "--disk", 3455 format!( 3456 "path={}", 3457 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3458 ) 3459 .as_str(), 3460 format!( 3461 "path={}", 3462 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3463 ) 3464 .as_str(), 3465 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 3466 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 3467 ]) 3468 .args(&[ 3469 "--cmdline", 3470 format!( 3471 "{} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts", 3472 DIRECT_KERNEL_BOOT_CMDLINE 3473 ) 3474 .as_str(), 3475 ]) 3476 .args(&[ 3477 "--net", 3478 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 3479 format!( 3480 "tap={},mac={},iommu=on", 3481 vfio_tap1, guest.network.l2_guest_mac1 3482 ) 3483 .as_str(), 3484 format!( 3485 "tap={},mac={},iommu=on", 3486 vfio_tap2, guest.network.l2_guest_mac2 3487 ) 3488 .as_str(), 3489 format!( 3490 "tap={},mac={},iommu=on", 3491 vfio_tap3, guest.network.l2_guest_mac3 3492 ) 3493 .as_str(), 3494 ]) 3495 .capture_output() 3496 .spawn() 3497 .unwrap(); 3498 3499 thread::sleep(std::time::Duration::new(30, 0)); 3500 3501 let r = std::panic::catch_unwind(|| { 3502 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 3503 thread::sleep(std::time::Duration::new(120, 0)); 3504 3505 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 3506 // added to its kernel command line. 3507 // Let's ssh into it and verify that it's there. If it is it means 3508 // we're in the right guest (The L2 one) because the QEMU L1 guest 3509 // does not have this command line tag. 3510 assert_eq!( 3511 guest 3512 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 3513 .unwrap() 3514 .trim() 3515 .parse::<u32>() 3516 .unwrap_or_default(), 3517 1 3518 ); 3519 3520 // Let's also verify from the second virtio-net device passed to 3521 // the L2 VM. 3522 assert_eq!( 3523 guest 3524 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 3525 .unwrap() 3526 .trim() 3527 .parse::<u32>() 3528 .unwrap_or_default(), 3529 1 3530 ); 3531 3532 // Check the amount of PCI devices appearing in L2 VM. 3533 assert_eq!( 3534 guest 3535 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3536 .unwrap() 3537 .trim() 3538 .parse::<u32>() 3539 .unwrap_or_default(), 3540 8, 3541 ); 3542 3543 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 3544 assert_eq!( 3545 guest 3546 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 3547 .unwrap() 3548 .trim() 3549 .parse::<u32>() 3550 .unwrap_or_default(), 3551 1 3552 ); 3553 3554 // Hotplug an extra virtio-net device through L2 VM. 3555 guest 3556 .ssh_command_l1( 3557 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 3558 ) 3559 .unwrap(); 3560 guest 3561 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 3562 .unwrap(); 3563 let vfio_hotplug_output = guest 3564 .ssh_command_l1( 3565 "sudo /mnt/ch-remote \ 3566 --api-socket=/tmp/ch_api.sock \ 3567 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 3568 ) 3569 .unwrap(); 3570 assert!(vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}")); 3571 3572 thread::sleep(std::time::Duration::new(10, 0)); 3573 3574 // Let's also verify from the third virtio-net device passed to 3575 // the L2 VM. This third device has been hotplugged through the L2 3576 // VM, so this is our way to validate hotplug works for VFIO PCI. 3577 assert_eq!( 3578 guest 3579 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 3580 .unwrap() 3581 .trim() 3582 .parse::<u32>() 3583 .unwrap_or_default(), 3584 1 3585 ); 3586 3587 // Check the amount of PCI devices appearing in L2 VM. 3588 // There should be one more device than before, raising the count 3589 // up to 9 PCI devices. 3590 assert_eq!( 3591 guest 3592 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3593 .unwrap() 3594 .trim() 3595 .parse::<u32>() 3596 .unwrap_or_default(), 3597 9, 3598 ); 3599 3600 // Let's now verify that we can correctly remove the virtio-net 3601 // device through the "remove-device" command responsible for 3602 // unplugging VFIO devices. 3603 guest 3604 .ssh_command_l1( 3605 "sudo /mnt/ch-remote \ 3606 --api-socket=/tmp/ch_api.sock \ 3607 remove-device vfio123", 3608 ) 3609 .unwrap(); 3610 thread::sleep(std::time::Duration::new(10, 0)); 3611 3612 // Check the amount of PCI devices appearing in L2 VM is back down 3613 // to 8 devices. 3614 assert_eq!( 3615 guest 3616 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3617 .unwrap() 3618 .trim() 3619 .parse::<u32>() 3620 .unwrap_or_default(), 3621 8, 3622 ); 3623 3624 // Perform memory hotplug in L2 and validate the memory is showing 3625 // up as expected. In order to check, we will use the virtio-net 3626 // device already passed through L2 as a VFIO device, this will 3627 // verify that VFIO devices are functional with memory hotplug. 3628 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 3629 guest 3630 .ssh_command_l2_1( 3631 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 3632 ) 3633 .unwrap(); 3634 guest 3635 .ssh_command_l1( 3636 "sudo /mnt/ch-remote \ 3637 --api-socket=/tmp/ch_api.sock \ 3638 resize --memory=1073741824", 3639 ) 3640 .unwrap(); 3641 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 3642 }); 3643 3644 let _ = child.kill(); 3645 let output = child.wait_with_output().unwrap(); 3646 3647 cleanup_vfio_network_interfaces(); 3648 3649 handle_child_output(r, &output); 3650 } 3651 3652 #[test] 3653 fn test_direct_kernel_boot_noacpi() { 3654 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3655 let guest = Guest::new(Box::new(focal)); 3656 3657 let kernel_path = direct_kernel_boot_path(); 3658 3659 let mut child = GuestCommand::new(&guest) 3660 .args(&["--cpus", "boot=1"]) 3661 .args(&["--memory", "size=512M"]) 3662 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3663 .args(&[ 3664 "--cmdline", 3665 format!("{} acpi=off", DIRECT_KERNEL_BOOT_CMDLINE).as_str(), 3666 ]) 3667 .default_disks() 3668 .default_net() 3669 .capture_output() 3670 .spawn() 3671 .unwrap(); 3672 3673 let r = std::panic::catch_unwind(|| { 3674 guest.wait_vm_boot(None).unwrap(); 3675 3676 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3677 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3678 assert!(guest.get_entropy().unwrap_or_default() >= 900); 3679 }); 3680 3681 let _ = child.kill(); 3682 let output = child.wait_with_output().unwrap(); 3683 3684 handle_child_output(r, &output); 3685 } 3686 3687 #[test] 3688 fn test_reboot() { 3689 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 3690 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3691 3692 vec![Box::new(bionic), Box::new(focal)] 3693 .drain(..) 3694 .for_each(|disk_config| { 3695 let guest = Guest::new(disk_config); 3696 3697 let mut cmd = GuestCommand::new(&guest); 3698 cmd.args(&["--cpus", "boot=1"]) 3699 .args(&["--memory", "size=512M"]) 3700 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3701 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3702 .default_disks() 3703 .default_net() 3704 .capture_output(); 3705 3706 let mut child = cmd.spawn().unwrap(); 3707 3708 let r = std::panic::catch_unwind(|| { 3709 guest.wait_vm_boot(Some(120)).unwrap(); 3710 3711 let fd_count_1 = get_fd_count(child.id()); 3712 guest.reboot_linux(0, Some(120)); 3713 let fd_count_2 = get_fd_count(child.id()); 3714 assert_eq!(fd_count_1, fd_count_2); 3715 3716 guest.ssh_command("sudo shutdown -h now").unwrap(); 3717 }); 3718 3719 let _ = child.wait_timeout(std::time::Duration::from_secs(40)); 3720 let _ = child.kill(); 3721 let output = child.wait_with_output().unwrap(); 3722 handle_child_output(r, &output); 3723 3724 let r = std::panic::catch_unwind(|| { 3725 // Check that the cloud-hypervisor binary actually terminated 3726 assert!(output.status.success()); 3727 }); 3728 3729 handle_child_output(r, &output); 3730 }); 3731 } 3732 3733 #[test] 3734 fn test_virtio_vsock() { 3735 _test_virtio_vsock(false) 3736 } 3737 3738 #[test] 3739 fn test_virtio_vsock_hotplug() { 3740 _test_virtio_vsock(true); 3741 } 3742 3743 #[test] 3744 // Start cloud-hypervisor with no VM parameters, only the API server running. 3745 // From the API: Create a VM, boot it and check that it looks as expected. 3746 fn test_api_create_boot() { 3747 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3748 let guest = Guest::new(Box::new(focal)); 3749 3750 let api_socket = temp_api_path(&guest.tmp_dir); 3751 3752 let mut child = GuestCommand::new(&guest) 3753 .args(&["--api-socket", &api_socket]) 3754 .capture_output() 3755 .spawn() 3756 .unwrap(); 3757 3758 thread::sleep(std::time::Duration::new(1, 0)); 3759 3760 // Verify API server is running 3761 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 3762 3763 // Create the VM first 3764 let cpu_count: u8 = 4; 3765 let http_body = guest.api_create_body( 3766 cpu_count, 3767 fw_path(FwType::RustHypervisorFirmware).as_str(), 3768 direct_kernel_boot_path().to_str().unwrap(), 3769 DIRECT_KERNEL_BOOT_CMDLINE, 3770 ); 3771 3772 curl_command( 3773 &api_socket, 3774 "PUT", 3775 "http://localhost/api/v1/vm.create", 3776 Some(&http_body), 3777 ); 3778 3779 // Then boot it 3780 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 3781 thread::sleep(std::time::Duration::new(20, 0)); 3782 3783 let r = std::panic::catch_unwind(|| { 3784 // Check that the VM booted as expected 3785 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 3786 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3787 assert!(guest.get_entropy().unwrap_or_default() >= 900); 3788 }); 3789 3790 let _ = child.kill(); 3791 let output = child.wait_with_output().unwrap(); 3792 3793 handle_child_output(r, &output); 3794 } 3795 3796 #[test] 3797 // Start cloud-hypervisor with no VM parameters, only the API server running. 3798 // From the API: Create a VM, boot it and check that it looks as expected. 3799 // Then we pause the VM, check that it's no longer available. 3800 // Finally we resume the VM and check that it's available. 3801 fn test_api_pause_resume() { 3802 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3803 let guest = Guest::new(Box::new(focal)); 3804 3805 let api_socket = temp_api_path(&guest.tmp_dir); 3806 3807 let mut child = GuestCommand::new(&guest) 3808 .args(&["--api-socket", &api_socket]) 3809 .capture_output() 3810 .spawn() 3811 .unwrap(); 3812 3813 thread::sleep(std::time::Duration::new(1, 0)); 3814 3815 // Verify API server is running 3816 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 3817 3818 // Create the VM first 3819 let cpu_count: u8 = 4; 3820 let http_body = guest.api_create_body( 3821 cpu_count, 3822 fw_path(FwType::RustHypervisorFirmware).as_str(), 3823 direct_kernel_boot_path().to_str().unwrap(), 3824 DIRECT_KERNEL_BOOT_CMDLINE, 3825 ); 3826 curl_command( 3827 &api_socket, 3828 "PUT", 3829 "http://localhost/api/v1/vm.create", 3830 Some(&http_body), 3831 ); 3832 3833 // Then boot it 3834 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 3835 thread::sleep(std::time::Duration::new(20, 0)); 3836 3837 let r = std::panic::catch_unwind(|| { 3838 // Check that the VM booted as expected 3839 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 3840 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3841 assert!(guest.get_entropy().unwrap_or_default() >= 900); 3842 3843 // We now pause the VM 3844 assert!(remote_command(&api_socket, "pause", None)); 3845 3846 // Check pausing again fails 3847 assert!(!remote_command(&api_socket, "pause", None)); 3848 3849 thread::sleep(std::time::Duration::new(2, 0)); 3850 3851 // SSH into the VM should fail 3852 assert!(ssh_command_ip( 3853 "grep -c processor /proc/cpuinfo", 3854 &guest.network.guest_ip, 3855 2, 3856 5 3857 ) 3858 .is_err()); 3859 3860 // Resume the VM 3861 assert!(remote_command(&api_socket, "resume", None)); 3862 3863 // Check resuming again fails 3864 assert!(!remote_command(&api_socket, "resume", None)); 3865 3866 thread::sleep(std::time::Duration::new(2, 0)); 3867 3868 // Now we should be able to SSH back in and get the right number of CPUs 3869 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 3870 }); 3871 3872 let _ = child.kill(); 3873 let output = child.wait_with_output().unwrap(); 3874 3875 handle_child_output(r, &output); 3876 } 3877 3878 #[test] 3879 fn test_virtio_iommu() { 3880 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 3881 } 3882 3883 #[test] 3884 // We cannot force the software running in the guest to reprogram the BAR 3885 // with some different addresses, but we have a reliable way of testing it 3886 // with a standard Linux kernel. 3887 // By removing a device from the PCI tree, and then rescanning the tree, 3888 // Linux consistently chooses to reorganize the PCI device BARs to other 3889 // locations in the guest address space. 3890 // This test creates a dedicated PCI network device to be checked as being 3891 // properly probed first, then removing it, and adding it again by doing a 3892 // rescan. 3893 fn test_pci_bar_reprogramming() { 3894 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3895 let guest = Guest::new(Box::new(focal)); 3896 3897 #[cfg(target_arch = "x86_64")] 3898 let kernel_path = direct_kernel_boot_path(); 3899 #[cfg(target_arch = "aarch64")] 3900 let kernel_path = edk2_path(); 3901 3902 let mut child = GuestCommand::new(&guest) 3903 .args(&["--cpus", "boot=1"]) 3904 .args(&["--memory", "size=512M"]) 3905 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3906 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3907 .default_disks() 3908 .args(&[ 3909 "--net", 3910 guest.default_net_string().as_str(), 3911 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3912 ]) 3913 .capture_output() 3914 .spawn() 3915 .unwrap(); 3916 3917 let r = std::panic::catch_unwind(|| { 3918 guest.wait_vm_boot(None).unwrap(); 3919 3920 // 2 network interfaces + default localhost ==> 3 interfaces 3921 assert_eq!( 3922 guest 3923 .ssh_command("ip -o link | wc -l") 3924 .unwrap() 3925 .trim() 3926 .parse::<u32>() 3927 .unwrap_or_default(), 3928 3 3929 ); 3930 3931 let init_bar_addr = guest 3932 .ssh_command( 3933 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 3934 ) 3935 .unwrap(); 3936 3937 // Remove the PCI device 3938 guest 3939 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 3940 .unwrap(); 3941 3942 // Only 1 network interface left + default localhost ==> 2 interfaces 3943 assert_eq!( 3944 guest 3945 .ssh_command("ip -o link | wc -l") 3946 .unwrap() 3947 .trim() 3948 .parse::<u32>() 3949 .unwrap_or_default(), 3950 2 3951 ); 3952 3953 // Remove the PCI device 3954 guest 3955 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 3956 .unwrap(); 3957 3958 // Back to 2 network interface + default localhost ==> 3 interfaces 3959 assert_eq!( 3960 guest 3961 .ssh_command("ip -o link | wc -l") 3962 .unwrap() 3963 .trim() 3964 .parse::<u32>() 3965 .unwrap_or_default(), 3966 3 3967 ); 3968 3969 let new_bar_addr = guest 3970 .ssh_command( 3971 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 3972 ) 3973 .unwrap(); 3974 3975 // Let's compare the BAR addresses for our virtio-net device. 3976 // They should be different as we expect the BAR reprogramming 3977 // to have happened. 3978 assert_ne!(init_bar_addr, new_bar_addr); 3979 }); 3980 3981 let _ = child.kill(); 3982 let output = child.wait_with_output().unwrap(); 3983 3984 handle_child_output(r, &output); 3985 } 3986 3987 #[test] 3988 fn test_memory_mergeable_off() { 3989 test_memory_mergeable(false) 3990 } 3991 3992 #[test] 3993 #[cfg(target_arch = "x86_64")] 3994 fn test_cpu_hotplug() { 3995 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3996 let guest = Guest::new(Box::new(focal)); 3997 let api_socket = temp_api_path(&guest.tmp_dir); 3998 3999 let kernel_path = direct_kernel_boot_path(); 4000 4001 let mut child = GuestCommand::new(&guest) 4002 .args(&["--cpus", "boot=2,max=4"]) 4003 .args(&["--memory", "size=512M"]) 4004 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4005 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4006 .default_disks() 4007 .default_net() 4008 .args(&["--api-socket", &api_socket]) 4009 .capture_output() 4010 .spawn() 4011 .unwrap(); 4012 4013 let r = std::panic::catch_unwind(|| { 4014 guest.wait_vm_boot(None).unwrap(); 4015 4016 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4017 4018 // Resize the VM 4019 let desired_vcpus = 4; 4020 resize_command(&api_socket, Some(desired_vcpus), None, None); 4021 4022 guest 4023 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4024 .unwrap(); 4025 guest 4026 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4027 .unwrap(); 4028 thread::sleep(std::time::Duration::new(10, 0)); 4029 assert_eq!( 4030 guest.get_cpu_count().unwrap_or_default(), 4031 u32::from(desired_vcpus) 4032 ); 4033 4034 guest.reboot_linux(0, None); 4035 4036 assert_eq!( 4037 guest.get_cpu_count().unwrap_or_default(), 4038 u32::from(desired_vcpus) 4039 ); 4040 4041 // Resize the VM 4042 let desired_vcpus = 2; 4043 resize_command(&api_socket, Some(desired_vcpus), None, None); 4044 4045 thread::sleep(std::time::Duration::new(10, 0)); 4046 assert_eq!( 4047 guest.get_cpu_count().unwrap_or_default(), 4048 u32::from(desired_vcpus) 4049 ); 4050 4051 // Resize the VM back up to 4 4052 let desired_vcpus = 4; 4053 resize_command(&api_socket, Some(desired_vcpus), None, None); 4054 4055 guest 4056 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4057 .unwrap(); 4058 guest 4059 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4060 .unwrap(); 4061 thread::sleep(std::time::Duration::new(10, 0)); 4062 assert_eq!( 4063 guest.get_cpu_count().unwrap_or_default(), 4064 u32::from(desired_vcpus) 4065 ); 4066 }); 4067 4068 let _ = child.kill(); 4069 let output = child.wait_with_output().unwrap(); 4070 4071 handle_child_output(r, &output); 4072 } 4073 4074 #[test] 4075 fn test_memory_hotplug() { 4076 #[cfg(target_arch = "aarch64")] 4077 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4078 #[cfg(target_arch = "x86_64")] 4079 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4080 let focal = UbuntuDiskConfig::new(focal_image); 4081 let guest = Guest::new(Box::new(focal)); 4082 let api_socket = temp_api_path(&guest.tmp_dir); 4083 4084 #[cfg(target_arch = "aarch64")] 4085 let kernel_path = edk2_path(); 4086 #[cfg(target_arch = "x86_64")] 4087 let kernel_path = direct_kernel_boot_path(); 4088 4089 let mut child = GuestCommand::new(&guest) 4090 .args(&["--cpus", "boot=2,max=4"]) 4091 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4092 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4093 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4094 .default_disks() 4095 .default_net() 4096 .args(&["--balloon", "size=0"]) 4097 .args(&["--api-socket", &api_socket]) 4098 .capture_output() 4099 .spawn() 4100 .unwrap(); 4101 4102 let r = std::panic::catch_unwind(|| { 4103 guest.wait_vm_boot(None).unwrap(); 4104 4105 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4106 4107 guest.enable_memory_hotplug(); 4108 4109 // Add RAM to the VM 4110 let desired_ram = 1024 << 20; 4111 resize_command(&api_socket, None, Some(desired_ram), None); 4112 4113 thread::sleep(std::time::Duration::new(10, 0)); 4114 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4115 4116 // Use balloon to remove RAM from the VM 4117 let desired_balloon = 512 << 20; 4118 resize_command(&api_socket, None, None, Some(desired_balloon)); 4119 4120 thread::sleep(std::time::Duration::new(10, 0)); 4121 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4122 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4123 4124 guest.reboot_linux(0, None); 4125 4126 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4127 4128 // Use balloon add RAM to the VM 4129 let desired_balloon = 0; 4130 resize_command(&api_socket, None, None, Some(desired_balloon)); 4131 4132 thread::sleep(std::time::Duration::new(10, 0)); 4133 4134 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4135 4136 guest.enable_memory_hotplug(); 4137 4138 // Add RAM to the VM 4139 let desired_ram = 2048 << 20; 4140 resize_command(&api_socket, None, Some(desired_ram), None); 4141 4142 thread::sleep(std::time::Duration::new(10, 0)); 4143 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4144 4145 // Remove RAM to the VM (only applies after reboot) 4146 let desired_ram = 1024 << 20; 4147 resize_command(&api_socket, None, Some(desired_ram), None); 4148 4149 guest.reboot_linux(1, None); 4150 4151 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4152 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4153 }); 4154 4155 let _ = child.kill(); 4156 let output = child.wait_with_output().unwrap(); 4157 4158 handle_child_output(r, &output); 4159 } 4160 4161 #[test] 4162 #[cfg(not(feature = "mshv"))] 4163 fn test_virtio_mem() { 4164 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4165 let guest = Guest::new(Box::new(focal)); 4166 let api_socket = temp_api_path(&guest.tmp_dir); 4167 4168 let kernel_path = direct_kernel_boot_path(); 4169 4170 let mut child = GuestCommand::new(&guest) 4171 .args(&["--cpus", "boot=2,max=4"]) 4172 .args(&[ 4173 "--memory", 4174 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4175 ]) 4176 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4177 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4178 .default_disks() 4179 .default_net() 4180 .args(&["--api-socket", &api_socket]) 4181 .capture_output() 4182 .spawn() 4183 .unwrap(); 4184 4185 let r = std::panic::catch_unwind(|| { 4186 guest.wait_vm_boot(None).unwrap(); 4187 4188 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4189 4190 guest.enable_memory_hotplug(); 4191 4192 // Add RAM to the VM 4193 let desired_ram = 1024 << 20; 4194 resize_command(&api_socket, None, Some(desired_ram), None); 4195 4196 thread::sleep(std::time::Duration::new(10, 0)); 4197 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4198 4199 // Add RAM to the VM 4200 let desired_ram = 2048 << 20; 4201 resize_command(&api_socket, None, Some(desired_ram), None); 4202 4203 thread::sleep(std::time::Duration::new(10, 0)); 4204 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4205 4206 // Remove RAM from the VM 4207 let desired_ram = 1024 << 20; 4208 resize_command(&api_socket, None, Some(desired_ram), None); 4209 4210 thread::sleep(std::time::Duration::new(10, 0)); 4211 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4212 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4213 4214 guest.reboot_linux(0, None); 4215 4216 // Check the amount of memory after reboot is 1GiB 4217 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4218 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4219 4220 // Check we can still resize to 512MiB 4221 let desired_ram = 512 << 20; 4222 resize_command(&api_socket, None, Some(desired_ram), None); 4223 thread::sleep(std::time::Duration::new(10, 0)); 4224 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4225 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4226 }); 4227 4228 let _ = child.kill(); 4229 let output = child.wait_with_output().unwrap(); 4230 4231 handle_child_output(r, &output); 4232 } 4233 4234 #[test] 4235 #[cfg(target_arch = "x86_64")] 4236 #[cfg(not(feature = "mshv"))] 4237 // Test both vCPU and memory resizing together 4238 fn test_resize() { 4239 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4240 let guest = Guest::new(Box::new(focal)); 4241 let api_socket = temp_api_path(&guest.tmp_dir); 4242 4243 let kernel_path = direct_kernel_boot_path(); 4244 4245 let mut child = GuestCommand::new(&guest) 4246 .args(&["--cpus", "boot=2,max=4"]) 4247 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4248 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4249 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4250 .default_disks() 4251 .default_net() 4252 .args(&["--api-socket", &api_socket]) 4253 .capture_output() 4254 .spawn() 4255 .unwrap(); 4256 4257 let r = std::panic::catch_unwind(|| { 4258 guest.wait_vm_boot(None).unwrap(); 4259 4260 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4261 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4262 4263 guest.enable_memory_hotplug(); 4264 4265 // Resize the VM 4266 let desired_vcpus = 4; 4267 let desired_ram = 1024 << 20; 4268 resize_command(&api_socket, Some(desired_vcpus), Some(desired_ram), None); 4269 4270 guest 4271 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4272 .unwrap(); 4273 guest 4274 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4275 .unwrap(); 4276 thread::sleep(std::time::Duration::new(10, 0)); 4277 assert_eq!( 4278 guest.get_cpu_count().unwrap_or_default(), 4279 u32::from(desired_vcpus) 4280 ); 4281 4282 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4283 }); 4284 4285 let _ = child.kill(); 4286 let output = child.wait_with_output().unwrap(); 4287 4288 handle_child_output(r, &output); 4289 } 4290 4291 #[test] 4292 fn test_memory_overhead() { 4293 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4294 let guest = Guest::new(Box::new(focal)); 4295 4296 let kernel_path = direct_kernel_boot_path(); 4297 4298 let guest_memory_size_kb = 512 * 1024; 4299 4300 let mut child = GuestCommand::new(&guest) 4301 .args(&["--cpus", "boot=1"]) 4302 .args(&[ 4303 "--memory", 4304 format!("size={}K", guest_memory_size_kb).as_str(), 4305 ]) 4306 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4307 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4308 .default_disks() 4309 .capture_output() 4310 .spawn() 4311 .unwrap(); 4312 4313 thread::sleep(std::time::Duration::new(20, 0)); 4314 4315 let r = std::panic::catch_unwind(|| { 4316 // On AArch64 when acpi is enabled, there is a 4 MiB gap between the RAM 4317 // that the VMM gives and the guest can see. 4318 // This is a temporary solution, will be fixed in future. 4319 #[cfg(target_arch = "aarch64")] 4320 let guest_memory_size_kb = guest_memory_size_kb - 4 * 1024; 4321 4322 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 4323 eprintln!( 4324 "Guest memory overhead: {} vs {}", 4325 overhead, MAXIMUM_VMM_OVERHEAD_KB 4326 ); 4327 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 4328 }); 4329 4330 let _ = child.kill(); 4331 let output = child.wait_with_output().unwrap(); 4332 4333 handle_child_output(r, &output); 4334 } 4335 4336 #[test] 4337 fn test_disk_hotplug() { 4338 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4339 let guest = Guest::new(Box::new(focal)); 4340 4341 #[cfg(target_arch = "x86_64")] 4342 let kernel_path = direct_kernel_boot_path(); 4343 #[cfg(target_arch = "aarch64")] 4344 let kernel_path = edk2_path(); 4345 4346 let api_socket = temp_api_path(&guest.tmp_dir); 4347 4348 let mut child = GuestCommand::new(&guest) 4349 .args(&["--api-socket", &api_socket]) 4350 .args(&["--cpus", "boot=1"]) 4351 .args(&["--memory", "size=512M"]) 4352 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4353 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4354 .default_disks() 4355 .default_net() 4356 .capture_output() 4357 .spawn() 4358 .unwrap(); 4359 4360 let r = std::panic::catch_unwind(|| { 4361 guest.wait_vm_boot(None).unwrap(); 4362 4363 // Check /dev/vdc is not there 4364 assert_eq!( 4365 guest 4366 .ssh_command("lsblk | grep -c vdc.*16M || true") 4367 .unwrap() 4368 .trim() 4369 .parse::<u32>() 4370 .unwrap_or(1), 4371 0 4372 ); 4373 4374 // Now let's add the extra disk. 4375 let mut blk_file_path = dirs::home_dir().unwrap(); 4376 blk_file_path.push("workloads"); 4377 blk_file_path.push("blk.img"); 4378 let (cmd_success, cmd_output) = remote_command_w_output( 4379 &api_socket, 4380 "add-disk", 4381 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4382 ); 4383 assert!(cmd_success); 4384 assert!(String::from_utf8_lossy(&cmd_output) 4385 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4386 4387 thread::sleep(std::time::Duration::new(10, 0)); 4388 4389 // Check that /dev/vdc exists and the block size is 16M. 4390 assert_eq!( 4391 guest 4392 .ssh_command("lsblk | grep vdc | grep -c 16M") 4393 .unwrap() 4394 .trim() 4395 .parse::<u32>() 4396 .unwrap_or_default(), 4397 1 4398 ); 4399 // And check the block device can be read. 4400 guest 4401 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4402 .unwrap(); 4403 4404 // Let's remove it the extra disk. 4405 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4406 thread::sleep(std::time::Duration::new(5, 0)); 4407 // And check /dev/vdc is not there 4408 assert_eq!( 4409 guest 4410 .ssh_command("lsblk | grep -c vdc.*16M || true") 4411 .unwrap() 4412 .trim() 4413 .parse::<u32>() 4414 .unwrap_or(1), 4415 0 4416 ); 4417 4418 // And add it back to validate unplug did work correctly. 4419 let (cmd_success, cmd_output) = remote_command_w_output( 4420 &api_socket, 4421 "add-disk", 4422 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4423 ); 4424 assert!(cmd_success); 4425 assert!(String::from_utf8_lossy(&cmd_output) 4426 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4427 4428 thread::sleep(std::time::Duration::new(10, 0)); 4429 4430 // Check that /dev/vdc exists and the block size is 16M. 4431 assert_eq!( 4432 guest 4433 .ssh_command("lsblk | grep vdc | grep -c 16M") 4434 .unwrap() 4435 .trim() 4436 .parse::<u32>() 4437 .unwrap_or_default(), 4438 1 4439 ); 4440 // And check the block device can be read. 4441 guest 4442 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4443 .unwrap(); 4444 4445 // Reboot the VM. 4446 guest.reboot_linux(0, None); 4447 4448 // Check still there after reboot 4449 assert_eq!( 4450 guest 4451 .ssh_command("lsblk | grep vdc | grep -c 16M") 4452 .unwrap() 4453 .trim() 4454 .parse::<u32>() 4455 .unwrap_or_default(), 4456 1 4457 ); 4458 4459 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4460 4461 thread::sleep(std::time::Duration::new(20, 0)); 4462 4463 // Check device has gone away 4464 assert_eq!( 4465 guest 4466 .ssh_command("lsblk | grep -c vdc.*16M || true") 4467 .unwrap() 4468 .trim() 4469 .parse::<u32>() 4470 .unwrap_or(1), 4471 0 4472 ); 4473 4474 guest.reboot_linux(1, None); 4475 4476 // Check device still absent 4477 assert_eq!( 4478 guest 4479 .ssh_command("lsblk | grep -c vdc.*16M || true") 4480 .unwrap() 4481 .trim() 4482 .parse::<u32>() 4483 .unwrap_or(1), 4484 0 4485 ); 4486 }); 4487 4488 let _ = child.kill(); 4489 let output = child.wait_with_output().unwrap(); 4490 4491 handle_child_output(r, &output); 4492 } 4493 4494 #[test] 4495 #[cfg(target_arch = "x86_64")] 4496 fn test_virtio_block_topology() { 4497 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4498 let guest = Guest::new(Box::new(focal)); 4499 4500 let kernel_path = direct_kernel_boot_path(); 4501 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 4502 4503 Command::new("qemu-img") 4504 .args(&[ 4505 "create", 4506 "-f", 4507 "raw", 4508 test_disk_path.to_str().unwrap(), 4509 "16M", 4510 ]) 4511 .output() 4512 .expect("qemu-img command failed"); 4513 let out = Command::new("losetup") 4514 .args(&[ 4515 "--show", 4516 "--find", 4517 "--sector-size=4096", 4518 test_disk_path.to_str().unwrap(), 4519 ]) 4520 .output() 4521 .expect("failed to create loop device") 4522 .stdout; 4523 let _tmp = String::from_utf8_lossy(&out); 4524 let loop_dev = _tmp.trim(); 4525 4526 let mut child = GuestCommand::new(&guest) 4527 .args(&["--cpus", "boot=1"]) 4528 .args(&["--memory", "size=512M"]) 4529 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4530 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4531 .args(&[ 4532 "--disk", 4533 format!( 4534 "path={}", 4535 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4536 ) 4537 .as_str(), 4538 format!( 4539 "path={}", 4540 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4541 ) 4542 .as_str(), 4543 format!("path={}", loop_dev).as_str(), 4544 ]) 4545 .default_net() 4546 .capture_output() 4547 .spawn() 4548 .unwrap(); 4549 4550 let r = std::panic::catch_unwind(|| { 4551 guest.wait_vm_boot(None).unwrap(); 4552 4553 // MIN-IO column 4554 assert_eq!( 4555 guest 4556 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 4557 .unwrap() 4558 .trim() 4559 .parse::<u32>() 4560 .unwrap_or_default(), 4561 4096 4562 ); 4563 // PHY-SEC column 4564 assert_eq!( 4565 guest 4566 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 4567 .unwrap() 4568 .trim() 4569 .parse::<u32>() 4570 .unwrap_or_default(), 4571 4096 4572 ); 4573 // LOG-SEC column 4574 assert_eq!( 4575 guest 4576 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 4577 .unwrap() 4578 .trim() 4579 .parse::<u32>() 4580 .unwrap_or_default(), 4581 4096 4582 ); 4583 }); 4584 4585 let _ = child.kill(); 4586 let output = child.wait_with_output().unwrap(); 4587 4588 handle_child_output(r, &output); 4589 4590 Command::new("losetup") 4591 .args(&["-d", loop_dev]) 4592 .output() 4593 .expect("loop device not found"); 4594 } 4595 4596 #[test] 4597 fn test_virtio_balloon_deflate_on_oom() { 4598 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4599 let guest = Guest::new(Box::new(focal)); 4600 4601 let kernel_path = direct_kernel_boot_path(); 4602 4603 let api_socket = temp_api_path(&guest.tmp_dir); 4604 4605 //Let's start a 4G guest with balloon occupied 2G memory 4606 let mut child = GuestCommand::new(&guest) 4607 .args(&["--api-socket", &api_socket]) 4608 .args(&["--cpus", "boot=1"]) 4609 .args(&["--memory", "size=4G"]) 4610 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4611 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4612 .args(&["--balloon", "size=2G,deflate_on_oom=on"]) 4613 .default_disks() 4614 .default_net() 4615 .capture_output() 4616 .spawn() 4617 .unwrap(); 4618 4619 let r = std::panic::catch_unwind(|| { 4620 guest.wait_vm_boot(None).unwrap(); 4621 4622 // Wait for balloon memory's initialization and check its size. 4623 // The virtio-balloon driver might take a few seconds to report the 4624 // balloon effective size back to the VMM. 4625 thread::sleep(std::time::Duration::new(20, 0)); 4626 4627 let orig_balloon = balloon_size(&api_socket); 4628 println!("The original balloon memory size is {} bytes", orig_balloon); 4629 assert!(orig_balloon == 2147483648); 4630 4631 // Two steps to verify if the 'deflate_on_oom' parameter works. 4632 // 1st: run a command in guest to eat up memory heavily, which 4633 // will consume much more memory than $(total_mem - balloon_size) 4634 // to trigger an oom. 4635 guest 4636 .ssh_command("stress --vm 25 --vm-keep --vm-bytes 1G --timeout 20") 4637 .unwrap(); 4638 4639 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 4640 let deflated_balloon = balloon_size(&api_socket); 4641 println!( 4642 "After deflating, balloon memory size is {} bytes", 4643 deflated_balloon 4644 ); 4645 // Verify the balloon size deflated 4646 assert!(deflated_balloon < 2147483648); 4647 }); 4648 4649 let _ = child.kill(); 4650 let output = child.wait_with_output().unwrap(); 4651 4652 handle_child_output(r, &output); 4653 } 4654 4655 #[test] 4656 fn test_virtio_balloon_free_page_reporting() { 4657 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4658 let guest = Guest::new(Box::new(focal)); 4659 4660 //Let's start a 4G guest with balloon occupied 2G memory 4661 let mut child = GuestCommand::new(&guest) 4662 .args(&["--cpus", "boot=1"]) 4663 .args(&["--memory", "size=4G"]) 4664 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4665 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4666 .args(&["--balloon", "size=0,free_page_reporting=on"]) 4667 .default_disks() 4668 .default_net() 4669 .capture_output() 4670 .spawn() 4671 .unwrap(); 4672 4673 let pid = child.id(); 4674 let r = std::panic::catch_unwind(|| { 4675 guest.wait_vm_boot(None).unwrap(); 4676 4677 // Check the initial RSS is less than 1GiB 4678 let rss = process_rss_kib(pid); 4679 println!("RSS {} < 1048576", rss); 4680 assert!(rss < 1048576); 4681 4682 // Spawn a command inside the guest to consume 2GiB of RAM for 60 4683 // seconds 4684 let guest_ip = guest.network.guest_ip.clone(); 4685 thread::spawn(move || { 4686 ssh_command_ip( 4687 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 4688 &guest_ip, 4689 DEFAULT_SSH_RETRIES, 4690 DEFAULT_SSH_TIMEOUT, 4691 ) 4692 .unwrap(); 4693 }); 4694 4695 // Wait for 50 seconds to make sure the stress command is consuming 4696 // the expected amount of memory. 4697 thread::sleep(std::time::Duration::new(50, 0)); 4698 let rss = process_rss_kib(pid); 4699 println!("RSS {} >= 2097152", rss); 4700 assert!(rss >= 2097152); 4701 4702 // Wait for an extra minute to make sure the stress command has 4703 // completed and that the guest reported the free pages to the VMM 4704 // through the virtio-balloon device. We expect the RSS to be under 4705 // 2GiB. 4706 thread::sleep(std::time::Duration::new(60, 0)); 4707 let rss = process_rss_kib(pid); 4708 println!("RSS {} < 2097152", rss); 4709 assert!(rss < 2097152); 4710 }); 4711 4712 let _ = child.kill(); 4713 let output = child.wait_with_output().unwrap(); 4714 4715 handle_child_output(r, &output); 4716 } 4717 4718 #[test] 4719 fn test_pmem_hotplug() { 4720 _test_pmem_hotplug(None) 4721 } 4722 4723 #[test] 4724 fn test_pmem_multi_segment_hotplug() { 4725 _test_pmem_hotplug(Some(15)) 4726 } 4727 4728 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 4729 #[cfg(target_arch = "aarch64")] 4730 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4731 #[cfg(target_arch = "x86_64")] 4732 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4733 let focal = UbuntuDiskConfig::new(focal_image); 4734 let guest = Guest::new(Box::new(focal)); 4735 4736 #[cfg(target_arch = "x86_64")] 4737 let kernel_path = direct_kernel_boot_path(); 4738 #[cfg(target_arch = "aarch64")] 4739 let kernel_path = edk2_path(); 4740 4741 let api_socket = temp_api_path(&guest.tmp_dir); 4742 4743 let mut cmd = GuestCommand::new(&guest); 4744 4745 cmd.args(&["--api-socket", &api_socket]) 4746 .args(&["--cpus", "boot=1"]) 4747 .args(&["--memory", "size=512M"]) 4748 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4749 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4750 .default_disks() 4751 .default_net() 4752 .capture_output(); 4753 4754 if pci_segment.is_some() { 4755 cmd.args(&["--platform", "num_pci_segments=16"]); 4756 } 4757 4758 let mut child = cmd.spawn().unwrap(); 4759 4760 let r = std::panic::catch_unwind(|| { 4761 guest.wait_vm_boot(None).unwrap(); 4762 4763 // Check /dev/pmem0 is not there 4764 assert_eq!( 4765 guest 4766 .ssh_command("lsblk | grep -c pmem0 || true") 4767 .unwrap() 4768 .trim() 4769 .parse::<u32>() 4770 .unwrap_or(1), 4771 0 4772 ); 4773 4774 let pmem_temp_file = TempFile::new().unwrap(); 4775 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 4776 let (cmd_success, cmd_output) = remote_command_w_output( 4777 &api_socket, 4778 "add-pmem", 4779 Some(&format!( 4780 "file={},id=test0{}", 4781 pmem_temp_file.as_path().to_str().unwrap(), 4782 if let Some(pci_segment) = pci_segment { 4783 format!(",pci_segment={}", pci_segment) 4784 } else { 4785 "".to_owned() 4786 } 4787 )), 4788 ); 4789 assert!(cmd_success); 4790 if let Some(pci_segment) = pci_segment { 4791 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 4792 "{{\"id\":\"test0\",\"bdf\":\"{:04x}:00:01.0\"}}", 4793 pci_segment 4794 ))); 4795 } else { 4796 assert!(String::from_utf8_lossy(&cmd_output) 4797 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4798 } 4799 4800 // Check that /dev/pmem0 exists and the block size is 128M 4801 assert_eq!( 4802 guest 4803 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 4804 .unwrap() 4805 .trim() 4806 .parse::<u32>() 4807 .unwrap_or_default(), 4808 1 4809 ); 4810 4811 guest.reboot_linux(0, None); 4812 4813 // Check still there after reboot 4814 assert_eq!( 4815 guest 4816 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 4817 .unwrap() 4818 .trim() 4819 .parse::<u32>() 4820 .unwrap_or_default(), 4821 1 4822 ); 4823 4824 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4825 4826 thread::sleep(std::time::Duration::new(20, 0)); 4827 4828 // Check device has gone away 4829 assert_eq!( 4830 guest 4831 .ssh_command("lsblk | grep -c pmem0.*128M || true") 4832 .unwrap() 4833 .trim() 4834 .parse::<u32>() 4835 .unwrap_or(1), 4836 0 4837 ); 4838 4839 guest.reboot_linux(1, None); 4840 4841 // Check still absent after reboot 4842 assert_eq!( 4843 guest 4844 .ssh_command("lsblk | grep -c pmem0.*128M || true") 4845 .unwrap() 4846 .trim() 4847 .parse::<u32>() 4848 .unwrap_or(1), 4849 0 4850 ); 4851 }); 4852 4853 let _ = child.kill(); 4854 let output = child.wait_with_output().unwrap(); 4855 4856 handle_child_output(r, &output); 4857 } 4858 4859 #[test] 4860 fn test_net_hotplug() { 4861 _test_net_hotplug(None) 4862 } 4863 4864 #[test] 4865 fn test_net_multi_segment_hotplug() { 4866 _test_net_hotplug(Some(15)) 4867 } 4868 4869 fn _test_net_hotplug(pci_segment: Option<u16>) { 4870 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4871 let guest = Guest::new(Box::new(focal)); 4872 4873 #[cfg(target_arch = "x86_64")] 4874 let kernel_path = direct_kernel_boot_path(); 4875 #[cfg(target_arch = "aarch64")] 4876 let kernel_path = edk2_path(); 4877 4878 let api_socket = temp_api_path(&guest.tmp_dir); 4879 4880 // Boot without network 4881 let mut cmd = GuestCommand::new(&guest); 4882 4883 cmd.args(&["--api-socket", &api_socket]) 4884 .args(&["--cpus", "boot=1"]) 4885 .args(&["--memory", "size=512M"]) 4886 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4887 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4888 .default_disks() 4889 .capture_output(); 4890 4891 if pci_segment.is_some() { 4892 cmd.args(&["--platform", "num_pci_segments=16"]); 4893 } 4894 4895 let mut child = cmd.spawn().unwrap(); 4896 4897 thread::sleep(std::time::Duration::new(20, 0)); 4898 4899 let r = std::panic::catch_unwind(|| { 4900 // Add network 4901 let (cmd_success, cmd_output) = remote_command_w_output( 4902 &api_socket, 4903 "add-net", 4904 Some( 4905 format!( 4906 "{}{},id=test0", 4907 guest.default_net_string(), 4908 if let Some(pci_segment) = pci_segment { 4909 format!(",pci_segment={}", pci_segment) 4910 } else { 4911 "".to_owned() 4912 } 4913 ) 4914 .as_str(), 4915 ), 4916 ); 4917 assert!(cmd_success); 4918 4919 if let Some(pci_segment) = pci_segment { 4920 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 4921 "{{\"id\":\"test0\",\"bdf\":\"{:04x}:00:01.0\"}}", 4922 pci_segment 4923 ))); 4924 } else { 4925 assert!(String::from_utf8_lossy(&cmd_output) 4926 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 4927 } 4928 4929 thread::sleep(std::time::Duration::new(5, 0)); 4930 4931 // 1 network interfaces + default localhost ==> 2 interfaces 4932 assert_eq!( 4933 guest 4934 .ssh_command("ip -o link | wc -l") 4935 .unwrap() 4936 .trim() 4937 .parse::<u32>() 4938 .unwrap_or_default(), 4939 2 4940 ); 4941 4942 // Remove network 4943 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 4944 thread::sleep(std::time::Duration::new(5, 0)); 4945 4946 let (cmd_success, cmd_output) = remote_command_w_output( 4947 &api_socket, 4948 "add-net", 4949 Some( 4950 format!( 4951 "{}{},id=test1", 4952 guest.default_net_string(), 4953 if let Some(pci_segment) = pci_segment { 4954 format!(",pci_segment={}", pci_segment) 4955 } else { 4956 "".to_owned() 4957 } 4958 ) 4959 .as_str(), 4960 ), 4961 ); 4962 assert!(cmd_success); 4963 4964 if let Some(pci_segment) = pci_segment { 4965 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 4966 "{{\"id\":\"test1\",\"bdf\":\"{:04x}:00:01.0\"}}", 4967 pci_segment 4968 ))); 4969 } else { 4970 assert!(String::from_utf8_lossy(&cmd_output) 4971 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 4972 } 4973 4974 thread::sleep(std::time::Duration::new(5, 0)); 4975 4976 // 1 network interfaces + default localhost ==> 2 interfaces 4977 assert_eq!( 4978 guest 4979 .ssh_command("ip -o link | wc -l") 4980 .unwrap() 4981 .trim() 4982 .parse::<u32>() 4983 .unwrap_or_default(), 4984 2 4985 ); 4986 4987 guest.reboot_linux(0, None); 4988 4989 // Check still there after reboot 4990 // 1 network interfaces + default localhost ==> 2 interfaces 4991 assert_eq!( 4992 guest 4993 .ssh_command("ip -o link | wc -l") 4994 .unwrap() 4995 .trim() 4996 .parse::<u32>() 4997 .unwrap_or_default(), 4998 2 4999 ); 5000 }); 5001 5002 let _ = child.kill(); 5003 let output = child.wait_with_output().unwrap(); 5004 5005 handle_child_output(r, &output); 5006 } 5007 5008 #[test] 5009 fn test_initramfs() { 5010 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5011 let guest = Guest::new(Box::new(focal)); 5012 let mut workload_path = dirs::home_dir().unwrap(); 5013 workload_path.push("workloads"); 5014 5015 #[cfg(target_arch = "x86_64")] 5016 let mut kernels = vec![direct_kernel_boot_path()]; 5017 #[cfg(target_arch = "aarch64")] 5018 let kernels = vec![direct_kernel_boot_path()]; 5019 5020 #[cfg(target_arch = "x86_64")] 5021 { 5022 let mut pvh_kernel_path = workload_path.clone(); 5023 pvh_kernel_path.push("vmlinux"); 5024 kernels.push(pvh_kernel_path); 5025 } 5026 5027 let mut initramfs_path = workload_path; 5028 initramfs_path.push("alpine_initramfs.img"); 5029 5030 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5031 let cmdline = format!("console=hvc0 quiet TEST_STRING={}", test_string); 5032 5033 kernels.iter().for_each(|k_path| { 5034 let mut child = GuestCommand::new(&guest) 5035 .args(&["--kernel", k_path.to_str().unwrap()]) 5036 .args(&["--initramfs", initramfs_path.to_str().unwrap()]) 5037 .args(&["--cmdline", &cmdline]) 5038 .capture_output() 5039 .spawn() 5040 .unwrap(); 5041 5042 thread::sleep(std::time::Duration::new(20, 0)); 5043 5044 let _ = child.kill(); 5045 let output = child.wait_with_output().unwrap(); 5046 5047 let r = std::panic::catch_unwind(|| { 5048 let s = String::from_utf8_lossy(&output.stdout); 5049 5050 assert_ne!(s.lines().position(|line| line == test_string), None); 5051 }); 5052 5053 handle_child_output(r, &output); 5054 }); 5055 } 5056 5057 // One thing to note about this test. The virtio-net device is heavily used 5058 // through each ssh command. There's no need to perform a dedicated test to 5059 // verify the migration went well for virtio-net. 5060 #[test] 5061 fn test_snapshot_restore() { 5062 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5063 let guest = Guest::new(Box::new(focal)); 5064 let kernel_path = direct_kernel_boot_path(); 5065 5066 let api_socket = temp_api_path(&guest.tmp_dir); 5067 5068 let net_id = "net123"; 5069 let net_params = format!( 5070 "id={},tap=,mac={},ip={},mask=255.255.255.0", 5071 net_id, guest.network.guest_mac, guest.network.host_ip 5072 ); 5073 5074 let cloudinit_params = format!( 5075 "path={},iommu=on", 5076 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5077 ); 5078 5079 let socket = temp_vsock_path(&guest.tmp_dir); 5080 5081 let mut child = GuestCommand::new(&guest) 5082 .args(&["--api-socket", &api_socket]) 5083 .args(&["--cpus", "boot=4"]) 5084 .args(&[ 5085 "--memory", 5086 "size=4G,hotplug_method=virtio-mem,hotplug_size=32G", 5087 ]) 5088 .args(&["--balloon", "size=0"]) 5089 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5090 .args(&[ 5091 "--disk", 5092 format!( 5093 "path={}", 5094 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5095 ) 5096 .as_str(), 5097 cloudinit_params.as_str(), 5098 ]) 5099 .args(&["--net", net_params.as_str()]) 5100 .args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]) 5101 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5102 .capture_output() 5103 .spawn() 5104 .unwrap(); 5105 5106 let console_text = String::from("On a branch floating down river a cricket, singing."); 5107 // Create the snapshot directory 5108 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 5109 5110 let r = std::panic::catch_unwind(|| { 5111 guest.wait_vm_boot(None).unwrap(); 5112 5113 // Check the number of vCPUs 5114 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5115 // Check the guest RAM 5116 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5117 // Increase guest RAM with virtio-mem 5118 resize_command(&api_socket, None, Some(6 << 30), None); 5119 thread::sleep(std::time::Duration::new(5, 0)); 5120 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5121 // Use balloon to remove RAM from the VM 5122 resize_command(&api_socket, None, None, Some(1 << 30)); 5123 thread::sleep(std::time::Duration::new(5, 0)); 5124 let total_memory = guest.get_total_memory().unwrap_or_default(); 5125 assert!(total_memory > 4_800_000); 5126 assert!(total_memory < 5_760_000); 5127 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 5128 guest.check_devices_common(Some(&socket), Some(&console_text)); 5129 5130 // x86_64: We check that removing and adding back the virtio-net device 5131 // does not break the snapshot/restore support for virtio-pci. 5132 // This is an important thing to test as the hotplug will 5133 // trigger a PCI BAR reprogramming, which is a good way of 5134 // checking if the stored resources are correctly restored. 5135 // Unplug the virtio-net device 5136 // AArch64: Device hotplug is currently not supported, skipping here. 5137 #[cfg(target_arch = "x86_64")] 5138 { 5139 assert!(remote_command(&api_socket, "remove-device", Some(net_id),)); 5140 thread::sleep(std::time::Duration::new(10, 0)); 5141 5142 // Plug the virtio-net device again 5143 assert!(remote_command( 5144 &api_socket, 5145 "add-net", 5146 Some(net_params.as_str()), 5147 )); 5148 thread::sleep(std::time::Duration::new(10, 0)); 5149 } 5150 5151 // Pause the VM 5152 assert!(remote_command(&api_socket, "pause", None)); 5153 5154 // Take a snapshot from the VM 5155 assert!(remote_command( 5156 &api_socket, 5157 "snapshot", 5158 Some(format!("file://{}", snapshot_dir).as_str()), 5159 )); 5160 5161 // Wait to make sure the snapshot is completed 5162 thread::sleep(std::time::Duration::new(10, 0)); 5163 }); 5164 5165 // Shutdown the source VM and check console output 5166 let _ = child.kill(); 5167 let output = child.wait_with_output().unwrap(); 5168 handle_child_output(r, &output); 5169 5170 let r = std::panic::catch_unwind(|| { 5171 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5172 }); 5173 5174 handle_child_output(r, &output); 5175 5176 // Remove the vsock socket file. 5177 Command::new("rm") 5178 .arg("-f") 5179 .arg(socket.as_str()) 5180 .output() 5181 .unwrap(); 5182 5183 // Restore the VM from the snapshot 5184 let mut child = GuestCommand::new(&guest) 5185 .args(&["--api-socket", &api_socket]) 5186 .args(&[ 5187 "--restore", 5188 format!("source_url=file://{}", snapshot_dir).as_str(), 5189 ]) 5190 .capture_output() 5191 .spawn() 5192 .unwrap(); 5193 5194 // Wait for the VM to be restored 5195 thread::sleep(std::time::Duration::new(10, 0)); 5196 5197 let r = std::panic::catch_unwind(|| { 5198 // Resume the VM 5199 assert!(remote_command(&api_socket, "resume", None)); 5200 5201 // Perform same checks to validate VM has been properly restored 5202 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5203 let total_memory = guest.get_total_memory().unwrap_or_default(); 5204 assert!(total_memory > 4_800_000); 5205 assert!(total_memory < 5_760_000); 5206 // Deflate balloon to restore entire RAM to the VM 5207 resize_command(&api_socket, None, None, Some(0)); 5208 thread::sleep(std::time::Duration::new(5, 0)); 5209 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5210 // Decrease guest RAM with virtio-mem 5211 resize_command(&api_socket, None, Some(5 << 30), None); 5212 thread::sleep(std::time::Duration::new(5, 0)); 5213 let total_memory = guest.get_total_memory().unwrap_or_default(); 5214 assert!(total_memory > 4_800_000); 5215 assert!(total_memory < 5_760_000); 5216 5217 guest.check_devices_common(Some(&socket), Some(&console_text)); 5218 }); 5219 // Shutdown the target VM and check console output 5220 let _ = child.kill(); 5221 let output = child.wait_with_output().unwrap(); 5222 handle_child_output(r, &output); 5223 5224 let r = std::panic::catch_unwind(|| { 5225 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5226 }); 5227 5228 handle_child_output(r, &output); 5229 } 5230 5231 #[test] 5232 fn test_counters() { 5233 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5234 let guest = Guest::new(Box::new(focal)); 5235 let api_socket = temp_api_path(&guest.tmp_dir); 5236 5237 let mut cmd = GuestCommand::new(&guest); 5238 cmd.args(&["--cpus", "boot=1"]) 5239 .args(&["--memory", "size=512M"]) 5240 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5241 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5242 .default_disks() 5243 .args(&["--net", guest.default_net_string().as_str()]) 5244 .args(&["--api-socket", &api_socket]) 5245 .capture_output(); 5246 5247 let mut child = cmd.spawn().unwrap(); 5248 5249 let r = std::panic::catch_unwind(|| { 5250 guest.wait_vm_boot(None).unwrap(); 5251 5252 let orig_counters = get_counters(&api_socket); 5253 guest 5254 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 5255 .unwrap(); 5256 5257 let new_counters = get_counters(&api_socket); 5258 5259 // Check that all the counters have increased 5260 assert!(new_counters > orig_counters); 5261 }); 5262 5263 let _ = child.kill(); 5264 let output = child.wait_with_output().unwrap(); 5265 5266 handle_child_output(r, &output); 5267 } 5268 5269 #[test] 5270 fn test_watchdog() { 5271 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5272 let guest = Guest::new(Box::new(focal)); 5273 let api_socket = temp_api_path(&guest.tmp_dir); 5274 5275 let kernel_path = direct_kernel_boot_path(); 5276 5277 let mut cmd = GuestCommand::new(&guest); 5278 cmd.args(&["--cpus", "boot=1"]) 5279 .args(&["--memory", "size=512M"]) 5280 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5281 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5282 .default_disks() 5283 .args(&["--net", guest.default_net_string().as_str()]) 5284 .args(&["--watchdog"]) 5285 .args(&["--api-socket", &api_socket]) 5286 .capture_output(); 5287 5288 let mut child = cmd.spawn().unwrap(); 5289 5290 let r = std::panic::catch_unwind(|| { 5291 guest.wait_vm_boot(None).unwrap(); 5292 5293 // Check for PCI device 5294 assert!(guest 5295 .does_device_vendor_pair_match("0x1063", "0x1af4") 5296 .unwrap_or_default()); 5297 5298 // Enable systemd watchdog 5299 guest 5300 .ssh_command("echo RuntimeWatchdogSec=15s | sudo tee -a /etc/systemd/system.conf") 5301 .unwrap(); 5302 5303 guest.ssh_command("sudo reboot").unwrap(); 5304 5305 guest.wait_vm_boot(None).unwrap(); 5306 5307 // Check that systemd has activated the watchdog 5308 assert_eq!( 5309 guest 5310 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 5311 .unwrap() 5312 .trim() 5313 .parse::<u32>() 5314 .unwrap_or_default(), 5315 2 5316 ); 5317 5318 // Ensure that the current boot journal is written so reboot counts are valid 5319 guest.ssh_command("sudo journalctl --sync").unwrap(); 5320 5321 let boot_count = guest 5322 .ssh_command("sudo journalctl --list-boots | wc -l") 5323 .unwrap() 5324 .trim() 5325 .parse::<u32>() 5326 .unwrap_or_default(); 5327 assert_eq!(boot_count, 2); 5328 // Allow some normal time to elapse to check we don't get spurious reboots 5329 thread::sleep(std::time::Duration::new(40, 0)); 5330 5331 // Check no reboot 5332 let boot_count = guest 5333 .ssh_command("sudo journalctl --list-boots | wc -l") 5334 .unwrap() 5335 .trim() 5336 .parse::<u32>() 5337 .unwrap_or_default(); 5338 assert_eq!(boot_count, 2); 5339 5340 // Ensure that the current boot journal is written so reboot counts are valid 5341 guest.ssh_command("sudo journalctl --sync").unwrap(); 5342 5343 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 5344 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 5345 5346 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 5347 guest.wait_vm_boot(Some(50)).unwrap(); 5348 5349 // Check that watchdog triggered reboot 5350 let boot_count = guest 5351 .ssh_command("sudo journalctl --list-boots | wc -l") 5352 .unwrap() 5353 .trim() 5354 .parse::<u32>() 5355 .unwrap_or_default(); 5356 assert_eq!(boot_count, 3); 5357 5358 #[cfg(target_arch = "x86_64")] 5359 { 5360 // Now pause the VM and remain offline for 30s 5361 assert!(remote_command(&api_socket, "pause", None)); 5362 thread::sleep(std::time::Duration::new(30, 0)); 5363 assert!(remote_command(&api_socket, "resume", None)); 5364 5365 // Check no reboot 5366 let boot_count = guest 5367 .ssh_command("sudo journalctl --list-boots | wc -l") 5368 .unwrap() 5369 .trim() 5370 .parse::<u32>() 5371 .unwrap_or_default(); 5372 assert_eq!(boot_count, 3); 5373 } 5374 }); 5375 5376 let _ = child.kill(); 5377 let output = child.wait_with_output().unwrap(); 5378 5379 handle_child_output(r, &output); 5380 } 5381 5382 #[test] 5383 fn test_tap_from_fd() { 5384 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5385 let guest = Guest::new(Box::new(focal)); 5386 let kernel_path = direct_kernel_boot_path(); 5387 5388 // Create a TAP interface with multi-queue enabled 5389 let num_queue_pairs: usize = 2; 5390 5391 use std::str::FromStr; 5392 let taps = net_util::open_tap( 5393 Some("chtap0"), 5394 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 5395 None, 5396 &mut None, 5397 num_queue_pairs, 5398 Some(libc::O_RDWR | libc::O_NONBLOCK), 5399 ) 5400 .unwrap(); 5401 5402 let mut child = GuestCommand::new(&guest) 5403 .args(&["--cpus", &format!("boot={}", num_queue_pairs)]) 5404 .args(&["--memory", "size=512M"]) 5405 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5406 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5407 .default_disks() 5408 .args(&[ 5409 "--net", 5410 &format!( 5411 "fd=[{},{}],mac={},num_queues={}", 5412 taps[0].as_raw_fd(), 5413 taps[1].as_raw_fd(), 5414 guest.network.guest_mac, 5415 num_queue_pairs * 2 5416 ), 5417 ]) 5418 .capture_output() 5419 .spawn() 5420 .unwrap(); 5421 5422 let r = std::panic::catch_unwind(|| { 5423 guest.wait_vm_boot(None).unwrap(); 5424 5425 assert_eq!( 5426 guest 5427 .ssh_command("ip -o link | wc -l") 5428 .unwrap() 5429 .trim() 5430 .parse::<u32>() 5431 .unwrap_or_default(), 5432 2 5433 ); 5434 5435 guest.reboot_linux(0, None); 5436 5437 assert_eq!( 5438 guest 5439 .ssh_command("ip -o link | wc -l") 5440 .unwrap() 5441 .trim() 5442 .parse::<u32>() 5443 .unwrap_or_default(), 5444 2 5445 ); 5446 }); 5447 5448 let _ = child.kill(); 5449 let output = child.wait_with_output().unwrap(); 5450 5451 handle_child_output(r, &output); 5452 } 5453 5454 // By design, a guest VM won't be able to connect to the host 5455 // machine when using a macvtap network interface (while it can 5456 // communicate externally). As a workaround, this integration 5457 // test creates two macvtap interfaces in 'bridge' mode on the 5458 // same physical net interface, one for the guest and one for 5459 // the host. With additional setup on the IP address and the 5460 // routing table, it enables the communications between the 5461 // guest VM and the host machine. 5462 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 5463 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 5464 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5465 let guest = Guest::new(Box::new(focal)); 5466 let api_socket = temp_api_path(&guest.tmp_dir); 5467 5468 #[cfg(target_arch = "x86_64")] 5469 let kernel_path = direct_kernel_boot_path(); 5470 #[cfg(target_arch = "aarch64")] 5471 let kernel_path = edk2_path(); 5472 5473 let phy_net = "eth0"; 5474 5475 // Create a macvtap interface for the guest VM to use 5476 assert!(exec_host_command_status(&format!( 5477 "sudo ip link add link {} name {} type macvtap mod bridge", 5478 phy_net, guest_macvtap_name 5479 )) 5480 .success()); 5481 assert!(exec_host_command_status(&format!( 5482 "sudo ip link set {} address {} up", 5483 guest_macvtap_name, guest.network.guest_mac 5484 )) 5485 .success()); 5486 assert!( 5487 exec_host_command_status(&format!("sudo ip link show {}", guest_macvtap_name)) 5488 .success() 5489 ); 5490 5491 let tap_index = 5492 fs::read_to_string(&format!("/sys/class/net/{}/ifindex", guest_macvtap_name)).unwrap(); 5493 let tap_device = format!("/dev/tap{}", tap_index.trim()); 5494 5495 assert!( 5496 exec_host_command_status(&format!("sudo chown $UID.$UID {}", tap_device)).success() 5497 ); 5498 5499 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 5500 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 5501 assert!(tap_fd1 > 0); 5502 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 5503 assert!(tap_fd2 > 0); 5504 5505 // Create a macvtap on the same physical net interface for 5506 // the host machine to use 5507 assert!(exec_host_command_status(&format!( 5508 "sudo ip link add link {} name {} type macvtap mod bridge", 5509 phy_net, host_macvtap_name 5510 )) 5511 .success()); 5512 // Use default mask "255.255.255.0" 5513 assert!(exec_host_command_status(&format!( 5514 "sudo ip address add {}/24 dev {}", 5515 guest.network.host_ip, host_macvtap_name 5516 )) 5517 .success()); 5518 assert!(exec_host_command_status(&format!( 5519 "sudo ip link set dev {} up", 5520 host_macvtap_name 5521 )) 5522 .success()); 5523 5524 let mut guest_command = GuestCommand::new(&guest); 5525 guest_command 5526 .args(&["--cpus", "boot=2"]) 5527 .args(&["--memory", "size=512M"]) 5528 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5529 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5530 .default_disks() 5531 .args(&["--api-socket", &api_socket]); 5532 5533 let net_params = format!( 5534 "fd=[{},{}],mac={},num_queues=4", 5535 tap_fd1, tap_fd2, guest.network.guest_mac 5536 ); 5537 5538 if !hotplug { 5539 guest_command.args(&["--net", &net_params]); 5540 } 5541 5542 let mut child = guest_command.capture_output().spawn().unwrap(); 5543 5544 if hotplug { 5545 // Give some time to the VMM process to listen to the API 5546 // socket. This is the only requirement to avoid the following 5547 // call to ch-remote from failing. 5548 thread::sleep(std::time::Duration::new(10, 0)); 5549 // Hotplug the virtio-net device 5550 let (cmd_success, cmd_output) = 5551 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 5552 assert!(cmd_success); 5553 #[cfg(target_arch = "x86_64")] 5554 assert!(String::from_utf8_lossy(&cmd_output) 5555 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 5556 #[cfg(target_arch = "aarch64")] 5557 assert!(String::from_utf8_lossy(&cmd_output) 5558 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 5559 } 5560 5561 // The functional connectivity provided by the virtio-net device 5562 // gets tested through wait_vm_boot() as it expects to receive a 5563 // HTTP request, and through the SSH command as well. 5564 let r = std::panic::catch_unwind(|| { 5565 guest.wait_vm_boot(None).unwrap(); 5566 5567 assert_eq!( 5568 guest 5569 .ssh_command("ip -o link | wc -l") 5570 .unwrap() 5571 .trim() 5572 .parse::<u32>() 5573 .unwrap_or_default(), 5574 2 5575 ); 5576 }); 5577 5578 let _ = child.kill(); 5579 5580 exec_host_command_status(&format!("sudo ip link del {}", guest_macvtap_name)); 5581 exec_host_command_status(&format!("sudo ip link del {}", host_macvtap_name)); 5582 5583 let output = child.wait_with_output().unwrap(); 5584 5585 handle_child_output(r, &output); 5586 } 5587 5588 #[test] 5589 fn test_macvtap() { 5590 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 5591 } 5592 5593 #[test] 5594 fn test_macvtap_hotplug() { 5595 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 5596 } 5597 5598 #[test] 5599 #[cfg(not(feature = "mshv"))] 5600 fn test_ovs_dpdk() { 5601 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5602 let guest1 = Guest::new(Box::new(focal1)); 5603 5604 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5605 let guest2 = Guest::new(Box::new(focal2)); 5606 let api_socket = temp_api_path(&guest2.tmp_dir); 5607 5608 let (mut child1, mut child2) = setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket); 5609 5610 // Create the snapshot directory 5611 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 5612 5613 let r = std::panic::catch_unwind(|| { 5614 // Remove one of the two ports from the OVS bridge 5615 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 5616 5617 // Spawn a new netcat listener in the first VM 5618 let guest_ip = guest1.network.guest_ip.clone(); 5619 thread::spawn(move || { 5620 ssh_command_ip( 5621 "nc -l 12345", 5622 &guest_ip, 5623 DEFAULT_SSH_RETRIES, 5624 DEFAULT_SSH_TIMEOUT, 5625 ) 5626 .unwrap(); 5627 }); 5628 5629 // Wait for the server to be listening 5630 thread::sleep(std::time::Duration::new(5, 0)); 5631 5632 // Check the connection fails this time 5633 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 5634 5635 // Add the OVS port back 5636 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 5637 5638 // And finally check the connection is functional again 5639 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 5640 5641 // Pause the VM 5642 assert!(remote_command(&api_socket, "pause", None)); 5643 5644 // Take a snapshot from the VM 5645 assert!(remote_command( 5646 &api_socket, 5647 "snapshot", 5648 Some(format!("file://{}", snapshot_dir).as_str()), 5649 )); 5650 5651 // Wait to make sure the snapshot is completed 5652 thread::sleep(std::time::Duration::new(10, 0)); 5653 }); 5654 5655 // Shutdown the source VM 5656 let _ = child2.kill(); 5657 let output = child2.wait_with_output().unwrap(); 5658 handle_child_output(r, &output); 5659 5660 // Remove the vhost-user socket file. 5661 Command::new("rm") 5662 .arg("-f") 5663 .arg("/tmp/dpdkvhostclient2") 5664 .output() 5665 .unwrap(); 5666 5667 // Restore the VM from the snapshot 5668 let mut child2 = GuestCommand::new(&guest2) 5669 .args(&["--api-socket", &api_socket]) 5670 .args(&[ 5671 "--restore", 5672 format!("source_url=file://{}", snapshot_dir).as_str(), 5673 ]) 5674 .capture_output() 5675 .spawn() 5676 .unwrap(); 5677 5678 // Wait for the VM to be restored 5679 thread::sleep(std::time::Duration::new(10, 0)); 5680 5681 let r = std::panic::catch_unwind(|| { 5682 // Resume the VM 5683 assert!(remote_command(&api_socket, "resume", None)); 5684 5685 // Spawn a new netcat listener in the first VM 5686 let guest_ip = guest1.network.guest_ip.clone(); 5687 thread::spawn(move || { 5688 ssh_command_ip( 5689 "nc -l 12345", 5690 &guest_ip, 5691 DEFAULT_SSH_RETRIES, 5692 DEFAULT_SSH_TIMEOUT, 5693 ) 5694 .unwrap(); 5695 }); 5696 5697 // Wait for the server to be listening 5698 thread::sleep(std::time::Duration::new(5, 0)); 5699 5700 // And check the connection is still functional after restore 5701 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 5702 }); 5703 5704 cleanup_ovs_dpdk(); 5705 5706 let _ = child1.kill(); 5707 let _ = child2.kill(); 5708 5709 let output = child1.wait_with_output().unwrap(); 5710 child2.wait().unwrap(); 5711 5712 handle_child_output(r, &output); 5713 } 5714 5715 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 5716 cleanup_spdk_nvme(); 5717 5718 assert!(exec_host_command_status(&format!( 5719 "mkdir -p {}", 5720 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 5721 )) 5722 .success()); 5723 assert!(exec_host_command_status(&format!( 5724 "truncate {} -s 128M", 5725 nvme_dir.join("test-disk.raw").to_str().unwrap() 5726 )) 5727 .success()); 5728 assert!(exec_host_command_status(&format!( 5729 "mkfs.ext4 {}", 5730 nvme_dir.join("test-disk.raw").to_str().unwrap() 5731 )) 5732 .success()); 5733 5734 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 5735 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 5736 .args(&["-i", "0", "-m", "0x1"]) 5737 .spawn() 5738 .unwrap(); 5739 thread::sleep(std::time::Duration::new(2, 0)); 5740 5741 assert!(exec_host_command_status( 5742 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER" 5743 ) 5744 .success()); 5745 assert!(exec_host_command_status(&format!( 5746 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 5747 nvme_dir.join("test-disk.raw").to_str().unwrap() 5748 )) 5749 .success()); 5750 assert!(exec_host_command_status( 5751 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 5752 ) 5753 .success()); 5754 assert!(exec_host_command_status( 5755 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 5756 ) 5757 .success()); 5758 assert!(exec_host_command_status(&format!( 5759 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 5760 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 5761 )) 5762 .success()); 5763 } 5764 5765 fn cleanup_spdk_nvme() { 5766 exec_host_command_status("pkill -f nvmf_tgt"); 5767 } 5768 5769 #[test] 5770 fn test_vfio_user() { 5771 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 5772 let jammy = UbuntuDiskConfig::new(jammy_image); 5773 let guest = Guest::new(Box::new(jammy)); 5774 5775 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 5776 setup_spdk_nvme(spdk_nvme_dir.as_path()); 5777 5778 let api_socket = temp_api_path(&guest.tmp_dir); 5779 let mut child = GuestCommand::new(&guest) 5780 .args(&["--api-socket", &api_socket]) 5781 .args(&["--cpus", "boot=1"]) 5782 .args(&["--memory", "size=512M,shared=on,hugepages=on"]) 5783 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 5784 .args(&["--serial", "tty", "--console", "off"]) 5785 .default_disks() 5786 .default_net() 5787 .capture_output() 5788 .spawn() 5789 .unwrap(); 5790 5791 let r = std::panic::catch_unwind(|| { 5792 guest.wait_vm_boot(None).unwrap(); 5793 5794 // Hotplug the SPDK-NVMe device to the VM 5795 let (cmd_success, cmd_output) = remote_command_w_output( 5796 &api_socket, 5797 "add-user-device", 5798 Some(&format!( 5799 "socket={},id=vfio_user0", 5800 spdk_nvme_dir 5801 .as_path() 5802 .join("nvme-vfio-user/cntrl") 5803 .to_str() 5804 .unwrap(), 5805 )), 5806 ); 5807 assert!(cmd_success); 5808 assert!(String::from_utf8_lossy(&cmd_output) 5809 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 5810 5811 thread::sleep(std::time::Duration::new(10, 0)); 5812 5813 // Check both if /dev/nvme exists and if the block size is 128M. 5814 assert_eq!( 5815 guest 5816 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 5817 .unwrap() 5818 .trim() 5819 .parse::<u32>() 5820 .unwrap_or_default(), 5821 1 5822 ); 5823 5824 // Check changes persist after reboot 5825 assert_eq!( 5826 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 5827 "" 5828 ); 5829 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 5830 guest 5831 .ssh_command("echo test123 | sudo tee /mnt/test") 5832 .unwrap(); 5833 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 5834 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 5835 5836 guest.reboot_linux(0, None); 5837 assert_eq!( 5838 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 5839 "" 5840 ); 5841 assert_eq!( 5842 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 5843 "test123" 5844 ); 5845 }); 5846 5847 cleanup_spdk_nvme(); 5848 5849 let _ = child.kill(); 5850 let output = child.wait_with_output().unwrap(); 5851 5852 handle_child_output(r, &output); 5853 } 5854 5855 #[test] 5856 #[cfg(target_arch = "x86_64")] 5857 fn test_vdpa_block() { 5858 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 5859 if !exec_host_command_status("lsmod | grep vdpa_sim_blk").success() { 5860 return; 5861 } 5862 5863 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5864 let guest = Guest::new(Box::new(focal)); 5865 let api_socket = temp_api_path(&guest.tmp_dir); 5866 5867 let kernel_path = direct_kernel_boot_path(); 5868 5869 let mut child = GuestCommand::new(&guest) 5870 .args(&["--cpus", "boot=2"]) 5871 .args(&["--memory", "size=512M,hugepages=on"]) 5872 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5873 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5874 .default_disks() 5875 .default_net() 5876 .args(&["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 5877 .args(&["--platform", "num_pci_segments=2,iommu_segments=1"]) 5878 .args(&["--api-socket", &api_socket]) 5879 .capture_output() 5880 .spawn() 5881 .unwrap(); 5882 5883 let r = std::panic::catch_unwind(|| { 5884 guest.wait_vm_boot(None).unwrap(); 5885 5886 // Check both if /dev/vdc exists and if the block size is 128M. 5887 assert_eq!( 5888 guest 5889 .ssh_command("lsblk | grep vdc | grep -c 128M") 5890 .unwrap() 5891 .trim() 5892 .parse::<u32>() 5893 .unwrap_or_default(), 5894 1 5895 ); 5896 5897 // Check the content of the block device after we wrote to it. 5898 // The vpda-sim-blk should let us read what we previously wrote. 5899 guest 5900 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 5901 .unwrap(); 5902 assert_eq!( 5903 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 5904 "foobar" 5905 ); 5906 5907 // Hotplug an extra vDPA block device behind the vIOMMU 5908 // Add a new vDPA device to the VM 5909 let (cmd_success, cmd_output) = remote_command_w_output( 5910 &api_socket, 5911 "add-vdpa", 5912 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 5913 ); 5914 assert!(cmd_success); 5915 assert!(String::from_utf8_lossy(&cmd_output) 5916 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 5917 5918 thread::sleep(std::time::Duration::new(10, 0)); 5919 5920 // Check IOMMU setup 5921 assert!(guest 5922 .does_device_vendor_pair_match("0x1057", "0x1af4") 5923 .unwrap_or_default()); 5924 assert_eq!( 5925 guest 5926 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 5927 .unwrap() 5928 .trim(), 5929 "0001:00:01.0" 5930 ); 5931 5932 // Check both if /dev/vdd exists and if the block size is 128M. 5933 assert_eq!( 5934 guest 5935 .ssh_command("lsblk | grep vdd | grep -c 128M") 5936 .unwrap() 5937 .trim() 5938 .parse::<u32>() 5939 .unwrap_or_default(), 5940 1 5941 ); 5942 5943 // Write some content to the block device we've just plugged. 5944 guest 5945 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 5946 .unwrap(); 5947 5948 // Check we can read the content back. 5949 assert_eq!( 5950 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 5951 "foobar" 5952 ); 5953 5954 // Unplug the device 5955 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 5956 assert!(cmd_success); 5957 thread::sleep(std::time::Duration::new(10, 0)); 5958 5959 // Check /dev/vdd doesn't exist anymore 5960 assert_eq!( 5961 guest 5962 .ssh_command("lsblk | grep -c vdd || true") 5963 .unwrap() 5964 .trim() 5965 .parse::<u32>() 5966 .unwrap_or(1), 5967 0 5968 ); 5969 }); 5970 5971 let _ = child.kill(); 5972 let output = child.wait_with_output().unwrap(); 5973 5974 handle_child_output(r, &output); 5975 } 5976 5977 #[test] 5978 #[cfg(target_arch = "x86_64")] 5979 fn test_vdpa_net() { 5980 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 5981 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 5982 return; 5983 } 5984 5985 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5986 let guest = Guest::new(Box::new(focal)); 5987 5988 let kernel_path = direct_kernel_boot_path(); 5989 5990 let mut child = GuestCommand::new(&guest) 5991 .args(&["--cpus", "boot=2"]) 5992 .args(&["--memory", "size=512M,hugepages=on"]) 5993 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5994 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5995 .default_disks() 5996 .default_net() 5997 .args(&["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 5998 .capture_output() 5999 .spawn() 6000 .unwrap(); 6001 6002 let r = std::panic::catch_unwind(|| { 6003 guest.wait_vm_boot(None).unwrap(); 6004 6005 // Check we can find network interface related to vDPA device 6006 assert_eq!( 6007 guest 6008 .ssh_command("ip -o link | grep -c ens6") 6009 .unwrap() 6010 .trim() 6011 .parse::<u32>() 6012 .unwrap_or(0), 6013 1 6014 ); 6015 6016 guest 6017 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 6018 .unwrap(); 6019 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 6020 6021 // Check there is no packet yet on both TX/RX of the network interface 6022 assert_eq!( 6023 guest 6024 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 6025 .unwrap() 6026 .trim() 6027 .parse::<u32>() 6028 .unwrap_or(0), 6029 2 6030 ); 6031 6032 // Send 6 packets with ping command 6033 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 6034 6035 // Check we can find 6 packets on both TX/RX of the network interface 6036 assert_eq!( 6037 guest 6038 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 6039 .unwrap() 6040 .trim() 6041 .parse::<u32>() 6042 .unwrap_or(0), 6043 2 6044 ); 6045 6046 // No need to check for hotplug as we already tested it through 6047 // test_vdpa_block() 6048 }); 6049 6050 let _ = child.kill(); 6051 let output = child.wait_with_output().unwrap(); 6052 6053 handle_child_output(r, &output); 6054 } 6055 } 6056 6057 mod sequential { 6058 use crate::*; 6059 6060 #[test] 6061 fn test_memory_mergeable_on() { 6062 test_memory_mergeable(true) 6063 } 6064 } 6065 6066 #[cfg(target_arch = "x86_64")] 6067 mod windows { 6068 use crate::*; 6069 6070 lazy_static! { 6071 static ref NEXT_DISK_ID: Mutex<u8> = Mutex::new(1); 6072 } 6073 6074 struct WindowsGuest { 6075 guest: Guest, 6076 auth: PasswordAuth, 6077 } 6078 6079 trait FsType { 6080 const FS_FAT: u8; 6081 const FS_NTFS: u8; 6082 } 6083 impl FsType for WindowsGuest { 6084 const FS_FAT: u8 = 0; 6085 const FS_NTFS: u8 = 1; 6086 } 6087 6088 impl WindowsGuest { 6089 fn new() -> Self { 6090 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 6091 let guest = Guest::new(Box::new(disk)); 6092 let auth = PasswordAuth { 6093 username: String::from("administrator"), 6094 password: String::from("Admin123"), 6095 }; 6096 6097 WindowsGuest { guest, auth } 6098 } 6099 6100 fn guest(&self) -> &Guest { 6101 &self.guest 6102 } 6103 6104 fn ssh_cmd(&self, cmd: &str) -> String { 6105 ssh_command_ip_with_auth( 6106 cmd, 6107 &self.auth, 6108 &self.guest.network.guest_ip, 6109 DEFAULT_SSH_RETRIES, 6110 DEFAULT_SSH_TIMEOUT, 6111 ) 6112 .unwrap() 6113 } 6114 6115 fn cpu_count(&self) -> u8 { 6116 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 6117 .trim() 6118 .parse::<u8>() 6119 .unwrap_or(0) 6120 } 6121 6122 fn ram_size(&self) -> usize { 6123 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 6124 .trim() 6125 .parse::<usize>() 6126 .unwrap_or(0) 6127 } 6128 6129 fn netdev_count(&self) -> u8 { 6130 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6131 .trim() 6132 .parse::<u8>() 6133 .unwrap_or(0) 6134 } 6135 6136 fn disk_count(&self) -> u8 { 6137 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6138 .trim() 6139 .parse::<u8>() 6140 .unwrap_or(0) 6141 } 6142 6143 fn reboot(&self) { 6144 let _ = self.ssh_cmd("shutdown /r /t 0"); 6145 } 6146 6147 fn shutdown(&self) { 6148 let _ = self.ssh_cmd("shutdown /s /t 0"); 6149 } 6150 6151 fn run_dnsmasq(&self) -> std::process::Child { 6152 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 6153 let dhcp_host = format!( 6154 "--dhcp-host={},{}", 6155 self.guest.network.guest_mac, self.guest.network.guest_ip 6156 ); 6157 let dhcp_range = format!( 6158 "--dhcp-range=eth,{},{}", 6159 self.guest.network.guest_ip, self.guest.network.guest_ip 6160 ); 6161 6162 Command::new("dnsmasq") 6163 .arg("--no-daemon") 6164 .arg("--log-queries") 6165 .arg(listen_address.as_str()) 6166 .arg("--except-interface=lo") 6167 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 6168 .arg("--conf-file=/dev/null") 6169 .arg(dhcp_host.as_str()) 6170 .arg(dhcp_range.as_str()) 6171 .spawn() 6172 .unwrap() 6173 } 6174 6175 // TODO Cleanup image file explicitly after test, if there's some space issues. 6176 fn disk_new(&self, fs: u8, sz: usize) -> String { 6177 let mut guard = NEXT_DISK_ID.lock().unwrap(); 6178 let id = *guard; 6179 *guard = id + 1; 6180 6181 let img = PathBuf::from(format!("/tmp/test-hotplug-{}.raw", id)); 6182 let _ = fs::remove_file(&img); 6183 6184 // Create an image file 6185 let out = Command::new("qemu-img") 6186 .args(&[ 6187 "create", 6188 "-f", 6189 "raw", 6190 img.to_str().unwrap(), 6191 format!("{}m", sz).as_str(), 6192 ]) 6193 .output() 6194 .expect("qemu-img command failed") 6195 .stdout; 6196 println!("{:?}", out); 6197 6198 // Associate image to a loop device 6199 let out = Command::new("losetup") 6200 .args(&["--show", "-f", img.to_str().unwrap()]) 6201 .output() 6202 .expect("failed to create loop device") 6203 .stdout; 6204 let _tmp = String::from_utf8_lossy(&out); 6205 let loop_dev = _tmp.trim(); 6206 println!("{:?}", out); 6207 6208 // Create a partition table 6209 // echo 'type=7' | sudo sfdisk "${LOOP}" 6210 let mut child = Command::new("sfdisk") 6211 .args(&[loop_dev]) 6212 .stdin(Stdio::piped()) 6213 .spawn() 6214 .unwrap(); 6215 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 6216 let _ = stdin 6217 .write_all("type=7".as_bytes()) 6218 .expect("failed to write stdin"); 6219 let out = child.wait_with_output().expect("sfdisk failed").stdout; 6220 println!("{:?}", out); 6221 6222 // Disengage the loop device 6223 let out = Command::new("losetup") 6224 .args(&["-d", loop_dev]) 6225 .output() 6226 .expect("loop device not found") 6227 .stdout; 6228 println!("{:?}", out); 6229 6230 // Re-associate loop device pointing to the partition only 6231 let out = Command::new("losetup") 6232 .args(&[ 6233 "--show", 6234 "--offset", 6235 (512 * 2048).to_string().as_str(), 6236 "-f", 6237 img.to_str().unwrap(), 6238 ]) 6239 .output() 6240 .expect("failed to create loop device") 6241 .stdout; 6242 let _tmp = String::from_utf8_lossy(&out); 6243 let loop_dev = _tmp.trim(); 6244 println!("{:?}", out); 6245 6246 // Create filesystem. 6247 let fs_cmd = match fs { 6248 WindowsGuest::FS_FAT => "mkfs.msdos", 6249 WindowsGuest::FS_NTFS => "mkfs.ntfs", 6250 _ => panic!("Unknown filesystem type '{}'", fs), 6251 }; 6252 let out = Command::new(fs_cmd) 6253 .args(&[&loop_dev]) 6254 .output() 6255 .unwrap_or_else(|_| panic!("{} failed", fs_cmd)) 6256 .stdout; 6257 println!("{:?}", out); 6258 6259 // Disengage the loop device 6260 let out = Command::new("losetup") 6261 .args(&["-d", loop_dev]) 6262 .output() 6263 .unwrap_or_else(|_| panic!("loop device '{}' not found", loop_dev)) 6264 .stdout; 6265 println!("{:?}", out); 6266 6267 img.to_str().unwrap().to_string() 6268 } 6269 6270 fn disks_set_rw(&self) { 6271 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 6272 } 6273 6274 fn disks_online(&self) { 6275 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 6276 } 6277 6278 fn disk_file_put(&self, fname: &str, data: &str) { 6279 let _ = self.ssh_cmd(&format!( 6280 "powershell -Command \"'{}' | Set-Content -Path {}\"", 6281 data, fname 6282 )); 6283 } 6284 6285 fn disk_file_read(&self, fname: &str) -> String { 6286 self.ssh_cmd(&format!( 6287 "powershell -Command \"Get-Content -Path {}\"", 6288 fname 6289 )) 6290 } 6291 6292 fn wait_for_boot(&self) -> bool { 6293 let cmd = "dir /b c:\\ | find \"Windows\""; 6294 let tmo_max = 180; 6295 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 6296 // interval must be small. 6297 let tmo_int = 2; 6298 let out = ssh_command_ip_with_auth( 6299 cmd, 6300 &self.auth, 6301 &self.guest.network.guest_ip, 6302 { 6303 let mut ret = 1; 6304 let mut tmo_acc = 0; 6305 loop { 6306 tmo_acc += tmo_int * ret; 6307 if tmo_acc >= tmo_max { 6308 break; 6309 } 6310 ret += 1; 6311 } 6312 ret 6313 }, 6314 tmo_int, 6315 ) 6316 .unwrap(); 6317 6318 if "Windows" == out.trim() { 6319 return true; 6320 } 6321 6322 false 6323 } 6324 } 6325 6326 fn vcpu_threads_count(pid: u32) -> u8 { 6327 // ps -T -p 12345 | grep vcpu | wc -l 6328 let out = Command::new("ps") 6329 .args(&["-T", "-p", format!("{}", pid).as_str()]) 6330 .output() 6331 .expect("ps command failed") 6332 .stdout; 6333 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 6334 } 6335 6336 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 6337 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 6338 let out = Command::new("ps") 6339 .args(&["-T", "-p", format!("{}", pid).as_str()]) 6340 .output() 6341 .expect("ps command failed") 6342 .stdout; 6343 let mut n = 0; 6344 String::from_utf8_lossy(&out) 6345 .split_whitespace() 6346 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 6347 n 6348 } 6349 6350 fn disk_ctrl_threads_count(pid: u32) -> u8 { 6351 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 6352 let out = Command::new("ps") 6353 .args(&["-T", "-p", format!("{}", pid).as_str()]) 6354 .output() 6355 .expect("ps command failed") 6356 .stdout; 6357 let mut n = 0; 6358 String::from_utf8_lossy(&out) 6359 .split_whitespace() 6360 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 6361 n 6362 } 6363 6364 #[test] 6365 fn test_windows_guest() { 6366 let windows_guest = WindowsGuest::new(); 6367 6368 let mut ovmf_path = dirs::home_dir().unwrap(); 6369 ovmf_path.push("workloads"); 6370 ovmf_path.push(OVMF_NAME); 6371 6372 let mut child = GuestCommand::new(windows_guest.guest()) 6373 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6374 .args(&["--memory", "size=4G"]) 6375 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6376 .args(&["--serial", "tty"]) 6377 .args(&["--console", "off"]) 6378 .default_disks() 6379 .default_net() 6380 .capture_output() 6381 .spawn() 6382 .unwrap(); 6383 6384 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 6385 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6386 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 6387 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6388 6389 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 6390 6391 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6392 6393 let r = std::panic::catch_unwind(|| { 6394 // Wait to make sure Windows boots up 6395 assert!(windows_guest.wait_for_boot()); 6396 6397 windows_guest.shutdown(); 6398 }); 6399 6400 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6401 let _ = child.kill(); 6402 let output = child.wait_with_output().unwrap(); 6403 6404 let _ = child_dnsmasq.kill(); 6405 let _ = child_dnsmasq.wait(); 6406 6407 handle_child_output(r, &output); 6408 } 6409 6410 #[test] 6411 fn test_windows_guest_multiple_queues() { 6412 let windows_guest = WindowsGuest::new(); 6413 6414 let mut ovmf_path = dirs::home_dir().unwrap(); 6415 ovmf_path.push("workloads"); 6416 ovmf_path.push(OVMF_NAME); 6417 6418 let mut child = GuestCommand::new(windows_guest.guest()) 6419 .args(&["--cpus", "boot=4,kvm_hyperv=on"]) 6420 .args(&["--memory", "size=4G"]) 6421 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6422 .args(&["--serial", "tty"]) 6423 .args(&["--console", "off"]) 6424 .args(&[ 6425 "--disk", 6426 format!( 6427 "path={},num_queues=4", 6428 windows_guest 6429 .guest() 6430 .disk_config 6431 .disk(DiskType::OperatingSystem) 6432 .unwrap() 6433 ) 6434 .as_str(), 6435 ]) 6436 .args(&[ 6437 "--net", 6438 format!( 6439 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 6440 windows_guest.guest().network.guest_mac, 6441 windows_guest.guest().network.host_ip 6442 ) 6443 .as_str(), 6444 ]) 6445 .capture_output() 6446 .spawn() 6447 .unwrap(); 6448 6449 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 6450 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6451 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 6452 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6453 6454 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 6455 6456 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6457 6458 let r = std::panic::catch_unwind(|| { 6459 // Wait to make sure Windows boots up 6460 assert!(windows_guest.wait_for_boot()); 6461 6462 windows_guest.shutdown(); 6463 }); 6464 6465 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6466 let _ = child.kill(); 6467 let output = child.wait_with_output().unwrap(); 6468 6469 let _ = child_dnsmasq.kill(); 6470 let _ = child_dnsmasq.wait(); 6471 6472 handle_child_output(r, &output); 6473 } 6474 6475 #[test] 6476 #[cfg(not(feature = "mshv"))] 6477 fn test_windows_guest_snapshot_restore() { 6478 let windows_guest = WindowsGuest::new(); 6479 6480 let mut ovmf_path = dirs::home_dir().unwrap(); 6481 ovmf_path.push("workloads"); 6482 ovmf_path.push(OVMF_NAME); 6483 6484 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6485 let api_socket = temp_api_path(&tmp_dir); 6486 6487 let mut child = GuestCommand::new(windows_guest.guest()) 6488 .args(&["--api-socket", &api_socket]) 6489 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6490 .args(&["--memory", "size=4G"]) 6491 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6492 .args(&["--serial", "tty"]) 6493 .args(&["--console", "off"]) 6494 .default_disks() 6495 .default_net() 6496 .capture_output() 6497 .spawn() 6498 .unwrap(); 6499 6500 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 6501 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6502 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 6503 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6504 6505 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 6506 6507 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6508 6509 // Wait to make sure Windows boots up 6510 assert!(windows_guest.wait_for_boot()); 6511 6512 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 6513 6514 // Pause the VM 6515 assert!(remote_command(&api_socket, "pause", None)); 6516 6517 // Take a snapshot from the VM 6518 assert!(remote_command( 6519 &api_socket, 6520 "snapshot", 6521 Some(format!("file://{}", snapshot_dir).as_str()), 6522 )); 6523 6524 // Wait to make sure the snapshot is completed 6525 thread::sleep(std::time::Duration::new(30, 0)); 6526 6527 let _ = child.kill(); 6528 child.wait().unwrap(); 6529 6530 // Restore the VM from the snapshot 6531 let mut child = GuestCommand::new(windows_guest.guest()) 6532 .args(&["--api-socket", &api_socket]) 6533 .args(&[ 6534 "--restore", 6535 format!("source_url=file://{}", snapshot_dir).as_str(), 6536 ]) 6537 .capture_output() 6538 .spawn() 6539 .unwrap(); 6540 6541 // Wait for the VM to be restored 6542 thread::sleep(std::time::Duration::new(20, 0)); 6543 6544 let r = std::panic::catch_unwind(|| { 6545 // Resume the VM 6546 assert!(remote_command(&api_socket, "resume", None)); 6547 6548 windows_guest.shutdown(); 6549 }); 6550 6551 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6552 let _ = child.kill(); 6553 let output = child.wait_with_output().unwrap(); 6554 6555 let _ = child_dnsmasq.kill(); 6556 let _ = child_dnsmasq.wait(); 6557 6558 handle_child_output(r, &output); 6559 } 6560 6561 #[test] 6562 #[cfg(not(feature = "mshv"))] 6563 fn test_windows_guest_cpu_hotplug() { 6564 let windows_guest = WindowsGuest::new(); 6565 6566 let mut ovmf_path = dirs::home_dir().unwrap(); 6567 ovmf_path.push("workloads"); 6568 ovmf_path.push(OVMF_NAME); 6569 6570 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6571 let api_socket = temp_api_path(&tmp_dir); 6572 6573 let mut child = GuestCommand::new(windows_guest.guest()) 6574 .args(&["--api-socket", &api_socket]) 6575 .args(&["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 6576 .args(&["--memory", "size=4G"]) 6577 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6578 .args(&["--serial", "tty"]) 6579 .args(&["--console", "off"]) 6580 .default_disks() 6581 .default_net() 6582 .capture_output() 6583 .spawn() 6584 .unwrap(); 6585 6586 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6587 6588 let r = std::panic::catch_unwind(|| { 6589 // Wait to make sure Windows boots up 6590 assert!(windows_guest.wait_for_boot()); 6591 6592 let vcpu_num = 2; 6593 // Check the initial number of CPUs the guest sees 6594 assert_eq!(windows_guest.cpu_count(), vcpu_num); 6595 // Check the initial number of vcpu threads in the CH process 6596 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 6597 6598 let vcpu_num = 6; 6599 // Hotplug some CPUs 6600 resize_command(&api_socket, Some(vcpu_num), None, None); 6601 // Wait to make sure CPUs are added 6602 thread::sleep(std::time::Duration::new(10, 0)); 6603 // Check the guest sees the correct number 6604 assert_eq!(windows_guest.cpu_count(), vcpu_num); 6605 // Check the CH process has the correct number of vcpu threads 6606 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 6607 6608 let vcpu_num = 4; 6609 // Remove some CPUs. Note that Windows doesn't support hot-remove. 6610 resize_command(&api_socket, Some(vcpu_num), None, None); 6611 // Wait to make sure CPUs are removed 6612 thread::sleep(std::time::Duration::new(10, 0)); 6613 // Reboot to let Windows catch up 6614 windows_guest.reboot(); 6615 // Wait to make sure Windows completely rebooted 6616 thread::sleep(std::time::Duration::new(60, 0)); 6617 // Check the guest sees the correct number 6618 assert_eq!(windows_guest.cpu_count(), vcpu_num); 6619 // Check the CH process has the correct number of vcpu threads 6620 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 6621 6622 windows_guest.shutdown(); 6623 }); 6624 6625 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6626 let _ = child.kill(); 6627 let output = child.wait_with_output().unwrap(); 6628 6629 let _ = child_dnsmasq.kill(); 6630 let _ = child_dnsmasq.wait(); 6631 6632 handle_child_output(r, &output); 6633 } 6634 6635 #[test] 6636 #[cfg(not(feature = "mshv"))] 6637 fn test_windows_guest_ram_hotplug() { 6638 let windows_guest = WindowsGuest::new(); 6639 6640 let mut ovmf_path = dirs::home_dir().unwrap(); 6641 ovmf_path.push("workloads"); 6642 ovmf_path.push(OVMF_NAME); 6643 6644 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6645 let api_socket = temp_api_path(&tmp_dir); 6646 6647 let mut child = GuestCommand::new(windows_guest.guest()) 6648 .args(&["--api-socket", &api_socket]) 6649 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6650 .args(&["--memory", "size=2G,hotplug_size=5G"]) 6651 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6652 .args(&["--serial", "tty"]) 6653 .args(&["--console", "off"]) 6654 .default_disks() 6655 .default_net() 6656 .capture_output() 6657 .spawn() 6658 .unwrap(); 6659 6660 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6661 6662 let r = std::panic::catch_unwind(|| { 6663 // Wait to make sure Windows boots up 6664 assert!(windows_guest.wait_for_boot()); 6665 6666 let ram_size = 2 * 1024 * 1024 * 1024; 6667 // Check the initial number of RAM the guest sees 6668 let current_ram_size = windows_guest.ram_size(); 6669 // This size seems to be reserved by the system and thus the 6670 // reported amount differs by this constant value. 6671 let reserved_ram_size = ram_size - current_ram_size; 6672 // Verify that there's not more than 4mb constant diff wasted 6673 // by the reserved ram. 6674 assert!(reserved_ram_size < 4 * 1024 * 1024); 6675 6676 let ram_size = 4 * 1024 * 1024 * 1024; 6677 // Hotplug some RAM 6678 resize_command(&api_socket, None, Some(ram_size), None); 6679 // Wait to make sure RAM has been added 6680 thread::sleep(std::time::Duration::new(10, 0)); 6681 // Check the guest sees the correct number 6682 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 6683 6684 let ram_size = 3 * 1024 * 1024 * 1024; 6685 // Unplug some RAM. Note that hot-remove most likely won't work. 6686 resize_command(&api_socket, None, Some(ram_size), None); 6687 // Wait to make sure RAM has been added 6688 thread::sleep(std::time::Duration::new(10, 0)); 6689 // Reboot to let Windows catch up 6690 windows_guest.reboot(); 6691 // Wait to make sure guest completely rebooted 6692 thread::sleep(std::time::Duration::new(60, 0)); 6693 // Check the guest sees the correct number 6694 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 6695 6696 windows_guest.shutdown(); 6697 }); 6698 6699 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6700 let _ = child.kill(); 6701 let output = child.wait_with_output().unwrap(); 6702 6703 let _ = child_dnsmasq.kill(); 6704 let _ = child_dnsmasq.wait(); 6705 6706 handle_child_output(r, &output); 6707 } 6708 6709 #[test] 6710 #[cfg(not(feature = "mshv"))] 6711 fn test_windows_guest_netdev_hotplug() { 6712 let windows_guest = WindowsGuest::new(); 6713 6714 let mut ovmf_path = dirs::home_dir().unwrap(); 6715 ovmf_path.push("workloads"); 6716 ovmf_path.push(OVMF_NAME); 6717 6718 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6719 let api_socket = temp_api_path(&tmp_dir); 6720 6721 let mut child = GuestCommand::new(windows_guest.guest()) 6722 .args(&["--api-socket", &api_socket]) 6723 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6724 .args(&["--memory", "size=4G"]) 6725 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6726 .args(&["--serial", "tty"]) 6727 .args(&["--console", "off"]) 6728 .default_disks() 6729 .default_net() 6730 .capture_output() 6731 .spawn() 6732 .unwrap(); 6733 6734 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6735 6736 let r = std::panic::catch_unwind(|| { 6737 // Wait to make sure Windows boots up 6738 assert!(windows_guest.wait_for_boot()); 6739 6740 // Initially present network device 6741 let netdev_num = 1; 6742 assert_eq!(windows_guest.netdev_count(), netdev_num); 6743 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6744 6745 // Hotplug network device 6746 let (cmd_success, cmd_output) = remote_command_w_output( 6747 &api_socket, 6748 "add-net", 6749 Some(windows_guest.guest().default_net_string().as_str()), 6750 ); 6751 assert!(cmd_success); 6752 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 6753 thread::sleep(std::time::Duration::new(5, 0)); 6754 // Verify the device is on the system 6755 let netdev_num = 2; 6756 assert_eq!(windows_guest.netdev_count(), netdev_num); 6757 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6758 6759 // Remove network device 6760 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 6761 assert!(cmd_success); 6762 thread::sleep(std::time::Duration::new(5, 0)); 6763 // Verify the device has been removed 6764 let netdev_num = 1; 6765 assert_eq!(windows_guest.netdev_count(), netdev_num); 6766 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6767 6768 windows_guest.shutdown(); 6769 }); 6770 6771 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6772 let _ = child.kill(); 6773 let output = child.wait_with_output().unwrap(); 6774 6775 let _ = child_dnsmasq.kill(); 6776 let _ = child_dnsmasq.wait(); 6777 6778 handle_child_output(r, &output); 6779 } 6780 6781 #[test] 6782 #[cfg(not(feature = "mshv"))] 6783 fn test_windows_guest_disk_hotplug() { 6784 let windows_guest = WindowsGuest::new(); 6785 6786 let mut ovmf_path = dirs::home_dir().unwrap(); 6787 ovmf_path.push("workloads"); 6788 ovmf_path.push(OVMF_NAME); 6789 6790 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6791 let api_socket = temp_api_path(&tmp_dir); 6792 6793 let mut child = GuestCommand::new(windows_guest.guest()) 6794 .args(&["--api-socket", &api_socket]) 6795 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6796 .args(&["--memory", "size=4G"]) 6797 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6798 .args(&["--serial", "tty"]) 6799 .args(&["--console", "off"]) 6800 .default_disks() 6801 .default_net() 6802 .capture_output() 6803 .spawn() 6804 .unwrap(); 6805 6806 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6807 6808 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 6809 6810 let r = std::panic::catch_unwind(|| { 6811 // Wait to make sure Windows boots up 6812 assert!(windows_guest.wait_for_boot()); 6813 6814 // Initially present disk device 6815 let disk_num = 1; 6816 assert_eq!(windows_guest.disk_count(), disk_num); 6817 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6818 6819 // Hotplug disk device 6820 let (cmd_success, cmd_output) = remote_command_w_output( 6821 &api_socket, 6822 "add-disk", 6823 Some(format!("path={},readonly=off", disk).as_str()), 6824 ); 6825 assert!(cmd_success); 6826 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 6827 thread::sleep(std::time::Duration::new(5, 0)); 6828 // Online disk device 6829 windows_guest.disks_set_rw(); 6830 windows_guest.disks_online(); 6831 // Verify the device is on the system 6832 let disk_num = 2; 6833 assert_eq!(windows_guest.disk_count(), disk_num); 6834 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6835 6836 let data = "hello"; 6837 let fname = "d:\\world"; 6838 windows_guest.disk_file_put(fname, data); 6839 6840 // Unmount disk device 6841 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 6842 assert!(cmd_success); 6843 thread::sleep(std::time::Duration::new(5, 0)); 6844 // Verify the device has been removed 6845 let disk_num = 1; 6846 assert_eq!(windows_guest.disk_count(), disk_num); 6847 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6848 6849 // Remount and check the file exists with the expected contents 6850 let (cmd_success, _cmd_output) = remote_command_w_output( 6851 &api_socket, 6852 "add-disk", 6853 Some(format!("path={},readonly=off", disk).as_str()), 6854 ); 6855 assert!(cmd_success); 6856 thread::sleep(std::time::Duration::new(5, 0)); 6857 let out = windows_guest.disk_file_read(fname); 6858 assert_eq!(data, out.trim()); 6859 6860 // Intentionally no unmount, it'll happen at shutdown. 6861 6862 windows_guest.shutdown(); 6863 }); 6864 6865 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6866 let _ = child.kill(); 6867 let output = child.wait_with_output().unwrap(); 6868 6869 let _ = child_dnsmasq.kill(); 6870 let _ = child_dnsmasq.wait(); 6871 6872 handle_child_output(r, &output); 6873 } 6874 6875 #[test] 6876 #[cfg(not(feature = "mshv"))] 6877 fn test_windows_guest_disk_hotplug_multi() { 6878 let windows_guest = WindowsGuest::new(); 6879 6880 let mut ovmf_path = dirs::home_dir().unwrap(); 6881 ovmf_path.push("workloads"); 6882 ovmf_path.push(OVMF_NAME); 6883 6884 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6885 let api_socket = temp_api_path(&tmp_dir); 6886 6887 let mut child = GuestCommand::new(windows_guest.guest()) 6888 .args(&["--api-socket", &api_socket]) 6889 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6890 .args(&["--memory", "size=2G"]) 6891 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6892 .args(&["--serial", "tty"]) 6893 .args(&["--console", "off"]) 6894 .default_disks() 6895 .default_net() 6896 .capture_output() 6897 .spawn() 6898 .unwrap(); 6899 6900 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6901 6902 // Predefined data to used at various test stages 6903 let disk_test_data: [[String; 4]; 2] = [ 6904 [ 6905 "_disk2".to_string(), 6906 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 6907 "d:\\world".to_string(), 6908 "hello".to_string(), 6909 ], 6910 [ 6911 "_disk3".to_string(), 6912 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 6913 "e:\\hello".to_string(), 6914 "world".to_string(), 6915 ], 6916 ]; 6917 6918 let r = std::panic::catch_unwind(|| { 6919 // Wait to make sure Windows boots up 6920 assert!(windows_guest.wait_for_boot()); 6921 6922 // Initially present disk device 6923 let disk_num = 1; 6924 assert_eq!(windows_guest.disk_count(), disk_num); 6925 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6926 6927 for it in &disk_test_data { 6928 let disk_id = it[0].as_str(); 6929 let disk = it[1].as_str(); 6930 // Hotplug disk device 6931 let (cmd_success, cmd_output) = remote_command_w_output( 6932 &api_socket, 6933 "add-disk", 6934 Some(format!("path={},readonly=off", disk).as_str()), 6935 ); 6936 assert!(cmd_success); 6937 assert!(String::from_utf8_lossy(&cmd_output) 6938 .contains(format!("\"id\":\"{}\"", disk_id).as_str())); 6939 thread::sleep(std::time::Duration::new(5, 0)); 6940 // Online disk devices 6941 windows_guest.disks_set_rw(); 6942 windows_guest.disks_online(); 6943 } 6944 // Verify the devices are on the system 6945 let disk_num = (disk_test_data.len() + 1) as u8; 6946 assert_eq!(windows_guest.disk_count(), disk_num); 6947 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6948 6949 // Put test data 6950 for it in &disk_test_data { 6951 let fname = it[2].as_str(); 6952 let data = it[3].as_str(); 6953 windows_guest.disk_file_put(fname, data); 6954 } 6955 6956 // Unmount disk devices 6957 for it in &disk_test_data { 6958 let disk_id = it[0].as_str(); 6959 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 6960 assert!(cmd_success); 6961 thread::sleep(std::time::Duration::new(5, 0)); 6962 } 6963 6964 // Verify the devices have been removed 6965 let disk_num = 1; 6966 assert_eq!(windows_guest.disk_count(), disk_num); 6967 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6968 6969 // Remount 6970 for it in &disk_test_data { 6971 let disk = it[1].as_str(); 6972 let (cmd_success, _cmd_output) = remote_command_w_output( 6973 &api_socket, 6974 "add-disk", 6975 Some(format!("path={},readonly=off", disk).as_str()), 6976 ); 6977 assert!(cmd_success); 6978 thread::sleep(std::time::Duration::new(5, 0)); 6979 } 6980 6981 // Check the files exists with the expected contents 6982 for it in &disk_test_data { 6983 let fname = it[2].as_str(); 6984 let data = it[3].as_str(); 6985 let out = windows_guest.disk_file_read(fname); 6986 assert_eq!(data, out.trim()); 6987 } 6988 6989 // Intentionally no unmount, it'll happen at shutdown. 6990 6991 windows_guest.shutdown(); 6992 }); 6993 6994 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6995 let _ = child.kill(); 6996 let output = child.wait_with_output().unwrap(); 6997 6998 let _ = child_dnsmasq.kill(); 6999 let _ = child_dnsmasq.wait(); 7000 7001 handle_child_output(r, &output); 7002 } 7003 7004 #[test] 7005 #[cfg(not(feature = "mshv"))] 7006 fn test_windows_guest_netdev_multi() { 7007 let windows_guest = WindowsGuest::new(); 7008 7009 let mut ovmf_path = dirs::home_dir().unwrap(); 7010 ovmf_path.push("workloads"); 7011 ovmf_path.push(OVMF_NAME); 7012 7013 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7014 let api_socket = temp_api_path(&tmp_dir); 7015 7016 let mut child = GuestCommand::new(windows_guest.guest()) 7017 .args(&["--api-socket", &api_socket]) 7018 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7019 .args(&["--memory", "size=4G"]) 7020 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7021 .args(&["--serial", "tty"]) 7022 .args(&["--console", "off"]) 7023 .default_disks() 7024 // The multi net dev config is borrowed from test_multiple_network_interfaces 7025 .args(&[ 7026 "--net", 7027 windows_guest.guest().default_net_string().as_str(), 7028 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 7029 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 7030 ]) 7031 .capture_output() 7032 .spawn() 7033 .unwrap(); 7034 7035 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7036 7037 let r = std::panic::catch_unwind(|| { 7038 // Wait to make sure Windows boots up 7039 assert!(windows_guest.wait_for_boot()); 7040 7041 let netdev_num = 3; 7042 assert_eq!(windows_guest.netdev_count(), netdev_num); 7043 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7044 7045 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 7046 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 7047 7048 windows_guest.shutdown(); 7049 }); 7050 7051 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7052 let _ = child.kill(); 7053 let output = child.wait_with_output().unwrap(); 7054 7055 let _ = child_dnsmasq.kill(); 7056 let _ = child_dnsmasq.wait(); 7057 7058 handle_child_output(r, &output); 7059 } 7060 } 7061 7062 #[cfg(target_arch = "x86_64")] 7063 mod sgx { 7064 use crate::*; 7065 7066 #[test] 7067 fn test_sgx() { 7068 let focal = UbuntuDiskConfig::new(FOCAL_SGX_IMAGE_NAME.to_string()); 7069 let guest = Guest::new(Box::new(focal)); 7070 let mut workload_path = dirs::home_dir().unwrap(); 7071 workload_path.push("workloads"); 7072 7073 let mut kernel_path = workload_path; 7074 kernel_path.push("vmlinux_w_sgx"); 7075 7076 let mut child = GuestCommand::new(&guest) 7077 .args(&["--cpus", "boot=1"]) 7078 .args(&["--memory", "size=512M"]) 7079 .args(&["--kernel", kernel_path.to_str().unwrap()]) 7080 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7081 .default_disks() 7082 .default_net() 7083 .args(&["--sgx-epc", "id=epc0,size=64M"]) 7084 .capture_output() 7085 .spawn() 7086 .unwrap(); 7087 7088 let r = std::panic::catch_unwind(|| { 7089 guest.wait_vm_boot(None).unwrap(); 7090 7091 // Check if SGX is correctly detected in the guest. 7092 guest.check_sgx_support().unwrap(); 7093 7094 // Validate the SGX EPC section is 64MiB. 7095 assert_eq!( 7096 guest 7097 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 7098 .unwrap() 7099 .trim(), 7100 "0x0000000004000000" 7101 ); 7102 7103 // Run a test relying on SGX enclaves and check if it runs 7104 // successfully. 7105 assert!(guest 7106 .ssh_command("cd /linux-sgx/SampleCode/LocalAttestation/bin/ && sudo ./app") 7107 .unwrap() 7108 .trim() 7109 .contains( 7110 "succeed to load enclaves.\nsucceed to \ 7111 establish secure channel.\nSucceed to exchange \ 7112 secure message...\nSucceed to close Session..." 7113 )); 7114 }); 7115 7116 let _ = child.kill(); 7117 let output = child.wait_with_output().unwrap(); 7118 7119 handle_child_output(r, &output); 7120 } 7121 } 7122 7123 #[cfg(target_arch = "x86_64")] 7124 mod vfio { 7125 use crate::*; 7126 7127 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 7128 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 7129 let guest = Guest::new(Box::new(hirsute)); 7130 let api_socket = temp_api_path(&guest.tmp_dir); 7131 7132 let mut child = GuestCommand::new(&guest) 7133 .args(&["--cpus", "boot=4"]) 7134 .args(&[ 7135 "--memory", 7136 format!("size=4G,hotplug_size=4G,hotplug_method={}", hotplug_method).as_str(), 7137 ]) 7138 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7139 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 7140 .args(&["--api-socket", &api_socket]) 7141 .default_disks() 7142 .default_net() 7143 .capture_output() 7144 .spawn() 7145 .unwrap(); 7146 7147 let r = std::panic::catch_unwind(|| { 7148 guest.wait_vm_boot(None).unwrap(); 7149 7150 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 7151 7152 guest.enable_memory_hotplug(); 7153 7154 // Add RAM to the VM 7155 let desired_ram = 6 << 30; 7156 resize_command(&api_socket, None, Some(desired_ram), None); 7157 thread::sleep(std::time::Duration::new(30, 0)); 7158 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7159 7160 // Check the VFIO device works when RAM is increased to 6GiB 7161 guest.check_nvidia_gpu(); 7162 }); 7163 7164 let _ = child.kill(); 7165 let output = child.wait_with_output().unwrap(); 7166 7167 handle_child_output(r, &output); 7168 } 7169 7170 #[test] 7171 fn test_nvidia_card_memory_hotplug_acpi() { 7172 test_nvidia_card_memory_hotplug("acpi") 7173 } 7174 7175 #[test] 7176 fn test_nvidia_card_memory_hotplug_virtio_mem() { 7177 test_nvidia_card_memory_hotplug("virtio-mem") 7178 } 7179 7180 #[test] 7181 fn test_nvidia_card_pci_hotplug() { 7182 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 7183 let guest = Guest::new(Box::new(hirsute)); 7184 let api_socket = temp_api_path(&guest.tmp_dir); 7185 7186 let mut child = GuestCommand::new(&guest) 7187 .args(&["--cpus", "boot=4"]) 7188 .args(&["--memory", "size=4G"]) 7189 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7190 .args(&["--api-socket", &api_socket]) 7191 .default_disks() 7192 .default_net() 7193 .capture_output() 7194 .spawn() 7195 .unwrap(); 7196 7197 let r = std::panic::catch_unwind(|| { 7198 guest.wait_vm_boot(None).unwrap(); 7199 7200 // Hotplug the card to the VM 7201 let (cmd_success, cmd_output) = remote_command_w_output( 7202 &api_socket, 7203 "add-device", 7204 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 7205 ); 7206 assert!(cmd_success); 7207 assert!(String::from_utf8_lossy(&cmd_output) 7208 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 7209 7210 thread::sleep(std::time::Duration::new(10, 0)); 7211 7212 // Check the VFIO device works after hotplug 7213 guest.check_nvidia_gpu(); 7214 }); 7215 7216 let _ = child.kill(); 7217 let output = child.wait_with_output().unwrap(); 7218 7219 handle_child_output(r, &output); 7220 } 7221 7222 #[test] 7223 fn test_nvidia_card_reboot() { 7224 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 7225 let guest = Guest::new(Box::new(hirsute)); 7226 let api_socket = temp_api_path(&guest.tmp_dir); 7227 7228 let mut child = GuestCommand::new(&guest) 7229 .args(&["--cpus", "boot=4"]) 7230 .args(&["--memory", "size=4G"]) 7231 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7232 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 7233 .args(&["--api-socket", &api_socket]) 7234 .default_disks() 7235 .default_net() 7236 .capture_output() 7237 .spawn() 7238 .unwrap(); 7239 7240 let r = std::panic::catch_unwind(|| { 7241 guest.wait_vm_boot(None).unwrap(); 7242 7243 // Check the VFIO device works after boot 7244 guest.check_nvidia_gpu(); 7245 7246 guest.reboot_linux(0, None); 7247 7248 // Check the VFIO device works after reboot 7249 guest.check_nvidia_gpu(); 7250 }); 7251 7252 let _ = child.kill(); 7253 let output = child.wait_with_output().unwrap(); 7254 7255 handle_child_output(r, &output); 7256 } 7257 } 7258 7259 mod live_migration { 7260 use crate::*; 7261 7262 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 7263 // same host. It ensures the following behaviors: 7264 // 1. The source VM is up and functional (including various virtio-devices are working properly); 7265 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 7266 // 3. The source VM terminated gracefully after live migration; 7267 // 4. The destination VM is functional (including various virtio-devices are working properly) after 7268 // live migration; 7269 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 7270 fn _test_live_migration(numa: bool, local: bool) { 7271 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7272 let guest = Guest::new(Box::new(focal)); 7273 let kernel_path = direct_kernel_boot_path(); 7274 let console_text = String::from("On a branch floating down river a cricket, singing."); 7275 let net_id = "net123"; 7276 let net_params = format!( 7277 "id={},tap=,mac={},ip={},mask=255.255.255.0", 7278 net_id, guest.network.guest_mac, guest.network.host_ip 7279 ); 7280 7281 let memory_param: &[&str] = if numa { 7282 &[ 7283 "--memory", 7284 "size=0,hotplug_method=virtio-mem", 7285 "--memory-zone", 7286 "id=mem0,size=1G,hotplug_size=32G", 7287 "id=mem1,size=1G,hotplug_size=32G", 7288 "id=mem2,size=2G,hotplug_size=32G", 7289 "--numa", 7290 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 7291 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 7292 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 7293 ] 7294 } else if local { 7295 &["--memory", "size=4G,shared=on"] 7296 } else { 7297 &["--memory", "size=4G"] 7298 }; 7299 7300 // Start the source VM 7301 let src_api_socket = temp_api_path(&guest.tmp_dir); 7302 let mut src_child = GuestCommand::new(&guest) 7303 .args(&["--cpus", "boot=6,max=12"]) 7304 .args(memory_param) 7305 .args(&["--kernel", kernel_path.to_str().unwrap()]) 7306 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7307 .default_disks() 7308 .args(&["--net", net_params.as_str()]) 7309 .args(&["--api-socket", &src_api_socket]) 7310 .capture_output() 7311 .spawn() 7312 .unwrap(); 7313 7314 // Start the destination VM 7315 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 7316 dest_api_socket.push_str(".dest"); 7317 let mut dest_child = GuestCommand::new(&guest) 7318 .args(&["--api-socket", &dest_api_socket]) 7319 .capture_output() 7320 .spawn() 7321 .unwrap(); 7322 7323 let r = std::panic::catch_unwind(|| { 7324 guest.wait_vm_boot(Some(30)).unwrap(); 7325 7326 // Make sure the source VM is functaionl 7327 // Check the number of vCPUs 7328 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 6); 7329 // Check the guest RAM 7330 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 7331 // Check the guest virtio-devices, e.g. block, rng, console, and net 7332 guest.check_devices_common(None, Some(&console_text)); 7333 7334 // Check the NUMA parameters are applied correctly and resize 7335 // each zone to test the case where we migrate a VM with the 7336 // virtio-mem regions being used. 7337 if numa { 7338 guest.check_numa_common( 7339 Some(&[960_000, 960_000, 1_920_000]), 7340 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 7341 Some(&["10 15 20", "20 10 25", "25 30 10"]), 7342 ); 7343 7344 // AArch64 currently does not support hotplug, and therefore we only 7345 // test hotplug-related function on x86_64 here. 7346 #[cfg(target_arch = "x86_64")] 7347 { 7348 guest.enable_memory_hotplug(); 7349 7350 // Resize every memory zone and check each associated NUMA node 7351 // has been assigned the right amount of memory. 7352 resize_zone_command(&src_api_socket, "mem0", "2G"); 7353 resize_zone_command(&src_api_socket, "mem1", "2G"); 7354 resize_zone_command(&src_api_socket, "mem2", "3G"); 7355 thread::sleep(std::time::Duration::new(5, 0)); 7356 7357 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 7358 } 7359 } 7360 7361 // x86_64: Following what's done in the `test_snapshot_restore`, we need 7362 // to make sure that removing and adding back the virtio-net device does 7363 // not break the live-migration support for virtio-pci. 7364 #[cfg(target_arch = "x86_64")] 7365 { 7366 assert!(remote_command( 7367 &src_api_socket, 7368 "remove-device", 7369 Some(net_id), 7370 )); 7371 thread::sleep(std::time::Duration::new(10, 0)); 7372 7373 // Plug the virtio-net device again 7374 assert!(remote_command( 7375 &src_api_socket, 7376 "add-net", 7377 Some(net_params.as_str()), 7378 )); 7379 thread::sleep(std::time::Duration::new(10, 0)); 7380 } 7381 7382 // Start the live-migration 7383 let migration_socket = String::from( 7384 guest 7385 .tmp_dir 7386 .as_path() 7387 .join("live-migration.sock") 7388 .to_str() 7389 .unwrap(), 7390 ); 7391 // Start to receive migration from the destintion VM 7392 let mut receive_migration = Command::new(clh_command("ch-remote")) 7393 .args(&[ 7394 &format!("--api-socket={}", &dest_api_socket), 7395 "receive-migration", 7396 &format! {"unix:{}", migration_socket}, 7397 ]) 7398 .stderr(Stdio::piped()) 7399 .stdout(Stdio::piped()) 7400 .spawn() 7401 .unwrap(); 7402 // Give it '1s' to make sure the 'migration_socket' file is properly created 7403 thread::sleep(std::time::Duration::new(1, 0)); 7404 // Start to send migration from the source VM 7405 7406 let mut args = [ 7407 format!("--api-socket={}", &src_api_socket), 7408 "send-migration".to_string(), 7409 format! {"unix:{}", migration_socket}, 7410 ] 7411 .to_vec(); 7412 7413 if local { 7414 args.insert(2, "--local".to_string()); 7415 } 7416 7417 let mut send_migration = Command::new(clh_command("ch-remote")) 7418 .args(&args) 7419 .stderr(Stdio::piped()) 7420 .stdout(Stdio::piped()) 7421 .spawn() 7422 .unwrap(); 7423 7424 // The 'send-migration' command should be executed successfully within the given timeout 7425 let success = if let Some(status) = send_migration 7426 .wait_timeout(std::time::Duration::from_secs(30)) 7427 .unwrap() 7428 { 7429 status.success() 7430 } else { 7431 false 7432 }; 7433 7434 if !success { 7435 let _ = send_migration.kill(); 7436 let output = send_migration.wait_with_output().unwrap(); 7437 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 7438 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 7439 } 7440 7441 // The 'receive-migration' command should be executed successfully within the given timeout 7442 let success = if let Some(status) = receive_migration 7443 .wait_timeout(std::time::Duration::from_secs(30)) 7444 .unwrap() 7445 { 7446 status.success() 7447 } else { 7448 false 7449 }; 7450 7451 if !success { 7452 let _ = receive_migration.kill(); 7453 let output = receive_migration.wait_with_output().unwrap(); 7454 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 7455 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 7456 } 7457 assert!( 7458 success, 7459 "Unsuccessful command: 'send-migration' or 'receive-migration'." 7460 ); 7461 }); 7462 7463 let print_and_panic = |src_vm: Child, dest_vm: Child, message: &str| -> ! { 7464 let mut src_vm = src_vm; 7465 let mut dest_vm = dest_vm; 7466 7467 let _ = src_vm.kill(); 7468 let src_output = src_vm.wait_with_output().unwrap(); 7469 eprintln!( 7470 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 7471 String::from_utf8_lossy(&src_output.stdout) 7472 ); 7473 eprintln!( 7474 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 7475 String::from_utf8_lossy(&src_output.stderr) 7476 ); 7477 let _ = dest_vm.kill(); 7478 let dest_output = dest_vm.wait_with_output().unwrap(); 7479 eprintln!( 7480 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 7481 String::from_utf8_lossy(&dest_output.stdout) 7482 ); 7483 eprintln!( 7484 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 7485 String::from_utf8_lossy(&dest_output.stderr) 7486 ); 7487 7488 panic!("Test failed: {}", message) 7489 }; 7490 7491 // Check and report any errors occured during the live-migration 7492 if r.is_err() { 7493 print_and_panic(src_child, dest_child, "Error occured during live-migration"); 7494 } 7495 7496 // Check the source vm has been terminated successful (give it '3s' to settle) 7497 thread::sleep(std::time::Duration::new(3, 0)); 7498 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 7499 print_and_panic( 7500 src_child, 7501 dest_child, 7502 "source VM was not terminated successfully.", 7503 ); 7504 }; 7505 7506 // Post live-migration check to make sure the destination VM is funcational 7507 let r = std::panic::catch_unwind(|| { 7508 // Perform same checks to validate VM has been properly migrated 7509 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 6); 7510 if numa { 7511 #[cfg(target_arch = "x86_64")] 7512 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 7513 #[cfg(target_arch = "aarch64")] 7514 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 7515 } else { 7516 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 7517 } 7518 guest.check_devices_common(None, Some(&console_text)); 7519 7520 // Perform NUMA related checks 7521 if numa { 7522 #[cfg(target_arch = "aarch64")] 7523 { 7524 guest.check_numa_common( 7525 Some(&[960_000, 960_000, 1_920_000]), 7526 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 7527 Some(&["10 15 20", "20 10 25", "25 30 10"]), 7528 ); 7529 } 7530 7531 // AArch64 currently does not support hotplug, and therefore we only 7532 // test hotplug-related function on x86_64 here. 7533 #[cfg(target_arch = "x86_64")] 7534 { 7535 guest.check_numa_common( 7536 Some(&[1_920_000, 1_920_000, 2_880_000]), 7537 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 7538 Some(&["10 15 20", "20 10 25", "25 30 10"]), 7539 ); 7540 7541 guest.enable_memory_hotplug(); 7542 7543 // Resize every memory zone and check each associated NUMA node 7544 // has been assigned the right amount of memory. 7545 resize_zone_command(&dest_api_socket, "mem0", "4G"); 7546 resize_zone_command(&dest_api_socket, "mem1", "4G"); 7547 resize_zone_command(&dest_api_socket, "mem2", "4G"); 7548 // Resize to the maximum amount of CPUs and check each NUMA 7549 // node has been assigned the right CPUs set. 7550 resize_command(&dest_api_socket, Some(12), None, None); 7551 thread::sleep(std::time::Duration::new(5, 0)); 7552 7553 guest.check_numa_common( 7554 Some(&[3_840_000, 3_840_000, 3_840_000]), 7555 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 7556 None, 7557 ); 7558 } 7559 } 7560 }); 7561 7562 // Clean-up the destination VM and make sure it terminated correctly 7563 let _ = dest_child.kill(); 7564 let dest_output = dest_child.wait_with_output().unwrap(); 7565 handle_child_output(r, &dest_output); 7566 7567 // Check the destination VM has the expected 'concole_text' from its output 7568 let r = std::panic::catch_unwind(|| { 7569 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 7570 }); 7571 handle_child_output(r, &dest_output); 7572 } 7573 7574 #[test] 7575 fn test_live_migration_basic() { 7576 _test_live_migration(false, false) 7577 } 7578 7579 #[test] 7580 fn test_live_migration_local() { 7581 _test_live_migration(false, true) 7582 } 7583 7584 #[test] 7585 #[cfg(not(feature = "mshv"))] 7586 fn test_live_migration_numa() { 7587 _test_live_migration(true, false) 7588 } 7589 7590 fn _test_live_migration_ovs_dpdk(local: bool) { 7591 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7592 let ovs_guest = Guest::new(Box::new(ovs_focal)); 7593 7594 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7595 let migration_guest = Guest::new(Box::new(migration_focal)); 7596 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 7597 7598 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 7599 let (mut ovs_child, mut src_child) = 7600 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket); 7601 7602 // Start the destination VM 7603 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 7604 dest_api_socket.push_str(".dest"); 7605 let mut dest_child = GuestCommand::new(&migration_guest) 7606 .args(&["--api-socket", &dest_api_socket]) 7607 .capture_output() 7608 .spawn() 7609 .unwrap(); 7610 7611 let r = std::panic::catch_unwind(|| { 7612 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 7613 thread::sleep(std::time::Duration::new(1, 0)); 7614 7615 // Start the live-migration 7616 let migration_socket = String::from( 7617 migration_guest 7618 .tmp_dir 7619 .as_path() 7620 .join("live-migration.sock") 7621 .to_str() 7622 .unwrap(), 7623 ); 7624 // Start to receive migration from the destintion VM 7625 let mut receive_migration = Command::new(clh_command("ch-remote")) 7626 .args(&[ 7627 &format!("--api-socket={}", &dest_api_socket), 7628 "receive-migration", 7629 &format! {"unix:{}", migration_socket}, 7630 ]) 7631 .stderr(Stdio::piped()) 7632 .stdout(Stdio::piped()) 7633 .spawn() 7634 .unwrap(); 7635 // Give it '1s' to make sure the 'migration_socket' file is properly created 7636 thread::sleep(std::time::Duration::new(1, 0)); 7637 // Start to send migration from the source VM 7638 let mut args = [ 7639 format!("--api-socket={}", &src_api_socket), 7640 "send-migration".to_string(), 7641 format! {"unix:{}", migration_socket}, 7642 ] 7643 .to_vec(); 7644 7645 if local { 7646 args.insert(2, "--local".to_string()); 7647 } 7648 7649 let mut send_migration = Command::new(clh_command("ch-remote")) 7650 .args(&args) 7651 .stderr(Stdio::piped()) 7652 .stdout(Stdio::piped()) 7653 .spawn() 7654 .unwrap(); 7655 7656 // The 'send-migration' command should be executed successfully within the given timeout 7657 let success = if let Some(status) = send_migration 7658 .wait_timeout(std::time::Duration::from_secs(30)) 7659 .unwrap() 7660 { 7661 status.success() 7662 } else { 7663 false 7664 }; 7665 7666 if !success { 7667 let _ = send_migration.kill(); 7668 let output = send_migration.wait_with_output().unwrap(); 7669 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 7670 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 7671 } 7672 7673 // The 'receive-migration' command should be executed successfully within the given timeout 7674 let success = if let Some(status) = receive_migration 7675 .wait_timeout(std::time::Duration::from_secs(30)) 7676 .unwrap() 7677 { 7678 status.success() 7679 } else { 7680 false 7681 }; 7682 7683 if !success { 7684 let _ = receive_migration.kill(); 7685 let output = receive_migration.wait_with_output().unwrap(); 7686 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 7687 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 7688 } 7689 assert!( 7690 success, 7691 "Unsuccessful command: 'send-migration' or 'receive-migration'." 7692 ); 7693 }); 7694 7695 let print_and_panic = |src_vm: Child, dest_vm: Child, ovs_vm: Child, message: &str| -> ! { 7696 let mut src_vm = src_vm; 7697 let mut dest_vm = dest_vm; 7698 let mut ovs_vm = ovs_vm; 7699 7700 let _ = src_vm.kill(); 7701 let src_output = src_vm.wait_with_output().unwrap(); 7702 eprintln!( 7703 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 7704 String::from_utf8_lossy(&src_output.stdout) 7705 ); 7706 eprintln!( 7707 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 7708 String::from_utf8_lossy(&src_output.stderr) 7709 ); 7710 let _ = dest_vm.kill(); 7711 let dest_output = dest_vm.wait_with_output().unwrap(); 7712 eprintln!( 7713 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 7714 String::from_utf8_lossy(&dest_output.stdout) 7715 ); 7716 eprintln!( 7717 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 7718 String::from_utf8_lossy(&dest_output.stderr) 7719 ); 7720 let _ = ovs_vm.kill(); 7721 let ovs_output = ovs_vm.wait_with_output().unwrap(); 7722 eprintln!( 7723 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 7724 String::from_utf8_lossy(&ovs_output.stdout) 7725 ); 7726 eprintln!( 7727 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 7728 String::from_utf8_lossy(&ovs_output.stderr) 7729 ); 7730 7731 cleanup_ovs_dpdk(); 7732 7733 panic!("Test failed: {}", message) 7734 }; 7735 7736 // Check and report any errors occured during the live-migration 7737 if r.is_err() { 7738 print_and_panic( 7739 src_child, 7740 dest_child, 7741 ovs_child, 7742 "Error occured during live-migration", 7743 ); 7744 } 7745 7746 // Check the source vm has been terminated successful (give it '3s' to settle) 7747 thread::sleep(std::time::Duration::new(3, 0)); 7748 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 7749 print_and_panic( 7750 src_child, 7751 dest_child, 7752 ovs_child, 7753 "source VM was not terminated successfully.", 7754 ); 7755 }; 7756 7757 // Post live-migration check to make sure the destination VM is funcational 7758 let r = std::panic::catch_unwind(|| { 7759 // Perform same checks to validate VM has been properly migrated 7760 // Spawn a new netcat listener in the OVS VM 7761 let guest_ip = ovs_guest.network.guest_ip.clone(); 7762 thread::spawn(move || { 7763 ssh_command_ip( 7764 "nc -l 12345", 7765 &guest_ip, 7766 DEFAULT_SSH_RETRIES, 7767 DEFAULT_SSH_TIMEOUT, 7768 ) 7769 .unwrap(); 7770 }); 7771 7772 // Wait for the server to be listening 7773 thread::sleep(std::time::Duration::new(5, 0)); 7774 7775 // And check the connection is still functional after live-migration 7776 migration_guest 7777 .ssh_command("nc -vz 172.100.0.1 12345") 7778 .unwrap(); 7779 7780 cleanup_ovs_dpdk(); 7781 }); 7782 7783 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 7784 let _ = dest_child.kill(); 7785 let _ = ovs_child.kill(); 7786 let dest_output = dest_child.wait_with_output().unwrap(); 7787 handle_child_output(r, &dest_output); 7788 let ovs_output = ovs_child.wait_with_output().unwrap(); 7789 handle_child_output(Ok(()), &ovs_output); 7790 } 7791 7792 #[test] 7793 #[cfg(not(feature = "mshv"))] 7794 fn test_live_migration_ovs_dpdk() { 7795 _test_live_migration_ovs_dpdk(false); 7796 } 7797 7798 #[test] 7799 #[cfg(not(feature = "mshv"))] 7800 fn test_live_migration_ovs_dpdk_local() { 7801 _test_live_migration_ovs_dpdk(true); 7802 } 7803 } 7804 7805 #[cfg(target_arch = "aarch64")] 7806 mod aarch64_acpi { 7807 use crate::*; 7808 7809 #[test] 7810 fn test_simple_launch_acpi() { 7811 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7812 7813 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 7814 let guest = Guest::new(disk_config); 7815 7816 let mut child = GuestCommand::new(&guest) 7817 .args(&["--cpus", "boot=1"]) 7818 .args(&["--memory", "size=512M"]) 7819 .args(&["--kernel", edk2_path().to_str().unwrap()]) 7820 .default_disks() 7821 .default_net() 7822 .args(&["--serial", "tty", "--console", "off"]) 7823 .capture_output() 7824 .spawn() 7825 .unwrap(); 7826 7827 let r = std::panic::catch_unwind(|| { 7828 guest.wait_vm_boot(Some(120)).unwrap(); 7829 7830 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 7831 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 7832 assert!(guest.get_entropy().unwrap_or_default() >= 900); 7833 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 7834 }); 7835 7836 let _ = child.kill(); 7837 let output = child.wait_with_output().unwrap(); 7838 7839 handle_child_output(r, &output); 7840 }); 7841 } 7842 7843 #[test] 7844 fn test_guest_numa_nodes_acpi() { 7845 _test_guest_numa_nodes(true); 7846 } 7847 7848 #[test] 7849 fn test_cpu_topology_421_acpi() { 7850 test_cpu_topology(4, 2, 1, true); 7851 } 7852 7853 #[test] 7854 fn test_cpu_topology_142_acpi() { 7855 test_cpu_topology(1, 4, 2, true); 7856 } 7857 7858 #[test] 7859 fn test_cpu_topology_262_acpi() { 7860 test_cpu_topology(2, 6, 2, true); 7861 } 7862 7863 #[test] 7864 fn test_power_button_acpi() { 7865 _test_power_button(true); 7866 } 7867 7868 // https://github.com/cloud-hypervisor/cloud-hypervisor/issues/3941 7869 #[test] 7870 #[ignore] 7871 fn test_virtio_iommu() { 7872 _test_virtio_iommu(true) 7873 } 7874 } 7875