1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 #[cfg(test)] 7 #[cfg(feature = "integration_tests")] 8 #[macro_use] 9 extern crate lazy_static; 10 11 #[cfg(test)] 12 #[cfg(feature = "integration_tests")] 13 extern crate test_infra; 14 15 #[cfg(test)] 16 #[cfg(feature = "integration_tests")] 17 mod tests { 18 use net_util::MacAddr; 19 use std::collections::HashMap; 20 use std::env; 21 use std::ffi::OsStr; 22 use std::fs; 23 use std::io; 24 use std::io::BufRead; 25 use std::io::Read; 26 use std::io::Seek; 27 use std::io::Write; 28 use std::os::unix::io::AsRawFd; 29 use std::path::PathBuf; 30 use std::process::{Child, Command, Stdio}; 31 use std::string::String; 32 use std::sync::mpsc::Receiver; 33 use std::sync::{mpsc, Mutex}; 34 use std::thread; 35 use test_infra::*; 36 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 37 #[cfg_attr(target_arch = "aarch64", allow(unused_imports))] 38 use wait_timeout::ChildExt; 39 40 lazy_static! { 41 static ref NEXT_VM_ID: Mutex<u8> = Mutex::new(1); 42 } 43 44 struct Guest { 45 tmp_dir: TempDir, 46 disk_config: Box<dyn DiskConfig>, 47 fw_path: String, 48 network: GuestNetworkConfig, 49 } 50 51 // Safe to implement as we know we have no interior mutability 52 impl std::panic::RefUnwindSafe for Guest {} 53 54 #[cfg(target_arch = "x86_64")] 55 const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw"; 56 #[cfg(target_arch = "x86_64")] 57 const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 58 #[cfg(target_arch = "x86_64")] 59 const FOCAL_SGX_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-sgx.raw"; 60 #[cfg(target_arch = "x86_64")] 61 const HIRSUTE_NVIDIA_IMAGE_NAME: &str = "hirsute-server-cloudimg-amd64-nvidia.raw"; 62 #[cfg(target_arch = "aarch64")] 63 const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 64 #[cfg(target_arch = "aarch64")] 65 const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom.raw"; 66 #[cfg(target_arch = "aarch64")] 67 const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom.qcow2"; 68 #[cfg(target_arch = "x86_64")] 69 const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 70 #[cfg(target_arch = "aarch64")] 71 const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom.vhd"; 72 #[cfg(target_arch = "x86_64")] 73 const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 74 #[cfg(target_arch = "aarch64")] 75 const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom.vhdx"; 76 #[cfg(target_arch = "x86_64")] 77 const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 78 #[cfg(target_arch = "x86_64")] 79 const WINDOWS_IMAGE_NAME: &str = "windows-server-2019.raw"; 80 #[cfg(target_arch = "x86_64")] 81 const OVMF_NAME: &str = "OVMF-4b47d0c6c8.fd"; 82 83 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 84 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 85 86 #[cfg(target_arch = "x86_64")] 87 const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 88 #[cfg(target_arch = "aarch64")] 89 const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 90 91 const PIPE_SIZE: i32 = 32 << 20; 92 93 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 94 95 fn clh_command(cmd: &str) -> String { 96 env::var("BUILD_TARGET").map_or( 97 format!("target/x86_64-unknown-linux-gnu/release/{}", cmd), 98 |target| format!("target/{}/release/{}", target, cmd), 99 ) 100 } 101 102 fn prepare_virtiofsd( 103 tmp_dir: &TempDir, 104 shared_dir: &str, 105 cache: &str, 106 ) -> (std::process::Child, String) { 107 let mut workload_path = dirs::home_dir().unwrap(); 108 workload_path.push("workloads"); 109 110 let mut virtiofsd_path = workload_path; 111 virtiofsd_path.push("virtiofsd"); 112 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 113 114 let virtiofsd_socket_path = 115 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 116 117 // Start the daemon 118 let child = Command::new(virtiofsd_path.as_str()) 119 .args(&[format!("--socket-path={}", virtiofsd_socket_path).as_str()]) 120 .args(&["-o", format!("source={}", shared_dir).as_str()]) 121 .args(&["-o", format!("cache={}", cache).as_str()]) 122 .spawn() 123 .unwrap(); 124 125 thread::sleep(std::time::Duration::new(10, 0)); 126 127 (child, virtiofsd_socket_path) 128 } 129 130 fn prepare_virtofsd_rs_daemon( 131 tmp_dir: &TempDir, 132 shared_dir: &str, 133 _cache: &str, 134 ) -> (std::process::Child, String) { 135 let mut workload_path = dirs::home_dir().unwrap(); 136 workload_path.push("workloads"); 137 138 let mut virtiofsd_path = workload_path; 139 virtiofsd_path.push("virtiofsd-rs"); 140 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 141 142 let virtiofsd_socket_path = 143 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 144 145 // Start the daemon 146 let child = Command::new(virtiofsd_path.as_str()) 147 .args(&["--shared-dir", shared_dir]) 148 .args(&["--socket", virtiofsd_socket_path.as_str()]) 149 .spawn() 150 .unwrap(); 151 152 thread::sleep(std::time::Duration::new(10, 0)); 153 154 (child, virtiofsd_socket_path) 155 } 156 157 fn prepare_vubd( 158 tmp_dir: &TempDir, 159 blk_img: &str, 160 num_queues: usize, 161 rdonly: bool, 162 direct: bool, 163 ) -> (std::process::Child, String) { 164 let mut workload_path = dirs::home_dir().unwrap(); 165 workload_path.push("workloads"); 166 167 let mut blk_file_path = workload_path; 168 blk_file_path.push(blk_img); 169 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 170 171 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 172 173 // Start the daemon 174 let child = Command::new(clh_command("vhost_user_block")) 175 .args(&[ 176 "--block-backend", 177 format!( 178 "path={},socket={},num_queues={},readonly={},direct={}", 179 blk_file_path, vubd_socket_path, num_queues, rdonly, direct 180 ) 181 .as_str(), 182 ]) 183 .spawn() 184 .unwrap(); 185 186 thread::sleep(std::time::Duration::new(10, 0)); 187 188 (child, vubd_socket_path) 189 } 190 191 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 192 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 193 } 194 195 fn temp_api_path(tmp_dir: &TempDir) -> String { 196 String::from( 197 tmp_dir 198 .as_path() 199 .join("cloud-hypervisor.sock") 200 .to_str() 201 .unwrap(), 202 ) 203 } 204 205 // Creates the directory and returns the path. 206 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 207 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 208 std::fs::create_dir(&snapshot_dir).unwrap(); 209 snapshot_dir 210 } 211 212 // Creates the path for direct kernel boot and return the path. 213 // For x86_64, this function returns the vmlinux kernel path. 214 // For AArch64, this function returns the PE kernel path. 215 fn direct_kernel_boot_path() -> PathBuf { 216 let mut workload_path = dirs::home_dir().unwrap(); 217 workload_path.push("workloads"); 218 219 let mut kernel_path = workload_path; 220 #[cfg(target_arch = "x86_64")] 221 kernel_path.push("vmlinux"); 222 #[cfg(target_arch = "aarch64")] 223 kernel_path.push("Image"); 224 225 kernel_path 226 } 227 228 #[cfg(target_arch = "aarch64")] 229 fn edk2_path() -> PathBuf { 230 let mut workload_path = dirs::home_dir().unwrap(); 231 workload_path.push("workloads"); 232 let mut edk2_path = workload_path; 233 edk2_path.push("CLOUDHV_EFI.fd"); 234 235 edk2_path 236 } 237 238 fn prepare_vhost_user_net_daemon( 239 tmp_dir: &TempDir, 240 ip: &str, 241 tap: Option<&str>, 242 num_queues: usize, 243 client_mode: bool, 244 ) -> (std::process::Command, String) { 245 let vunet_socket_path = 246 String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 247 248 // Start the daemon 249 let net_params = if let Some(tap_str) = tap { 250 format!( 251 "tap={},ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 252 tap_str, ip, vunet_socket_path, num_queues, client_mode 253 ) 254 } else { 255 format!( 256 "ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 257 ip, vunet_socket_path, num_queues, client_mode 258 ) 259 }; 260 261 let mut command = Command::new(clh_command("vhost_user_net")); 262 command.args(&["--net-backend", net_params.as_str()]); 263 264 (command, vunet_socket_path) 265 } 266 267 fn curl_command(api_socket: &str, method: &str, url: &str, http_body: Option<&str>) { 268 let mut curl_args: Vec<&str> = 269 ["--unix-socket", api_socket, "-i", "-X", method, url].to_vec(); 270 271 if let Some(body) = http_body { 272 curl_args.push("-H"); 273 curl_args.push("Accept: application/json"); 274 curl_args.push("-H"); 275 curl_args.push("Content-Type: application/json"); 276 curl_args.push("-d"); 277 curl_args.push(body); 278 } 279 280 let status = Command::new("curl") 281 .args(curl_args) 282 .status() 283 .expect("Failed to launch curl command"); 284 285 assert!(status.success()); 286 } 287 288 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 289 let mut cmd = Command::new(clh_command("ch-remote")); 290 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 291 292 if let Some(arg) = arg { 293 cmd.arg(arg); 294 } 295 296 cmd.status().expect("Failed to launch ch-remote").success() 297 } 298 299 fn remote_command_w_output( 300 api_socket: &str, 301 command: &str, 302 arg: Option<&str>, 303 ) -> (bool, Vec<u8>) { 304 let mut cmd = Command::new(clh_command("ch-remote")); 305 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 306 307 if let Some(arg) = arg { 308 cmd.arg(arg); 309 } 310 311 let output = cmd.output().expect("Failed to launch ch-remote"); 312 313 (output.status.success(), output.stdout) 314 } 315 316 fn resize_command( 317 api_socket: &str, 318 desired_vcpus: Option<u8>, 319 desired_ram: Option<usize>, 320 desired_balloon: Option<usize>, 321 ) -> bool { 322 let mut cmd = Command::new(clh_command("ch-remote")); 323 cmd.args(&[&format!("--api-socket={}", api_socket), "resize"]); 324 325 if let Some(desired_vcpus) = desired_vcpus { 326 cmd.arg(format!("--cpus={}", desired_vcpus)); 327 } 328 329 if let Some(desired_ram) = desired_ram { 330 cmd.arg(format!("--memory={}", desired_ram)); 331 } 332 333 if let Some(desired_balloon) = desired_balloon { 334 cmd.arg(format!("--balloon={}", desired_balloon)); 335 } 336 337 cmd.status().expect("Failed to launch ch-remote").success() 338 } 339 340 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 341 let mut cmd = Command::new(clh_command("ch-remote")); 342 cmd.args(&[ 343 &format!("--api-socket={}", api_socket), 344 "resize-zone", 345 &format!("--id={}", id), 346 &format!("--size={}", desired_size), 347 ]); 348 349 cmd.status().expect("Failed to launch ch-remote").success() 350 } 351 352 // setup OVS-DPDK bridge and ports 353 fn setup_ovs_dpdk() { 354 // setup OVS-DPDK 355 assert!(exec_host_command_status("service openvswitch-switch start").success()); 356 assert!(exec_host_command_status("ovs-vsctl init").success()); 357 assert!(exec_host_command_status( 358 "ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true" 359 ) 360 .success()); 361 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 362 363 // Create OVS-DPDK bridge and ports 364 assert!(exec_host_command_status( 365 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 366 ) 367 .success()); 368 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 369 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 370 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 371 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 372 } 373 fn cleanup_ovs_dpdk() { 374 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 375 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 376 } 377 // Setup two guests and ensure they are connected through ovs-dpdk 378 fn setup_ovs_dpdk_guests(guest1: &Guest, guest2: &Guest, api_socket: &str) -> (Child, Child) { 379 setup_ovs_dpdk(); 380 381 let mut child1 = GuestCommand::new(guest1) 382 .args(&["--cpus", "boot=2"]) 383 .args(&["--memory", "size=0,shared=on"]) 384 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 385 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 386 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 387 .default_disks() 388 .args(&["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 389 .capture_output() 390 .spawn() 391 .unwrap(); 392 393 #[cfg(target_arch = "x86_64")] 394 let guest_net_iface = "ens5"; 395 #[cfg(target_arch = "aarch64")] 396 let guest_net_iface = "enp0s5"; 397 398 let r = std::panic::catch_unwind(|| { 399 guest1.wait_vm_boot(None).unwrap(); 400 401 guest1 402 .ssh_command(&format!( 403 "sudo ip addr add 172.100.0.1/24 dev {}", 404 guest_net_iface 405 )) 406 .unwrap(); 407 guest1 408 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 409 .unwrap(); 410 411 let guest_ip = guest1.network.guest_ip.clone(); 412 thread::spawn(move || { 413 ssh_command_ip( 414 "nc -l 12345", 415 &guest_ip, 416 DEFAULT_SSH_RETRIES, 417 DEFAULT_SSH_TIMEOUT, 418 ) 419 .unwrap(); 420 }); 421 }); 422 if r.is_err() { 423 cleanup_ovs_dpdk(); 424 425 let _ = child1.kill(); 426 let output = child1.wait_with_output().unwrap(); 427 handle_child_output(r, &output); 428 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 429 } 430 431 let mut child2 = GuestCommand::new(guest2) 432 .args(&["--api-socket", api_socket]) 433 .args(&["--cpus", "boot=2"]) 434 .args(&["--memory", "size=0,shared=on"]) 435 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 436 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 437 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 438 .default_disks() 439 .args(&["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 440 .capture_output() 441 .spawn() 442 .unwrap(); 443 444 let r = std::panic::catch_unwind(|| { 445 guest2.wait_vm_boot(None).unwrap(); 446 447 guest2 448 .ssh_command(&format!( 449 "sudo ip addr add 172.100.0.2/24 dev {}", 450 guest_net_iface 451 )) 452 .unwrap(); 453 guest2 454 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 455 .unwrap(); 456 457 // Check the connection works properly between the two VMs 458 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 459 }); 460 if r.is_err() { 461 cleanup_ovs_dpdk(); 462 463 let _ = child1.kill(); 464 let _ = child2.kill(); 465 let output = child2.wait_with_output().unwrap(); 466 handle_child_output(r, &output); 467 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 468 } 469 470 (child1, child2) 471 } 472 473 #[derive(Debug)] 474 enum Error { 475 Parsing(std::num::ParseIntError), 476 SshCommand(SshCommandError), 477 WaitForBoot(WaitForBootError), 478 } 479 480 impl From<SshCommandError> for Error { 481 fn from(e: SshCommandError) -> Self { 482 Self::SshCommand(e) 483 } 484 } 485 486 impl Guest { 487 fn new_from_ip_range(mut disk_config: Box<dyn DiskConfig>, class: &str, id: u8) -> Self { 488 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 489 490 let mut workload_path = dirs::home_dir().unwrap(); 491 workload_path.push("workloads"); 492 493 let mut fw_path = workload_path; 494 #[cfg(target_arch = "aarch64")] 495 fw_path.push("CLOUDHV_EFI.fd"); 496 #[cfg(target_arch = "x86_64")] 497 fw_path.push("hypervisor-fw"); 498 let fw_path = String::from(fw_path.to_str().unwrap()); 499 let network = GuestNetworkConfig { 500 guest_ip: format!("{}.{}.2", class, id), 501 l2_guest_ip1: format!("{}.{}.3", class, id), 502 l2_guest_ip2: format!("{}.{}.4", class, id), 503 l2_guest_ip3: format!("{}.{}.5", class, id), 504 host_ip: format!("{}.{}.1", class, id), 505 guest_mac: format!("12:34:56:78:90:{:02x}", id), 506 l2_guest_mac1: format!("de:ad:be:ef:12:{:02x}", id), 507 l2_guest_mac2: format!("de:ad:be:ef:34:{:02x}", id), 508 l2_guest_mac3: format!("de:ad:be:ef:56:{:02x}", id), 509 tcp_listener_port: DEFAULT_TCP_LISTENER_PORT + id as u16, 510 }; 511 512 disk_config.prepare_files(&tmp_dir, &network); 513 514 Guest { 515 tmp_dir, 516 disk_config, 517 fw_path, 518 network, 519 } 520 } 521 522 fn new(disk_config: Box<dyn DiskConfig>) -> Self { 523 let mut guard = NEXT_VM_ID.lock().unwrap(); 524 let id = *guard; 525 *guard = id + 1; 526 527 Self::new_from_ip_range(disk_config, "192.168", id) 528 } 529 530 fn default_net_string(&self) -> String { 531 format!( 532 "tap=,mac={},ip={},mask=255.255.255.0", 533 self.network.guest_mac, self.network.host_ip 534 ) 535 } 536 537 fn default_net_string_w_iommu(&self) -> String { 538 format!( 539 "tap=,mac={},ip={},mask=255.255.255.0,iommu=on", 540 self.network.guest_mac, self.network.host_ip 541 ) 542 } 543 544 fn ssh_command(&self, command: &str) -> Result<String, SshCommandError> { 545 ssh_command_ip( 546 command, 547 &self.network.guest_ip, 548 DEFAULT_SSH_RETRIES, 549 DEFAULT_SSH_TIMEOUT, 550 ) 551 } 552 553 fn ssh_command_l1(&self, command: &str) -> Result<String, SshCommandError> { 554 ssh_command_ip( 555 command, 556 &self.network.guest_ip, 557 DEFAULT_SSH_RETRIES, 558 DEFAULT_SSH_TIMEOUT, 559 ) 560 } 561 562 fn ssh_command_l2_1(&self, command: &str) -> Result<String, SshCommandError> { 563 ssh_command_ip( 564 command, 565 &self.network.l2_guest_ip1, 566 DEFAULT_SSH_RETRIES, 567 DEFAULT_SSH_TIMEOUT, 568 ) 569 } 570 571 fn ssh_command_l2_2(&self, command: &str) -> Result<String, SshCommandError> { 572 ssh_command_ip( 573 command, 574 &self.network.l2_guest_ip2, 575 DEFAULT_SSH_RETRIES, 576 DEFAULT_SSH_TIMEOUT, 577 ) 578 } 579 580 fn ssh_command_l2_3(&self, command: &str) -> Result<String, SshCommandError> { 581 ssh_command_ip( 582 command, 583 &self.network.l2_guest_ip3, 584 DEFAULT_SSH_RETRIES, 585 DEFAULT_SSH_TIMEOUT, 586 ) 587 } 588 589 fn api_create_body(&self, cpu_count: u8) -> String { 590 #[cfg(target_arch = "x86_64")] 591 format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}", 592 cpu_count, 593 cpu_count, 594 self.fw_path.as_str(), 595 self.network.host_ip, 596 self.network.guest_mac, 597 self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(), 598 self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(), 599 } 600 #[cfg(target_arch = "aarch64")] 601 format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"{}\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}", 602 cpu_count, 603 cpu_count, 604 direct_kernel_boot_path().to_str().unwrap(), 605 DIRECT_KERNEL_BOOT_CMDLINE, 606 self.network.host_ip, 607 self.network.guest_mac, 608 self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(), 609 self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(), 610 } 611 } 612 613 fn get_cpu_count(&self) -> Result<u32, Error> { 614 self.ssh_command("grep -c processor /proc/cpuinfo")? 615 .trim() 616 .parse() 617 .map_err(Error::Parsing) 618 } 619 620 fn get_initial_apicid(&self) -> Result<u32, Error> { 621 self.ssh_command("grep \"initial apicid\" /proc/cpuinfo | grep -o \"[0-9]*\"")? 622 .trim() 623 .parse() 624 .map_err(Error::Parsing) 625 } 626 627 fn get_total_memory(&self) -> Result<u32, Error> { 628 self.ssh_command("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")? 629 .trim() 630 .parse() 631 .map_err(Error::Parsing) 632 } 633 634 fn get_total_memory_l2(&self) -> Result<u32, Error> { 635 self.ssh_command_l2_1("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")? 636 .trim() 637 .parse() 638 .map_err(Error::Parsing) 639 } 640 641 fn get_numa_node_memory(&self, node_id: usize) -> Result<u32, Error> { 642 self.ssh_command( 643 format!( 644 "grep MemTotal /sys/devices/system/node/node{}/meminfo \ 645 | cut -d \":\" -f 2 | grep -o \"[0-9]*\"", 646 node_id 647 ) 648 .as_str(), 649 )? 650 .trim() 651 .parse() 652 .map_err(Error::Parsing) 653 } 654 655 fn wait_vm_boot(&self, custom_timeout: Option<i32>) -> Result<(), Error> { 656 self.network 657 .wait_vm_boot(custom_timeout) 658 .map_err(Error::WaitForBoot) 659 } 660 661 fn check_numa_node_cpus(&self, node_id: usize, cpus: Vec<usize>) -> Result<(), Error> { 662 for cpu in cpus.iter() { 663 let cmd = format!( 664 "[ -d \"/sys/devices/system/node/node{}/cpu{}\" ]", 665 node_id, cpu 666 ); 667 self.ssh_command(cmd.as_str())?; 668 } 669 670 Ok(()) 671 } 672 673 fn check_numa_node_distances( 674 &self, 675 node_id: usize, 676 distances: &str, 677 ) -> Result<bool, Error> { 678 let cmd = format!("cat /sys/devices/system/node/node{}/distance", node_id); 679 if self.ssh_command(cmd.as_str())?.trim() == distances { 680 Ok(true) 681 } else { 682 Ok(false) 683 } 684 } 685 686 fn check_numa_common( 687 &self, 688 mem_ref: Option<&[u32]>, 689 node_ref: Option<&[Vec<usize>]>, 690 distance_ref: Option<&[&str]>, 691 ) { 692 if let Some(mem_ref) = mem_ref { 693 // Check each NUMA node has been assigned the right amount of 694 // memory. 695 for (i, &m) in mem_ref.iter().enumerate() { 696 assert!(self.get_numa_node_memory(i).unwrap_or_default() > m); 697 } 698 } 699 700 if let Some(node_ref) = node_ref { 701 // Check each NUMA node has been assigned the right CPUs set. 702 for (i, n) in node_ref.iter().enumerate() { 703 self.check_numa_node_cpus(i, n.clone()).unwrap(); 704 } 705 } 706 707 if let Some(distance_ref) = distance_ref { 708 // Check each NUMA node has been assigned the right distances. 709 for (i, &d) in distance_ref.iter().enumerate() { 710 assert!(self.check_numa_node_distances(i, d).unwrap()); 711 } 712 } 713 } 714 715 fn check_sgx_support(&self) -> Result<(), Error> { 716 self.ssh_command( 717 "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX: \ 718 Software Guard Extensions supported = true'", 719 )?; 720 self.ssh_command( 721 "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX_LC: \ 722 SGX launch config supported = true'", 723 )?; 724 self.ssh_command( 725 "cpuid -l 0x12 -s 0 | tr -s [:space:] | grep -q 'SGX1 \ 726 supported = true'", 727 )?; 728 729 Ok(()) 730 } 731 732 fn get_entropy(&self) -> Result<u32, Error> { 733 self.ssh_command("cat /proc/sys/kernel/random/entropy_avail")? 734 .trim() 735 .parse() 736 .map_err(Error::Parsing) 737 } 738 739 fn get_pci_bridge_class(&self) -> Result<String, Error> { 740 Ok(self 741 .ssh_command("cat /sys/bus/pci/devices/0000:00:00.0/class")? 742 .trim() 743 .to_string()) 744 } 745 746 fn get_pci_device_ids(&self) -> Result<String, Error> { 747 Ok(self 748 .ssh_command("cat /sys/bus/pci/devices/*/device")? 749 .trim() 750 .to_string()) 751 } 752 753 fn get_pci_vendor_ids(&self) -> Result<String, Error> { 754 Ok(self 755 .ssh_command("cat /sys/bus/pci/devices/*/vendor")? 756 .trim() 757 .to_string()) 758 } 759 760 fn does_device_vendor_pair_match( 761 &self, 762 device_id: &str, 763 vendor_id: &str, 764 ) -> Result<bool, Error> { 765 // We are checking if console device's device id and vendor id pair matches 766 let devices = self.get_pci_device_ids()?; 767 let devices: Vec<&str> = devices.split('\n').collect(); 768 let vendors = self.get_pci_vendor_ids()?; 769 let vendors: Vec<&str> = vendors.split('\n').collect(); 770 771 for (index, d_id) in devices.iter().enumerate() { 772 if *d_id == device_id { 773 if let Some(v_id) = vendors.get(index) { 774 if *v_id == vendor_id { 775 return Ok(true); 776 } 777 } 778 } 779 } 780 781 Ok(false) 782 } 783 784 fn valid_virtio_fs_cache_size( 785 &self, 786 dax: bool, 787 cache_size: Option<u64>, 788 ) -> Result<bool, Error> { 789 // SHM region is called different things depending on kernel 790 let shm_region = self 791 .ssh_command("sudo grep 'virtio[0-9]\\|virtio-pci-shm' /proc/iomem || true")? 792 .trim() 793 .to_string(); 794 795 if shm_region.is_empty() { 796 return Ok(!dax); 797 } 798 799 // From this point, the region is not empty, hence it is an error 800 // if DAX is off. 801 if !dax { 802 return Ok(false); 803 } 804 805 let cache = if let Some(cache) = cache_size { 806 cache 807 } else { 808 // 8Gib by default 809 0x0002_0000_0000 810 }; 811 812 let args: Vec<&str> = shm_region.split(':').collect(); 813 if args.is_empty() { 814 return Ok(false); 815 } 816 817 let args: Vec<&str> = args[0].trim().split('-').collect(); 818 if args.len() != 2 { 819 return Ok(false); 820 } 821 822 let start_addr = u64::from_str_radix(args[0], 16).map_err(Error::Parsing)?; 823 let end_addr = u64::from_str_radix(args[1], 16).map_err(Error::Parsing)?; 824 825 Ok(cache == (end_addr - start_addr + 1)) 826 } 827 828 fn check_vsock(&self, socket: &str) { 829 // Listen from guest on vsock CID=3 PORT=16 830 // SOCKET-LISTEN:<domain>:<protocol>:<local-address> 831 let guest_ip = self.network.guest_ip.clone(); 832 let listen_socat = thread::spawn(move || { 833 ssh_command_ip("sudo socat - SOCKET-LISTEN:40:0:x00x00x10x00x00x00x03x00x00x00x00x00x00x00 > vsock_log", &guest_ip, DEFAULT_SSH_RETRIES, DEFAULT_SSH_TIMEOUT).unwrap(); 834 }); 835 836 // Make sure socat is listening, which might take a few second on slow systems 837 thread::sleep(std::time::Duration::new(10, 0)); 838 839 // Write something to vsock from the host 840 assert!(exec_host_command_status(&format!( 841 "echo -e \"CONNECT 16\\nHelloWorld!\" | socat - UNIX-CONNECT:{}", 842 socket 843 )) 844 .success()); 845 846 // Wait for the thread to terminate. 847 listen_socat.join().unwrap(); 848 849 assert_eq!( 850 self.ssh_command("cat vsock_log").unwrap().trim(), 851 "HelloWorld!" 852 ); 853 } 854 855 fn check_nvidia_gpu(&self) { 856 // Run CUDA sample to validate it can find the device 857 let device_query_result = self 858 .ssh_command( 859 "sudo /root/NVIDIA_CUDA-11.3_Samples/bin/x86_64/linux/release/deviceQuery", 860 ) 861 .unwrap(); 862 assert!(device_query_result.contains("Detected 1 CUDA Capable device")); 863 assert!(device_query_result.contains("Device 0: \"NVIDIA Tesla T4\"")); 864 assert!(device_query_result.contains("Result = PASS")); 865 866 // Run NVIDIA DCGM Diagnostics to validate the device is functional 867 self.ssh_command("sudo nv-hostengine").unwrap(); 868 869 assert!(self 870 .ssh_command("sudo dcgmi discovery -l") 871 .unwrap() 872 .contains("Name: NVIDIA Tesla T4")); 873 assert_eq!( 874 self.ssh_command("sudo dcgmi diag -r 'diagnostic' | grep Pass | wc -l") 875 .unwrap() 876 .trim(), 877 "10" 878 ); 879 } 880 881 fn reboot_linux(&self, current_reboot_count: u32, custom_timeout: Option<i32>) { 882 let list_boots_cmd = "sudo journalctl --list-boots | wc -l"; 883 let boot_count = self 884 .ssh_command(list_boots_cmd) 885 .unwrap() 886 .trim() 887 .parse::<u32>() 888 .unwrap_or_default(); 889 890 assert_eq!(boot_count, current_reboot_count + 1); 891 self.ssh_command("sudo reboot").unwrap(); 892 893 self.wait_vm_boot(custom_timeout).unwrap(); 894 let boot_count = self 895 .ssh_command(list_boots_cmd) 896 .unwrap() 897 .trim() 898 .parse::<u32>() 899 .unwrap_or_default(); 900 assert_eq!(boot_count, current_reboot_count + 2); 901 } 902 903 fn enable_memory_hotplug(&self) { 904 self.ssh_command( 905 "echo online | sudo tee /sys/devices/system/memory/auto_online_blocks", 906 ) 907 .unwrap(); 908 } 909 910 fn check_devices_common(&self, socket: Option<&String>, console_text: Option<&String>) { 911 // Check block devices are readable 912 self.ssh_command("sudo dd if=/dev/vda of=/dev/null bs=1M iflag=direct count=1024") 913 .unwrap(); 914 self.ssh_command("sudo dd if=/dev/vdb of=/dev/null bs=1M iflag=direct count=8") 915 .unwrap(); 916 // Check if the rng device is readable 917 self.ssh_command("sudo head -c 1000 /dev/hwrng > /dev/null") 918 .unwrap(); 919 // Check vsock 920 if let Some(socket) = socket { 921 self.check_vsock(socket.as_str()); 922 } 923 // Check if the console is usable 924 if let Some(console_text) = console_text { 925 let console_cmd = format!("echo {} | sudo tee /dev/hvc0", console_text); 926 self.ssh_command(&console_cmd).unwrap(); 927 } 928 // The net device is 'automatically' exercised through the above 'ssh' commands 929 } 930 } 931 932 struct GuestCommand<'a> { 933 command: Command, 934 guest: &'a Guest, 935 capture_output: bool, 936 } 937 938 impl<'a> GuestCommand<'a> { 939 fn new(guest: &'a Guest) -> Self { 940 Self::new_with_binary_name(guest, "cloud-hypervisor") 941 } 942 943 fn new_with_binary_name(guest: &'a Guest, binary_name: &str) -> Self { 944 Self { 945 command: Command::new(clh_command(binary_name)), 946 guest, 947 capture_output: false, 948 } 949 } 950 951 fn capture_output(&mut self) -> &mut Self { 952 self.capture_output = true; 953 self 954 } 955 956 fn spawn(&mut self) -> io::Result<Child> { 957 println!( 958 "\n\n==== Start cloud-hypervisor command-line ====\n\n\ 959 {:?}\n\ 960 \n==== End cloud-hypervisor command-line ====\n\n", 961 self.command 962 ); 963 964 if self.capture_output { 965 let child = self 966 .command 967 .arg("-v") 968 .stderr(Stdio::piped()) 969 .stdout(Stdio::piped()) 970 .spawn() 971 .unwrap(); 972 973 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 974 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 975 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 976 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 977 978 if pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE { 979 Ok(child) 980 } else { 981 Err(std::io::Error::new( 982 std::io::ErrorKind::Other, 983 "resizing pipe w/ 'fnctl' failed!", 984 )) 985 } 986 } else { 987 self.command.arg("-v").spawn() 988 } 989 } 990 991 fn args<I, S>(&mut self, args: I) -> &mut Self 992 where 993 I: IntoIterator<Item = S>, 994 S: AsRef<OsStr>, 995 { 996 self.command.args(args); 997 self 998 } 999 1000 fn default_disks(&mut self) -> &mut Self { 1001 if self.guest.disk_config.disk(DiskType::CloudInit).is_some() { 1002 self.args(&[ 1003 "--disk", 1004 format!( 1005 "path={}", 1006 self.guest 1007 .disk_config 1008 .disk(DiskType::OperatingSystem) 1009 .unwrap() 1010 ) 1011 .as_str(), 1012 format!( 1013 "path={}", 1014 self.guest.disk_config.disk(DiskType::CloudInit).unwrap() 1015 ) 1016 .as_str(), 1017 ]) 1018 } else { 1019 self.args(&[ 1020 "--disk", 1021 format!( 1022 "path={}", 1023 self.guest 1024 .disk_config 1025 .disk(DiskType::OperatingSystem) 1026 .unwrap() 1027 ) 1028 .as_str(), 1029 ]) 1030 } 1031 } 1032 1033 fn default_net(&mut self) -> &mut Self { 1034 self.args(&["--net", self.guest.default_net_string().as_str()]) 1035 } 1036 } 1037 1038 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 1039 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1040 let guest = Guest::new(Box::new(focal)); 1041 let total_vcpus = threads_per_core * cores_per_package * packages; 1042 let direct_kernel_boot_path = direct_kernel_boot_path(); 1043 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 1044 if use_fw { 1045 kernel_path = guest.fw_path.as_str(); 1046 } 1047 1048 let mut child = GuestCommand::new(&guest) 1049 .args(&[ 1050 "--cpus", 1051 &format!( 1052 "boot={},topology={}:{}:1:{}", 1053 total_vcpus, threads_per_core, cores_per_package, packages 1054 ), 1055 ]) 1056 .args(&["--memory", "size=512M"]) 1057 .args(&["--kernel", kernel_path]) 1058 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1059 .default_disks() 1060 .default_net() 1061 .capture_output() 1062 .spawn() 1063 .unwrap(); 1064 1065 let r = std::panic::catch_unwind(|| { 1066 guest.wait_vm_boot(None).unwrap(); 1067 assert_eq!( 1068 guest.get_cpu_count().unwrap_or_default(), 1069 u32::from(total_vcpus) 1070 ); 1071 assert_eq!( 1072 guest 1073 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1074 .unwrap() 1075 .trim() 1076 .parse::<u8>() 1077 .unwrap_or(0), 1078 threads_per_core 1079 ); 1080 1081 assert_eq!( 1082 guest 1083 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1084 .unwrap() 1085 .trim() 1086 .parse::<u8>() 1087 .unwrap_or(0), 1088 cores_per_package 1089 ); 1090 1091 assert_eq!( 1092 guest 1093 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1094 .unwrap() 1095 .trim() 1096 .parse::<u8>() 1097 .unwrap_or(0), 1098 packages 1099 ); 1100 }); 1101 1102 let _ = child.kill(); 1103 let output = child.wait_with_output().unwrap(); 1104 1105 handle_child_output(r, &output); 1106 } 1107 1108 #[allow(unused_variables)] 1109 fn _test_guest_numa_nodes(acpi: bool) { 1110 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1111 let guest = Guest::new(Box::new(focal)); 1112 let api_socket = temp_api_path(&guest.tmp_dir); 1113 #[cfg(target_arch = "x86_64")] 1114 let kernel_path = direct_kernel_boot_path(); 1115 #[cfg(target_arch = "aarch64")] 1116 let kernel_path = if acpi { 1117 edk2_path() 1118 } else { 1119 direct_kernel_boot_path() 1120 }; 1121 1122 let mut child = GuestCommand::new(&guest) 1123 .args(&["--cpus", "boot=6,max=12"]) 1124 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 1125 .args(&[ 1126 "--memory-zone", 1127 "id=mem0,size=1G,hotplug_size=3G", 1128 "id=mem1,size=2G,hotplug_size=3G", 1129 "id=mem2,size=3G,hotplug_size=3G", 1130 ]) 1131 .args(&[ 1132 "--numa", 1133 "guest_numa_id=0,cpus=0-2:9,distances=1@15:2@20,memory_zones=mem0", 1134 "guest_numa_id=1,cpus=3-4:6-8,distances=0@20:2@25,memory_zones=mem1", 1135 "guest_numa_id=2,cpus=5:10-11,distances=0@25:1@30,memory_zones=mem2", 1136 ]) 1137 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1138 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1139 .args(&["--api-socket", &api_socket]) 1140 .capture_output() 1141 .default_disks() 1142 .default_net() 1143 .spawn() 1144 .unwrap(); 1145 1146 let r = std::panic::catch_unwind(|| { 1147 guest.wait_vm_boot(None).unwrap(); 1148 1149 guest.check_numa_common( 1150 Some(&[960_000, 1_920_000, 2_880_000]), 1151 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 1152 Some(&["10 15 20", "20 10 25", "25 30 10"]), 1153 ); 1154 1155 // AArch64 currently does not support hotplug, and therefore we only 1156 // test hotplug-related function on x86_64 here. 1157 #[cfg(target_arch = "x86_64")] 1158 { 1159 guest.enable_memory_hotplug(); 1160 1161 // Resize every memory zone and check each associated NUMA node 1162 // has been assigned the right amount of memory. 1163 resize_zone_command(&api_socket, "mem0", "4G"); 1164 resize_zone_command(&api_socket, "mem1", "4G"); 1165 resize_zone_command(&api_socket, "mem2", "4G"); 1166 // Resize to the maximum amount of CPUs and check each NUMA 1167 // node has been assigned the right CPUs set. 1168 resize_command(&api_socket, Some(12), None, None); 1169 thread::sleep(std::time::Duration::new(5, 0)); 1170 1171 guest.check_numa_common( 1172 Some(&[3_840_000, 3_840_000, 3_840_000]), 1173 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 1174 None, 1175 ); 1176 } 1177 }); 1178 1179 let _ = child.kill(); 1180 let output = child.wait_with_output().unwrap(); 1181 1182 handle_child_output(r, &output); 1183 } 1184 1185 #[allow(unused_variables)] 1186 fn _test_power_button(acpi: bool) { 1187 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1188 let guest = Guest::new(Box::new(focal)); 1189 let mut cmd = GuestCommand::new(&guest); 1190 let api_socket = temp_api_path(&guest.tmp_dir); 1191 1192 #[cfg(target_arch = "x86_64")] 1193 let kernel_path = direct_kernel_boot_path(); 1194 #[cfg(target_arch = "aarch64")] 1195 let kernel_path = if acpi { 1196 edk2_path() 1197 } else { 1198 direct_kernel_boot_path() 1199 }; 1200 1201 cmd.args(&["--cpus", "boot=1"]) 1202 .args(&["--memory", "size=512M"]) 1203 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1204 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1205 .capture_output() 1206 .default_disks() 1207 .default_net() 1208 .args(&["--api-socket", &api_socket]); 1209 1210 let child = cmd.spawn().unwrap(); 1211 1212 let r = std::panic::catch_unwind(|| { 1213 guest.wait_vm_boot(None).unwrap(); 1214 assert!(remote_command(&api_socket, "power-button", None)); 1215 }); 1216 1217 let output = child.wait_with_output().unwrap(); 1218 assert!(output.status.success()); 1219 handle_child_output(r, &output); 1220 } 1221 1222 type PrepareNetDaemon = 1223 dyn Fn(&TempDir, &str, Option<&str>, usize, bool) -> (std::process::Command, String); 1224 1225 fn test_vhost_user_net( 1226 tap: Option<&str>, 1227 num_queues: usize, 1228 prepare_daemon: &PrepareNetDaemon, 1229 generate_host_mac: bool, 1230 client_mode_daemon: bool, 1231 ) { 1232 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1233 let guest = Guest::new(Box::new(focal)); 1234 let api_socket = temp_api_path(&guest.tmp_dir); 1235 1236 let kernel_path = direct_kernel_boot_path(); 1237 1238 let host_mac = if generate_host_mac { 1239 Some(MacAddr::local_random()) 1240 } else { 1241 None 1242 }; 1243 1244 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 1245 &guest.tmp_dir, 1246 &guest.network.host_ip, 1247 tap, 1248 num_queues, 1249 client_mode_daemon, 1250 ); 1251 1252 let net_params = format!( 1253 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={}", 1254 guest.network.guest_mac, 1255 vunet_socket_path, 1256 num_queues, 1257 if let Some(host_mac) = host_mac { 1258 format!(",host_mac={}", host_mac) 1259 } else { 1260 "".to_owned() 1261 }, 1262 if client_mode_daemon { 1263 "server" 1264 } else { 1265 "client" 1266 }, 1267 ); 1268 1269 let mut ch_command = GuestCommand::new(&guest); 1270 ch_command 1271 .args(&["--cpus", format!("boot={}", num_queues / 2).as_str()]) 1272 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1273 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1274 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1275 .default_disks() 1276 .args(&["--net", net_params.as_str()]) 1277 .args(&["--api-socket", &api_socket]) 1278 .capture_output(); 1279 1280 let mut daemon_child: std::process::Child; 1281 let mut child: std::process::Child; 1282 1283 if client_mode_daemon { 1284 child = ch_command.spawn().unwrap(); 1285 // Make sure the VMM is waiting for the backend to connect 1286 thread::sleep(std::time::Duration::new(10, 0)); 1287 daemon_child = daemon_command.spawn().unwrap(); 1288 } else { 1289 daemon_child = daemon_command.spawn().unwrap(); 1290 // Make sure the backend is waiting for the VMM to connect 1291 thread::sleep(std::time::Duration::new(10, 0)); 1292 child = ch_command.spawn().unwrap(); 1293 } 1294 1295 let r = std::panic::catch_unwind(|| { 1296 guest.wait_vm_boot(None).unwrap(); 1297 1298 if let Some(tap_name) = tap { 1299 let tap_count = 1300 exec_host_command_output(&format!("ip link | grep -c {}", tap_name)); 1301 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1302 } 1303 1304 if let Some(host_mac) = tap { 1305 let mac_count = 1306 exec_host_command_output(&format!("ip link | grep -c {}", host_mac)); 1307 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1308 } 1309 1310 // 1 network interface + default localhost ==> 2 interfaces 1311 // It's important to note that this test is fully exercising the 1312 // vhost-user-net implementation and the associated backend since 1313 // it does not define any --net network interface. That means all 1314 // the ssh communication in that test happens through the network 1315 // interface backed by vhost-user-net. 1316 assert_eq!( 1317 guest 1318 .ssh_command("ip -o link | wc -l") 1319 .unwrap() 1320 .trim() 1321 .parse::<u32>() 1322 .unwrap_or_default(), 1323 2 1324 ); 1325 1326 // The following pci devices will appear on guest with PCI-MSI 1327 // interrupt vectors assigned. 1328 // 1 virtio-console with 3 vectors: config, Rx, Tx 1329 // 1 virtio-blk with 2 vectors: config, Request 1330 // 1 virtio-blk with 2 vectors: config, Request 1331 // 1 virtio-rng with 2 vectors: config, Request 1332 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1333 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1334 // Based on the above, the total vectors should 14. 1335 #[cfg(target_arch = "x86_64")] 1336 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1337 #[cfg(target_arch = "aarch64")] 1338 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1339 assert_eq!( 1340 guest 1341 .ssh_command(grep_cmd) 1342 .unwrap() 1343 .trim() 1344 .parse::<u32>() 1345 .unwrap_or_default(), 1346 10 + (num_queues as u32) 1347 ); 1348 1349 // ACPI feature is needed. 1350 #[cfg(all(target_arch = "x86_64", feature = "acpi"))] 1351 { 1352 guest.enable_memory_hotplug(); 1353 1354 // Add RAM to the VM 1355 let desired_ram = 1024 << 20; 1356 resize_command(&api_socket, None, Some(desired_ram), None); 1357 1358 thread::sleep(std::time::Duration::new(10, 0)); 1359 1360 // Here by simply checking the size (through ssh), we validate 1361 // the connection is still working, which means vhost-user-net 1362 // keeps working after the resize. 1363 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1364 } 1365 }); 1366 1367 let _ = child.kill(); 1368 let output = child.wait_with_output().unwrap(); 1369 1370 thread::sleep(std::time::Duration::new(5, 0)); 1371 let _ = daemon_child.kill(); 1372 let _ = daemon_child.wait(); 1373 1374 handle_child_output(r, &output); 1375 } 1376 1377 type PrepareBlkDaemon = 1378 dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1379 1380 fn test_vhost_user_blk( 1381 num_queues: usize, 1382 readonly: bool, 1383 direct: bool, 1384 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1385 ) { 1386 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1387 let guest = Guest::new(Box::new(focal)); 1388 let api_socket = temp_api_path(&guest.tmp_dir); 1389 1390 let kernel_path = direct_kernel_boot_path(); 1391 1392 let (blk_params, daemon_child) = { 1393 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1394 // Start the daemon 1395 let (daemon_child, vubd_socket_path) = 1396 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1397 1398 ( 1399 format!( 1400 "vhost_user=true,socket={},num_queues={},queue_size=128", 1401 vubd_socket_path, num_queues, 1402 ), 1403 Some(daemon_child), 1404 ) 1405 }; 1406 1407 let mut child = GuestCommand::new(&guest) 1408 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 1409 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1410 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1411 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1412 .args(&[ 1413 "--disk", 1414 format!( 1415 "path={}", 1416 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1417 ) 1418 .as_str(), 1419 format!( 1420 "path={}", 1421 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1422 ) 1423 .as_str(), 1424 blk_params.as_str(), 1425 ]) 1426 .default_net() 1427 .args(&["--api-socket", &api_socket]) 1428 .capture_output() 1429 .spawn() 1430 .unwrap(); 1431 1432 let r = std::panic::catch_unwind(|| { 1433 guest.wait_vm_boot(None).unwrap(); 1434 1435 // Check both if /dev/vdc exists and if the block size is 16M. 1436 assert_eq!( 1437 guest 1438 .ssh_command("lsblk | grep vdc | grep -c 16M") 1439 .unwrap() 1440 .trim() 1441 .parse::<u32>() 1442 .unwrap_or_default(), 1443 1 1444 ); 1445 1446 // Check if this block is RO or RW. 1447 assert_eq!( 1448 guest 1449 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1450 .unwrap() 1451 .trim() 1452 .parse::<u32>() 1453 .unwrap_or_default(), 1454 readonly as u32 1455 ); 1456 1457 // Check if the number of queues in /sys/block/vdc/mq matches the 1458 // expected num_queues. 1459 assert_eq!( 1460 guest 1461 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1462 .unwrap() 1463 .trim() 1464 .parse::<u32>() 1465 .unwrap_or_default(), 1466 num_queues as u32 1467 ); 1468 1469 // Mount the device 1470 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1471 guest.ssh_command("mkdir mount_image").unwrap(); 1472 guest 1473 .ssh_command( 1474 format!( 1475 "sudo mount -o {} -t ext4 /dev/vdc mount_image/", 1476 mount_ro_rw_flag 1477 ) 1478 .as_str(), 1479 ) 1480 .unwrap(); 1481 1482 // Check the content of the block device. The file "foo" should 1483 // contain "bar". 1484 assert_eq!( 1485 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1486 "bar" 1487 ); 1488 1489 // ACPI feature is needed. 1490 #[cfg(all(target_arch = "x86_64", feature = "acpi"))] 1491 { 1492 guest.enable_memory_hotplug(); 1493 1494 // Add RAM to the VM 1495 let desired_ram = 1024 << 20; 1496 resize_command(&api_socket, None, Some(desired_ram), None); 1497 1498 thread::sleep(std::time::Duration::new(10, 0)); 1499 1500 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1501 1502 // Check again the content of the block device after the resize 1503 // has been performed. 1504 assert_eq!( 1505 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1506 "bar" 1507 ); 1508 } 1509 1510 // Unmount the device 1511 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1512 guest.ssh_command("rm -r mount_image").unwrap(); 1513 }); 1514 1515 let _ = child.kill(); 1516 let output = child.wait_with_output().unwrap(); 1517 1518 if let Some(mut daemon_child) = daemon_child { 1519 thread::sleep(std::time::Duration::new(5, 0)); 1520 let _ = daemon_child.kill(); 1521 let _ = daemon_child.wait(); 1522 } 1523 1524 handle_child_output(r, &output); 1525 } 1526 1527 fn test_boot_from_vhost_user_blk( 1528 num_queues: usize, 1529 readonly: bool, 1530 direct: bool, 1531 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1532 ) { 1533 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1534 let guest = Guest::new(Box::new(focal)); 1535 1536 let kernel_path = direct_kernel_boot_path(); 1537 1538 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1539 1540 let (blk_boot_params, daemon_child) = { 1541 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1542 // Start the daemon 1543 let (daemon_child, vubd_socket_path) = prepare_daemon( 1544 &guest.tmp_dir, 1545 disk_path.as_str(), 1546 num_queues, 1547 readonly, 1548 direct, 1549 ); 1550 1551 ( 1552 format!( 1553 "vhost_user=true,socket={},num_queues={},queue_size=128", 1554 vubd_socket_path, num_queues, 1555 ), 1556 Some(daemon_child), 1557 ) 1558 }; 1559 1560 let mut child = GuestCommand::new(&guest) 1561 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 1562 .args(&["--memory", "size=512M,shared=on"]) 1563 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1564 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1565 .args(&[ 1566 "--disk", 1567 blk_boot_params.as_str(), 1568 format!( 1569 "path={}", 1570 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1571 ) 1572 .as_str(), 1573 ]) 1574 .default_net() 1575 .capture_output() 1576 .spawn() 1577 .unwrap(); 1578 1579 let r = std::panic::catch_unwind(|| { 1580 guest.wait_vm_boot(None).unwrap(); 1581 1582 // Just check the VM booted correctly. 1583 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1584 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1585 }); 1586 let _ = child.kill(); 1587 let output = child.wait_with_output().unwrap(); 1588 1589 if let Some(mut daemon_child) = daemon_child { 1590 thread::sleep(std::time::Duration::new(5, 0)); 1591 let _ = daemon_child.kill(); 1592 let _ = daemon_child.wait(); 1593 } 1594 1595 handle_child_output(r, &output); 1596 } 1597 1598 fn test_virtio_fs( 1599 dax: bool, 1600 cache_size: Option<u64>, 1601 virtiofsd_cache: &str, 1602 prepare_daemon: &dyn Fn(&TempDir, &str, &str) -> (std::process::Child, String), 1603 hotplug: bool, 1604 ) { 1605 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1606 let guest = Guest::new(Box::new(focal)); 1607 let api_socket = temp_api_path(&guest.tmp_dir); 1608 1609 let mut workload_path = dirs::home_dir().unwrap(); 1610 workload_path.push("workloads"); 1611 1612 let mut shared_dir = workload_path; 1613 shared_dir.push("shared_dir"); 1614 1615 let kernel_path = direct_kernel_boot_path(); 1616 1617 let (dax_vmm_param, dax_mount_param) = if dax { ("on", "-o dax") } else { ("off", "") }; 1618 let cache_size_vmm_param = if let Some(cache) = cache_size { 1619 format!(",cache_size={}", cache) 1620 } else { 1621 "".to_string() 1622 }; 1623 1624 let (mut daemon_child, virtiofsd_socket_path) = prepare_daemon( 1625 &guest.tmp_dir, 1626 shared_dir.to_str().unwrap(), 1627 virtiofsd_cache, 1628 ); 1629 1630 let mut guest_command = GuestCommand::new(&guest); 1631 guest_command 1632 .args(&["--cpus", "boot=1"]) 1633 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1634 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1635 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1636 .default_disks() 1637 .default_net() 1638 .args(&["--api-socket", &api_socket]); 1639 1640 let fs_params = format!( 1641 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024,dax={}{}", 1642 virtiofsd_socket_path, dax_vmm_param, cache_size_vmm_param 1643 ); 1644 1645 if !hotplug { 1646 guest_command.args(&["--fs", fs_params.as_str()]); 1647 } 1648 1649 let mut child = guest_command.capture_output().spawn().unwrap(); 1650 1651 let r = std::panic::catch_unwind(|| { 1652 guest.wait_vm_boot(None).unwrap(); 1653 1654 if hotplug { 1655 // Add fs to the VM 1656 let (cmd_success, cmd_output) = 1657 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1658 assert!(cmd_success); 1659 assert!(String::from_utf8_lossy(&cmd_output) 1660 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1661 1662 thread::sleep(std::time::Duration::new(10, 0)); 1663 } 1664 1665 // Mount shared directory through virtio_fs filesystem 1666 let mount_cmd = format!( 1667 "mkdir -p mount_dir && \ 1668 sudo mount -t virtiofs {} myfs mount_dir/", 1669 dax_mount_param 1670 ); 1671 guest.ssh_command(&mount_cmd).unwrap(); 1672 1673 assert!(guest 1674 .valid_virtio_fs_cache_size(dax, cache_size) 1675 .unwrap_or_default()); 1676 1677 // Check file1 exists and its content is "foo" 1678 assert_eq!( 1679 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1680 "foo" 1681 ); 1682 // Check file2 does not exist 1683 guest 1684 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1685 .unwrap(); 1686 1687 // Check file3 exists and its content is "bar" 1688 assert_eq!( 1689 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1690 "bar" 1691 ); 1692 1693 // ACPI feature is needed. 1694 #[cfg(all(target_arch = "x86_64", feature = "acpi"))] 1695 { 1696 guest.enable_memory_hotplug(); 1697 1698 // Add RAM to the VM 1699 let desired_ram = 1024 << 20; 1700 resize_command(&api_socket, None, Some(desired_ram), None); 1701 1702 thread::sleep(std::time::Duration::new(30, 0)); 1703 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1704 1705 // After the resize, check again that file1 exists and its 1706 // content is "foo". 1707 assert_eq!( 1708 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1709 "foo" 1710 ); 1711 } 1712 1713 if hotplug { 1714 // Remove from VM 1715 guest.ssh_command("sudo umount mount_dir").unwrap(); 1716 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1717 } 1718 }); 1719 1720 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1721 thread::sleep(std::time::Duration::new(10, 0)); 1722 let (daemon_child, virtiofsd_socket_path) = prepare_daemon( 1723 &guest.tmp_dir, 1724 shared_dir.to_str().unwrap(), 1725 virtiofsd_cache, 1726 ); 1727 1728 let r = std::panic::catch_unwind(|| { 1729 thread::sleep(std::time::Duration::new(10, 0)); 1730 let fs_params = format!( 1731 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024,dax={}{}", 1732 virtiofsd_socket_path, dax_vmm_param, cache_size_vmm_param 1733 ); 1734 1735 // Add back and check it works 1736 let (cmd_success, cmd_output) = 1737 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1738 assert!(cmd_success); 1739 assert!(String::from_utf8_lossy(&cmd_output) 1740 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1741 thread::sleep(std::time::Duration::new(10, 0)); 1742 // Mount shared directory through virtio_fs filesystem 1743 let mount_cmd = format!( 1744 "mkdir -p mount_dir && \ 1745 sudo mount -t virtiofs {} myfs mount_dir/", 1746 dax_mount_param 1747 ); 1748 guest.ssh_command(&mount_cmd).unwrap(); 1749 // Check file1 exists and its content is "foo" 1750 assert_eq!( 1751 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1752 "foo" 1753 ); 1754 }); 1755 1756 (r, Some(daemon_child)) 1757 } else { 1758 (r, None) 1759 }; 1760 1761 let _ = child.kill(); 1762 let output = child.wait_with_output().unwrap(); 1763 1764 let _ = daemon_child.kill(); 1765 let _ = daemon_child.wait(); 1766 1767 if let Some(mut daemon_child) = hotplug_daemon_child { 1768 let _ = daemon_child.kill(); 1769 let _ = daemon_child.wait(); 1770 } 1771 1772 handle_child_output(r, &output); 1773 } 1774 1775 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1776 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1777 let guest = Guest::new(Box::new(focal)); 1778 1779 let kernel_path = direct_kernel_boot_path(); 1780 1781 let pmem_temp_file = TempFile::new().unwrap(); 1782 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1783 1784 std::process::Command::new("mkfs.ext4") 1785 .arg(pmem_temp_file.as_path()) 1786 .output() 1787 .expect("Expect creating disk image to succeed"); 1788 1789 let mut child = GuestCommand::new(&guest) 1790 .args(&["--cpus", "boot=1"]) 1791 .args(&["--memory", "size=512M"]) 1792 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1793 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1794 .default_disks() 1795 .default_net() 1796 .args(&[ 1797 "--pmem", 1798 format!( 1799 "file={}{}{}", 1800 pmem_temp_file.as_path().to_str().unwrap(), 1801 if specify_size { ",size=128M" } else { "" }, 1802 if discard_writes { 1803 ",discard_writes=on" 1804 } else { 1805 "" 1806 } 1807 ) 1808 .as_str(), 1809 ]) 1810 .capture_output() 1811 .spawn() 1812 .unwrap(); 1813 1814 let r = std::panic::catch_unwind(|| { 1815 guest.wait_vm_boot(None).unwrap(); 1816 1817 // Check for the presence of /dev/pmem0 1818 assert_eq!( 1819 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1820 "/dev/pmem0" 1821 ); 1822 1823 // Check changes persist after reboot 1824 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1825 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1826 guest 1827 .ssh_command("echo test123 | sudo tee /mnt/test") 1828 .unwrap(); 1829 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1830 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1831 1832 guest.reboot_linux(0, None); 1833 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1834 assert_eq!( 1835 guest 1836 .ssh_command("sudo cat /mnt/test || true") 1837 .unwrap() 1838 .trim(), 1839 if discard_writes { "" } else { "test123" } 1840 ); 1841 }); 1842 1843 let _ = child.kill(); 1844 let output = child.wait_with_output().unwrap(); 1845 1846 handle_child_output(r, &output); 1847 } 1848 1849 fn get_fd_count(pid: u32) -> usize { 1850 fs::read_dir(format!("/proc/{}/fd", pid)).unwrap().count() 1851 } 1852 1853 fn _test_virtio_vsock(hotplug: bool) { 1854 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1855 let guest = Guest::new(Box::new(focal)); 1856 1857 let kernel_path = direct_kernel_boot_path(); 1858 1859 let socket = temp_vsock_path(&guest.tmp_dir); 1860 let api_socket = temp_api_path(&guest.tmp_dir); 1861 1862 let mut cmd = GuestCommand::new(&guest); 1863 cmd.args(&["--api-socket", &api_socket]); 1864 cmd.args(&["--cpus", "boot=1"]); 1865 cmd.args(&["--memory", "size=512M"]); 1866 cmd.args(&["--kernel", kernel_path.to_str().unwrap()]); 1867 cmd.args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1868 cmd.default_disks(); 1869 cmd.default_net(); 1870 1871 if !hotplug { 1872 cmd.args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]); 1873 } 1874 1875 let mut child = cmd.capture_output().spawn().unwrap(); 1876 1877 let r = std::panic::catch_unwind(|| { 1878 guest.wait_vm_boot(None).unwrap(); 1879 1880 if hotplug { 1881 let (cmd_success, cmd_output) = remote_command_w_output( 1882 &api_socket, 1883 "add-vsock", 1884 Some(format!("cid=3,socket={},id=test0", socket).as_str()), 1885 ); 1886 assert!(cmd_success); 1887 assert!(String::from_utf8_lossy(&cmd_output) 1888 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1889 thread::sleep(std::time::Duration::new(10, 0)); 1890 // Check adding a second one fails 1891 assert!(!remote_command( 1892 &api_socket, 1893 "add-vsock", 1894 Some("cid=1234,socket=/tmp/fail") 1895 )); 1896 } 1897 1898 // Validate vsock works as expected. 1899 guest.check_vsock(socket.as_str()); 1900 1901 // AArch64 currently does not support reboot, and therefore we 1902 // skip the reboot test here. 1903 #[cfg(target_arch = "x86_64")] 1904 { 1905 guest.reboot_linux(0, None); 1906 1907 // Validate vsock still works after a reboot. 1908 guest.check_vsock(socket.as_str()); 1909 } 1910 if hotplug { 1911 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1912 } 1913 }); 1914 1915 let _ = child.kill(); 1916 let output = child.wait_with_output().unwrap(); 1917 1918 handle_child_output(r, &output); 1919 } 1920 1921 fn get_ksm_pages_shared() -> u32 { 1922 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1923 .unwrap() 1924 .trim() 1925 .parse::<u32>() 1926 .unwrap() 1927 } 1928 1929 fn test_memory_mergeable(mergeable: bool) { 1930 let memory_param = if mergeable { 1931 "mergeable=on" 1932 } else { 1933 "mergeable=off" 1934 }; 1935 1936 // We are assuming the rest of the system in our CI is not using mergeable memeory 1937 let ksm_ps_init = get_ksm_pages_shared(); 1938 assert!(ksm_ps_init == 0); 1939 1940 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1941 let guest1 = Guest::new(Box::new(focal1)); 1942 let mut child1 = GuestCommand::new(&guest1) 1943 .args(&["--cpus", "boot=1"]) 1944 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1945 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1946 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1947 .default_disks() 1948 .args(&["--net", guest1.default_net_string().as_str()]) 1949 .args(&["--serial", "tty", "--console", "off"]) 1950 .capture_output() 1951 .spawn() 1952 .unwrap(); 1953 1954 let r = std::panic::catch_unwind(|| { 1955 guest1.wait_vm_boot(None).unwrap(); 1956 }); 1957 if r.is_err() { 1958 let _ = child1.kill(); 1959 let output = child1.wait_with_output().unwrap(); 1960 handle_child_output(r, &output); 1961 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1962 } 1963 1964 let ksm_ps_guest1 = get_ksm_pages_shared(); 1965 1966 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1967 let guest2 = Guest::new(Box::new(focal2)); 1968 let mut child2 = GuestCommand::new(&guest2) 1969 .args(&["--cpus", "boot=1"]) 1970 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1971 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1972 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1973 .default_disks() 1974 .args(&["--net", guest2.default_net_string().as_str()]) 1975 .args(&["--serial", "tty", "--console", "off"]) 1976 .capture_output() 1977 .spawn() 1978 .unwrap(); 1979 1980 let r = std::panic::catch_unwind(|| { 1981 guest2.wait_vm_boot(None).unwrap(); 1982 let ksm_ps_guest2 = get_ksm_pages_shared(); 1983 1984 if mergeable { 1985 println!( 1986 "ksm pages_shared after vm1 booted '{}', ksm pages_shared after vm2 booted '{}'", 1987 ksm_ps_guest1, ksm_ps_guest2 1988 ); 1989 // We are expecting the number of shared pages to increase as the number of VM increases 1990 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1991 } else { 1992 assert!(ksm_ps_guest1 == 0); 1993 assert!(ksm_ps_guest2 == 0); 1994 } 1995 }); 1996 1997 let _ = child1.kill(); 1998 let _ = child2.kill(); 1999 2000 let output = child1.wait_with_output().unwrap(); 2001 child2.wait().unwrap(); 2002 2003 handle_child_output(r, &output); 2004 } 2005 2006 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 2007 let smaps = fs::File::open(format!("/proc/{}/smaps", pid)).unwrap(); 2008 let reader = io::BufReader::new(smaps); 2009 2010 let mut skip_map: bool = false; 2011 let mut region_name: String = "".to_string(); 2012 let mut region_maps = HashMap::new(); 2013 for line in reader.lines() { 2014 let l = line.unwrap(); 2015 2016 if l.contains('-') { 2017 let values: Vec<&str> = l.split_whitespace().collect(); 2018 region_name = values.last().unwrap().trim().to_string(); 2019 if region_name == "0" { 2020 region_name = "anonymous".to_string() 2021 } 2022 } 2023 2024 // Each section begins with something that looks like: 2025 // Size: 2184 kB 2026 if l.starts_with("Size:") { 2027 let values: Vec<&str> = l.split_whitespace().collect(); 2028 let map_size = values[1].parse::<u32>().unwrap(); 2029 // We skip the assigned guest RAM map, its RSS is only 2030 // dependent on the guest actual memory usage. 2031 // Everything else can be added to the VMM overhead. 2032 skip_map = map_size >= guest_memory_size; 2033 continue; 2034 } 2035 2036 // If this is a map we're taking into account, then we only 2037 // count the RSS. The sum of all counted RSS is the VMM overhead. 2038 if !skip_map && l.starts_with("Rss:") { 2039 let values: Vec<&str> = l.split_whitespace().collect(); 2040 let value = values[1].trim().parse::<u32>().unwrap(); 2041 *region_maps.entry(region_name.clone()).or_insert(0) += value; 2042 } 2043 } 2044 2045 region_maps 2046 } 2047 2048 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 2049 let mut total = 0; 2050 2051 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 2052 eprintln!("{}: {}", region_name, value); 2053 total += value; 2054 } 2055 2056 total 2057 } 2058 2059 // 10MB is our maximum accepted overhead. 2060 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 2061 2062 #[derive(PartialEq, PartialOrd)] 2063 struct Counters { 2064 rx_bytes: u64, 2065 rx_frames: u64, 2066 tx_bytes: u64, 2067 tx_frames: u64, 2068 read_bytes: u64, 2069 write_bytes: u64, 2070 read_ops: u64, 2071 write_ops: u64, 2072 } 2073 2074 fn get_counters(api_socket: &str) -> Counters { 2075 // Get counters 2076 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 2077 assert!(cmd_success); 2078 2079 let counters: HashMap<&str, HashMap<&str, u64>> = 2080 serde_json::from_slice(&cmd_output).unwrap_or_default(); 2081 2082 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 2083 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 2084 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 2085 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 2086 2087 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 2088 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 2089 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 2090 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 2091 2092 Counters { 2093 rx_bytes, 2094 rx_frames, 2095 tx_bytes, 2096 tx_frames, 2097 read_bytes, 2098 write_bytes, 2099 read_ops, 2100 write_ops, 2101 } 2102 } 2103 2104 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 2105 let (tx, rx) = mpsc::channel::<String>(); 2106 thread::spawn(move || loop { 2107 thread::sleep(std::time::Duration::new(1, 0)); 2108 let mut buf = [0; 512]; 2109 match pty.read(&mut buf) { 2110 Ok(_) => { 2111 let output = std::str::from_utf8(&buf).unwrap().to_string(); 2112 match tx.send(output) { 2113 Ok(_) => (), 2114 Err(_) => break, 2115 } 2116 } 2117 Err(_) => break, 2118 } 2119 }); 2120 rx 2121 } 2122 2123 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 2124 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2125 assert!(cmd_success); 2126 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2127 assert_eq!("Pty", info["config"][pty_type]["mode"]); 2128 PathBuf::from( 2129 info["config"][pty_type]["file"] 2130 .as_str() 2131 .expect("Missing pty path"), 2132 ) 2133 } 2134 2135 // VFIO test network setup. 2136 // We reserve a different IP class for it: 172.18.0.0/24. 2137 fn setup_vfio_network_interfaces() { 2138 // 'vfio-br0' 2139 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 2140 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 2141 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 2142 // 'vfio-tap0' 2143 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 2144 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 2145 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 2146 // 'vfio-tap1' 2147 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 2148 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 2149 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 2150 // 'vfio-tap2' 2151 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 2152 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 2153 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 2154 // 'vfio-tap3' 2155 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 2156 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 2157 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 2158 } 2159 2160 // Tear VFIO test network down 2161 fn cleanup_vfio_network_interfaces() { 2162 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 2163 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 2164 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 2165 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 2166 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 2167 } 2168 2169 fn balloon_size(api_socket: &str) -> u64 { 2170 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2171 assert!(cmd_success); 2172 2173 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2174 let total_mem = &info["config"]["memory"]["size"] 2175 .to_string() 2176 .parse::<u64>() 2177 .unwrap(); 2178 let actual_mem = &info["memory_actual_size"] 2179 .to_string() 2180 .parse::<u64>() 2181 .unwrap(); 2182 total_mem - actual_mem 2183 } 2184 2185 mod parallel { 2186 use std::io::SeekFrom; 2187 2188 use crate::tests::*; 2189 2190 #[test] 2191 #[cfg(target_arch = "x86_64")] 2192 fn test_simple_launch() { 2193 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 2194 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2195 2196 vec![Box::new(bionic), Box::new(focal)] 2197 .drain(..) 2198 .for_each(|disk_config| { 2199 let guest = Guest::new(disk_config); 2200 2201 let mut child = GuestCommand::new(&guest) 2202 .args(&["--cpus", "boot=1"]) 2203 .args(&["--memory", "size=512M"]) 2204 .args(&["--kernel", guest.fw_path.as_str()]) 2205 .default_disks() 2206 .default_net() 2207 .args(&["--serial", "tty", "--console", "off"]) 2208 .capture_output() 2209 .spawn() 2210 .unwrap(); 2211 2212 let r = std::panic::catch_unwind(|| { 2213 guest.wait_vm_boot(Some(120)).unwrap(); 2214 2215 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2216 assert_eq!(guest.get_initial_apicid().unwrap_or(1), 0); 2217 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2218 assert!(guest.get_entropy().unwrap_or_default() >= 900); 2219 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2220 }); 2221 2222 let _ = child.kill(); 2223 let output = child.wait_with_output().unwrap(); 2224 2225 handle_child_output(r, &output); 2226 }); 2227 } 2228 2229 #[test] 2230 fn test_multi_cpu() { 2231 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 2232 let guest = Guest::new(Box::new(bionic)); 2233 let mut cmd = GuestCommand::new(&guest); 2234 cmd.args(&["--cpus", "boot=2,max=4"]) 2235 .args(&["--memory", "size=512M"]) 2236 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2237 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2238 .capture_output() 2239 .default_disks() 2240 .default_net(); 2241 2242 let mut child = cmd.spawn().unwrap(); 2243 2244 let r = std::panic::catch_unwind(|| { 2245 guest.wait_vm_boot(Some(120)).unwrap(); 2246 2247 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2248 2249 #[cfg(target_arch = "x86_64")] 2250 assert_eq!( 2251 guest 2252 .ssh_command( 2253 r#"dmesg | grep "smpboot: Allowing" | sed "s/\[\ *[0-9.]*\] //""# 2254 ) 2255 .unwrap() 2256 .trim(), 2257 "smpboot: Allowing 4 CPUs, 2 hotplug CPUs" 2258 ); 2259 #[cfg(target_arch = "aarch64")] 2260 assert_eq!( 2261 guest 2262 .ssh_command( 2263 r#"dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2264 ) 2265 .unwrap() 2266 .trim(), 2267 "smp: Brought up 1 node, 2 CPUs" 2268 ); 2269 }); 2270 2271 let _ = child.kill(); 2272 let output = child.wait_with_output().unwrap(); 2273 2274 handle_child_output(r, &output); 2275 } 2276 2277 #[test] 2278 fn test_cpu_topology_421() { 2279 test_cpu_topology(4, 2, 1, false); 2280 } 2281 2282 #[test] 2283 fn test_cpu_topology_142() { 2284 test_cpu_topology(1, 4, 2, false); 2285 } 2286 2287 #[test] 2288 fn test_cpu_topology_262() { 2289 test_cpu_topology(2, 6, 2, false); 2290 } 2291 2292 #[test] 2293 #[cfg(target_arch = "x86_64")] 2294 fn test_cpu_physical_bits() { 2295 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2296 let guest = Guest::new(Box::new(focal)); 2297 let max_phys_bits: u8 = 36; 2298 let mut child = GuestCommand::new(&guest) 2299 .args(&["--cpus", &format!("max_phys_bits={}", max_phys_bits)]) 2300 .args(&["--memory", "size=512M"]) 2301 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2302 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2303 .default_disks() 2304 .default_net() 2305 .capture_output() 2306 .spawn() 2307 .unwrap(); 2308 2309 let r = std::panic::catch_unwind(|| { 2310 guest.wait_vm_boot(None).unwrap(); 2311 2312 assert!( 2313 guest 2314 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2315 .unwrap() 2316 .trim() 2317 .parse::<u8>() 2318 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2319 ); 2320 }); 2321 2322 let _ = child.kill(); 2323 let output = child.wait_with_output().unwrap(); 2324 2325 handle_child_output(r, &output); 2326 } 2327 2328 #[test] 2329 fn test_large_vm() { 2330 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2331 let guest = Guest::new(Box::new(focal)); 2332 let mut cmd = GuestCommand::new(&guest); 2333 cmd.args(&["--cpus", "boot=48"]) 2334 .args(&["--memory", "size=5120M"]) 2335 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2336 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2337 .args(&["--serial", "tty"]) 2338 .args(&["--console", "off"]) 2339 .capture_output() 2340 .default_disks() 2341 .default_net(); 2342 2343 let mut child = cmd.spawn().unwrap(); 2344 2345 guest.wait_vm_boot(None).unwrap(); 2346 2347 let r = std::panic::catch_unwind(|| { 2348 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2349 assert_eq!( 2350 guest 2351 .ssh_command( 2352 "lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"" 2353 ) 2354 .unwrap() 2355 .trim(), 2356 "0-47" 2357 ); 2358 2359 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2360 }); 2361 2362 let _ = child.kill(); 2363 let output = child.wait_with_output().unwrap(); 2364 2365 handle_child_output(r, &output); 2366 } 2367 2368 #[test] 2369 fn test_huge_memory() { 2370 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2371 let guest = Guest::new(Box::new(focal)); 2372 let mut cmd = GuestCommand::new(&guest); 2373 cmd.args(&["--cpus", "boot=1"]) 2374 .args(&["--memory", "size=128G"]) 2375 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2376 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2377 .capture_output() 2378 .default_disks() 2379 .default_net(); 2380 2381 let mut child = cmd.spawn().unwrap(); 2382 2383 guest.wait_vm_boot(Some(120)).unwrap(); 2384 2385 let r = std::panic::catch_unwind(|| { 2386 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2387 }); 2388 2389 let _ = child.kill(); 2390 let output = child.wait_with_output().unwrap(); 2391 2392 handle_child_output(r, &output); 2393 } 2394 2395 #[test] 2396 fn test_power_button() { 2397 _test_power_button(false); 2398 } 2399 2400 #[test] 2401 fn test_user_defined_memory_regions() { 2402 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2403 let guest = Guest::new(Box::new(focal)); 2404 let api_socket = temp_api_path(&guest.tmp_dir); 2405 2406 let kernel_path = direct_kernel_boot_path(); 2407 2408 let mut child = GuestCommand::new(&guest) 2409 .args(&["--cpus", "boot=1"]) 2410 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 2411 .args(&[ 2412 "--memory-zone", 2413 "id=mem0,size=1G,hotplug_size=2G", 2414 "id=mem1,size=1G,file=/dev/shm", 2415 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2416 ]) 2417 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2418 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2419 .args(&["--api-socket", &api_socket]) 2420 .capture_output() 2421 .default_disks() 2422 .default_net() 2423 .spawn() 2424 .unwrap(); 2425 2426 let r = std::panic::catch_unwind(|| { 2427 guest.wait_vm_boot(None).unwrap(); 2428 2429 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2430 2431 guest.enable_memory_hotplug(); 2432 2433 resize_zone_command(&api_socket, "mem0", "3G"); 2434 thread::sleep(std::time::Duration::new(5, 0)); 2435 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2436 resize_zone_command(&api_socket, "mem2", "3G"); 2437 thread::sleep(std::time::Duration::new(5, 0)); 2438 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2439 resize_zone_command(&api_socket, "mem0", "2G"); 2440 thread::sleep(std::time::Duration::new(5, 0)); 2441 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2442 resize_zone_command(&api_socket, "mem2", "2G"); 2443 thread::sleep(std::time::Duration::new(5, 0)); 2444 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2445 2446 guest.reboot_linux(0, None); 2447 2448 // Check the amount of RAM after reboot 2449 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2450 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2451 2452 // Check if we can still resize down to the initial 'boot'size 2453 resize_zone_command(&api_socket, "mem0", "1G"); 2454 thread::sleep(std::time::Duration::new(5, 0)); 2455 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2456 resize_zone_command(&api_socket, "mem2", "1G"); 2457 thread::sleep(std::time::Duration::new(5, 0)); 2458 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2459 }); 2460 2461 let _ = child.kill(); 2462 let output = child.wait_with_output().unwrap(); 2463 2464 handle_child_output(r, &output); 2465 } 2466 2467 #[test] 2468 fn test_guest_numa_nodes() { 2469 _test_guest_numa_nodes(false); 2470 } 2471 2472 #[test] 2473 fn test_pci_msi() { 2474 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2475 let guest = Guest::new(Box::new(focal)); 2476 let mut cmd = GuestCommand::new(&guest); 2477 cmd.args(&["--cpus", "boot=1"]) 2478 .args(&["--memory", "size=512M"]) 2479 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2480 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2481 .capture_output() 2482 .default_disks() 2483 .default_net(); 2484 2485 let mut child = cmd.spawn().unwrap(); 2486 2487 guest.wait_vm_boot(None).unwrap(); 2488 2489 #[cfg(target_arch = "x86_64")] 2490 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2491 #[cfg(target_arch = "aarch64")] 2492 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2493 2494 let r = std::panic::catch_unwind(|| { 2495 assert_eq!( 2496 guest 2497 .ssh_command(grep_cmd) 2498 .unwrap() 2499 .trim() 2500 .parse::<u32>() 2501 .unwrap_or_default(), 2502 12 2503 ); 2504 }); 2505 2506 let _ = child.kill(); 2507 let output = child.wait_with_output().unwrap(); 2508 2509 handle_child_output(r, &output); 2510 } 2511 2512 #[test] 2513 fn test_direct_kernel_boot() { 2514 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2515 let guest = Guest::new(Box::new(focal)); 2516 2517 let kernel_path = direct_kernel_boot_path(); 2518 2519 let mut child = GuestCommand::new(&guest) 2520 .args(&["--cpus", "boot=1"]) 2521 .args(&["--memory", "size=512M"]) 2522 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2523 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2524 .default_disks() 2525 .default_net() 2526 .capture_output() 2527 .spawn() 2528 .unwrap(); 2529 2530 let r = std::panic::catch_unwind(|| { 2531 guest.wait_vm_boot(None).unwrap(); 2532 2533 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2534 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2535 assert!(guest.get_entropy().unwrap_or_default() >= 900); 2536 2537 let grep_cmd = if cfg!(target_arch = "x86_64") { 2538 "grep -c PCI-MSI /proc/interrupts" 2539 } else { 2540 "grep -c ITS-MSI /proc/interrupts" 2541 }; 2542 assert_eq!( 2543 guest 2544 .ssh_command(grep_cmd) 2545 .unwrap() 2546 .trim() 2547 .parse::<u32>() 2548 .unwrap_or_default(), 2549 12 2550 ); 2551 }); 2552 2553 let _ = child.kill(); 2554 let output = child.wait_with_output().unwrap(); 2555 2556 handle_child_output(r, &output); 2557 } 2558 2559 fn _test_virtio_block(image_name: &str, disable_io_uring: bool) { 2560 let focal = UbuntuDiskConfig::new(image_name.to_string()); 2561 let guest = Guest::new(Box::new(focal)); 2562 2563 let mut workload_path = dirs::home_dir().unwrap(); 2564 workload_path.push("workloads"); 2565 2566 let mut blk_file_path = workload_path; 2567 blk_file_path.push("blk.img"); 2568 2569 let kernel_path = direct_kernel_boot_path(); 2570 2571 let mut cloud_child = GuestCommand::new(&guest) 2572 .args(&["--cpus", "boot=4"]) 2573 .args(&["--memory", "size=512M,shared=on"]) 2574 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2575 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2576 .args(&[ 2577 "--disk", 2578 format!( 2579 "path={}", 2580 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2581 ) 2582 .as_str(), 2583 format!( 2584 "path={}", 2585 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2586 ) 2587 .as_str(), 2588 format!( 2589 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={}", 2590 blk_file_path.to_str().unwrap(), 2591 disable_io_uring 2592 ) 2593 .as_str(), 2594 ]) 2595 .default_net() 2596 .capture_output() 2597 .spawn() 2598 .unwrap(); 2599 2600 let r = std::panic::catch_unwind(|| { 2601 guest.wait_vm_boot(None).unwrap(); 2602 2603 // Check both if /dev/vdc exists and if the block size is 16M. 2604 assert_eq!( 2605 guest 2606 .ssh_command("lsblk | grep vdc | grep -c 16M") 2607 .unwrap() 2608 .trim() 2609 .parse::<u32>() 2610 .unwrap_or_default(), 2611 1 2612 ); 2613 2614 // Check both if /dev/vdc exists and if this block is RO. 2615 assert_eq!( 2616 guest 2617 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 2618 .unwrap() 2619 .trim() 2620 .parse::<u32>() 2621 .unwrap_or_default(), 2622 1 2623 ); 2624 2625 // Check if the number of queues is 4. 2626 assert_eq!( 2627 guest 2628 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 2629 .unwrap() 2630 .trim() 2631 .parse::<u32>() 2632 .unwrap_or_default(), 2633 4 2634 ); 2635 }); 2636 2637 let _ = cloud_child.kill(); 2638 let output = cloud_child.wait_with_output().unwrap(); 2639 2640 handle_child_output(r, &output); 2641 } 2642 2643 #[test] 2644 fn test_virtio_block() { 2645 _test_virtio_block(FOCAL_IMAGE_NAME, false) 2646 } 2647 2648 #[test] 2649 fn test_virtio_block_disable_io_uring() { 2650 _test_virtio_block(FOCAL_IMAGE_NAME, true) 2651 } 2652 2653 #[test] 2654 fn test_virtio_block_qcow2() { 2655 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false) 2656 } 2657 2658 #[test] 2659 fn test_virtio_block_vhd() { 2660 let mut workload_path = dirs::home_dir().unwrap(); 2661 workload_path.push("workloads"); 2662 2663 let mut raw_file_path = workload_path.clone(); 2664 let mut vhd_file_path = workload_path; 2665 raw_file_path.push(FOCAL_IMAGE_NAME); 2666 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 2667 2668 // Generate VHD file from RAW file 2669 std::process::Command::new("qemu-img") 2670 .arg("convert") 2671 .arg("-p") 2672 .args(&["-f", "raw"]) 2673 .args(&["-O", "vpc"]) 2674 .args(&["-o", "subformat=fixed"]) 2675 .arg(raw_file_path.to_str().unwrap()) 2676 .arg(vhd_file_path.to_str().unwrap()) 2677 .output() 2678 .expect("Expect generating VHD image from RAW image"); 2679 2680 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false) 2681 } 2682 2683 #[test] 2684 fn test_virtio_block_vhdx() { 2685 let mut workload_path = dirs::home_dir().unwrap(); 2686 workload_path.push("workloads"); 2687 2688 let mut raw_file_path = workload_path.clone(); 2689 let mut vhdx_file_path = workload_path; 2690 raw_file_path.push(FOCAL_IMAGE_NAME); 2691 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 2692 2693 // Generate dynamic VHDX file from RAW file 2694 std::process::Command::new("qemu-img") 2695 .arg("convert") 2696 .arg("-p") 2697 .args(&["-f", "raw"]) 2698 .args(&["-O", "vhdx"]) 2699 .arg(raw_file_path.to_str().unwrap()) 2700 .arg(vhdx_file_path.to_str().unwrap()) 2701 .output() 2702 .expect("Expect generating dynamic VHDx image from RAW image"); 2703 2704 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false) 2705 } 2706 2707 #[test] 2708 fn test_virtio_block_dynamic_vhdx_expand() { 2709 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 2710 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 2711 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 2712 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 2713 2714 let mut workload_path = dirs::home_dir().unwrap(); 2715 workload_path.push("workloads"); 2716 2717 let mut vhdx_file_path = workload_path; 2718 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 2719 let vhdx_path = vhdx_file_path.to_str().unwrap(); 2720 2721 // Generate a 100 MiB dynamic VHDX file 2722 std::process::Command::new("qemu-img") 2723 .arg("create") 2724 .args(&["-f", "vhdx"]) 2725 .arg(vhdx_path) 2726 .arg(VIRTUAL_DISK_SIZE.to_string()) 2727 .output() 2728 .expect("Expect generating dynamic VHDx image from RAW image"); 2729 2730 // Check if the size matches with empty VHDx file size 2731 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 2732 2733 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2734 let guest = Guest::new(Box::new(focal)); 2735 let kernel_path = direct_kernel_boot_path(); 2736 2737 let mut cloud_child = GuestCommand::new(&guest) 2738 .args(&["--cpus", "boot=1"]) 2739 .args(&["--memory", "size=512M"]) 2740 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2741 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2742 .args(&[ 2743 "--disk", 2744 format!( 2745 "path={}", 2746 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2747 ) 2748 .as_str(), 2749 format!( 2750 "path={}", 2751 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2752 ) 2753 .as_str(), 2754 format!("path={}", vhdx_path).as_str(), 2755 ]) 2756 .default_net() 2757 .capture_output() 2758 .spawn() 2759 .unwrap(); 2760 2761 let r = std::panic::catch_unwind(|| { 2762 guest.wait_vm_boot(None).unwrap(); 2763 2764 // Check both if /dev/vdc exists and if the block size is 100 MiB. 2765 assert_eq!( 2766 guest 2767 .ssh_command("lsblk | grep vdc | grep -c 100M") 2768 .unwrap() 2769 .trim() 2770 .parse::<u32>() 2771 .unwrap_or_default(), 2772 1 2773 ); 2774 2775 // Write 100 MB of data to the VHDx disk 2776 guest 2777 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 2778 .unwrap(); 2779 }); 2780 2781 // Check if the size matches with expected expanded VHDx file size 2782 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 2783 2784 let _ = cloud_child.kill(); 2785 let output = cloud_child.wait_with_output().unwrap(); 2786 2787 handle_child_output(r, &output); 2788 } 2789 2790 fn vhdx_image_size(disk_name: &str) -> u64 { 2791 std::fs::File::open(disk_name) 2792 .unwrap() 2793 .seek(SeekFrom::End(0)) 2794 .unwrap() 2795 } 2796 2797 #[test] 2798 fn test_vhost_user_net_default() { 2799 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 2800 } 2801 2802 #[test] 2803 fn test_vhost_user_net_named_tap() { 2804 test_vhost_user_net( 2805 Some("mytap0"), 2806 2, 2807 &prepare_vhost_user_net_daemon, 2808 false, 2809 false, 2810 ) 2811 } 2812 2813 #[test] 2814 fn test_vhost_user_net_existing_tap() { 2815 test_vhost_user_net( 2816 Some("vunet-tap0"), 2817 2, 2818 &prepare_vhost_user_net_daemon, 2819 false, 2820 false, 2821 ) 2822 } 2823 2824 #[test] 2825 fn test_vhost_user_net_multiple_queues() { 2826 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 2827 } 2828 2829 #[test] 2830 fn test_vhost_user_net_tap_multiple_queues() { 2831 test_vhost_user_net( 2832 Some("vunet-tap1"), 2833 4, 2834 &prepare_vhost_user_net_daemon, 2835 false, 2836 false, 2837 ) 2838 } 2839 2840 #[test] 2841 fn test_vhost_user_net_host_mac() { 2842 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 2843 } 2844 2845 #[test] 2846 fn test_vhost_user_net_client_mode() { 2847 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 2848 } 2849 2850 #[test] 2851 fn test_vhost_user_blk_default() { 2852 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 2853 } 2854 2855 #[test] 2856 fn test_vhost_user_blk_readonly() { 2857 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 2858 } 2859 2860 #[test] 2861 fn test_vhost_user_blk_direct() { 2862 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 2863 } 2864 2865 #[test] 2866 fn test_boot_from_vhost_user_blk_default() { 2867 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 2868 } 2869 2870 #[test] 2871 #[cfg(target_arch = "x86_64")] 2872 fn test_split_irqchip() { 2873 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2874 let guest = Guest::new(Box::new(focal)); 2875 2876 let mut child = GuestCommand::new(&guest) 2877 .args(&["--cpus", "boot=1"]) 2878 .args(&["--memory", "size=512M"]) 2879 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2880 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2881 .default_disks() 2882 .default_net() 2883 .capture_output() 2884 .spawn() 2885 .unwrap(); 2886 2887 let r = std::panic::catch_unwind(|| { 2888 guest.wait_vm_boot(None).unwrap(); 2889 2890 assert_eq!( 2891 guest 2892 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 2893 .unwrap() 2894 .trim() 2895 .parse::<u32>() 2896 .unwrap_or(1), 2897 0 2898 ); 2899 assert_eq!( 2900 guest 2901 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 2902 .unwrap() 2903 .trim() 2904 .parse::<u32>() 2905 .unwrap_or(1), 2906 0 2907 ); 2908 }); 2909 2910 let _ = child.kill(); 2911 let output = child.wait_with_output().unwrap(); 2912 2913 handle_child_output(r, &output); 2914 } 2915 2916 #[test] 2917 fn test_virtio_fs_dax_on_default_cache_size() { 2918 test_virtio_fs(true, None, "none", &prepare_virtiofsd, false) 2919 } 2920 2921 #[test] 2922 fn test_virtio_fs_dax_on_cache_size_1_gib() { 2923 test_virtio_fs(true, Some(0x4000_0000), "none", &prepare_virtiofsd, false) 2924 } 2925 2926 #[test] 2927 fn test_virtio_fs_dax_off() { 2928 test_virtio_fs(false, None, "none", &prepare_virtiofsd, false) 2929 } 2930 2931 #[test] 2932 fn test_virtio_fs_dax_on_default_cache_size_w_virtiofsd_rs_daemon() { 2933 test_virtio_fs(true, None, "none", &prepare_virtofsd_rs_daemon, false) 2934 } 2935 2936 #[test] 2937 fn test_virtio_fs_dax_on_cache_size_1_gib_w_virtiofsd_rs_daemon() { 2938 test_virtio_fs( 2939 true, 2940 Some(0x4000_0000), 2941 "none", 2942 &prepare_virtofsd_rs_daemon, 2943 false, 2944 ) 2945 } 2946 2947 #[test] 2948 fn test_virtio_fs_dax_off_w_virtiofsd_rs_daemon() { 2949 test_virtio_fs(false, None, "none", &prepare_virtofsd_rs_daemon, false) 2950 } 2951 2952 #[test] 2953 #[cfg(target_arch = "x86_64")] 2954 fn test_virtio_fs_hotplug_dax_on() { 2955 test_virtio_fs(true, None, "none", &prepare_virtiofsd, true) 2956 } 2957 2958 #[test] 2959 #[cfg(target_arch = "x86_64")] 2960 fn test_virtio_fs_hotplug_dax_off() { 2961 test_virtio_fs(false, None, "none", &prepare_virtiofsd, true) 2962 } 2963 2964 #[test] 2965 #[cfg(target_arch = "x86_64")] 2966 fn test_virtio_fs_hotplug_dax_on_w_virtiofsd_rs_daemon() { 2967 test_virtio_fs(true, None, "none", &prepare_virtofsd_rs_daemon, true) 2968 } 2969 2970 #[test] 2971 #[cfg(target_arch = "x86_64")] 2972 fn test_virtio_fs_hotplug_dax_off_w_virtiofsd_rs_daemon() { 2973 test_virtio_fs(false, None, "none", &prepare_virtofsd_rs_daemon, true) 2974 } 2975 2976 #[test] 2977 fn test_virtio_pmem_persist_writes() { 2978 test_virtio_pmem(false, false) 2979 } 2980 2981 #[test] 2982 fn test_virtio_pmem_discard_writes() { 2983 test_virtio_pmem(true, false) 2984 } 2985 2986 #[test] 2987 fn test_virtio_pmem_with_size() { 2988 test_virtio_pmem(true, true) 2989 } 2990 2991 #[test] 2992 fn test_boot_from_virtio_pmem() { 2993 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2994 let guest = Guest::new(Box::new(focal)); 2995 2996 let kernel_path = direct_kernel_boot_path(); 2997 2998 let mut child = GuestCommand::new(&guest) 2999 .args(&["--cpus", "boot=1"]) 3000 .args(&["--memory", "size=512M"]) 3001 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3002 .args(&[ 3003 "--disk", 3004 format!( 3005 "path={}", 3006 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3007 ) 3008 .as_str(), 3009 ]) 3010 .default_net() 3011 .args(&[ 3012 "--pmem", 3013 format!( 3014 "file={},size={}", 3015 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3016 fs::metadata(&guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3017 .unwrap() 3018 .len() 3019 ) 3020 .as_str(), 3021 ]) 3022 .args(&[ 3023 "--cmdline", 3024 DIRECT_KERNEL_BOOT_CMDLINE 3025 .replace("vda1", "pmem0p1") 3026 .as_str(), 3027 ]) 3028 .capture_output() 3029 .spawn() 3030 .unwrap(); 3031 3032 let r = std::panic::catch_unwind(|| { 3033 guest.wait_vm_boot(None).unwrap(); 3034 3035 // Simple checks to validate the VM booted properly 3036 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3037 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3038 }); 3039 3040 let _ = child.kill(); 3041 let output = child.wait_with_output().unwrap(); 3042 3043 handle_child_output(r, &output); 3044 } 3045 3046 #[test] 3047 fn test_multiple_network_interfaces() { 3048 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3049 let guest = Guest::new(Box::new(focal)); 3050 3051 let kernel_path = direct_kernel_boot_path(); 3052 3053 let mut child = GuestCommand::new(&guest) 3054 .args(&["--cpus", "boot=1"]) 3055 .args(&["--memory", "size=512M"]) 3056 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3057 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3058 .default_disks() 3059 .args(&[ 3060 "--net", 3061 guest.default_net_string().as_str(), 3062 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3063 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3064 ]) 3065 .capture_output() 3066 .spawn() 3067 .unwrap(); 3068 3069 let r = std::panic::catch_unwind(|| { 3070 guest.wait_vm_boot(None).unwrap(); 3071 3072 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3073 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3074 3075 // 3 network interfaces + default localhost ==> 4 interfaces 3076 assert_eq!( 3077 guest 3078 .ssh_command("ip -o link | wc -l") 3079 .unwrap() 3080 .trim() 3081 .parse::<u32>() 3082 .unwrap_or_default(), 3083 4 3084 ); 3085 }); 3086 3087 let _ = child.kill(); 3088 let output = child.wait_with_output().unwrap(); 3089 3090 handle_child_output(r, &output); 3091 } 3092 3093 #[test] 3094 fn test_serial_off() { 3095 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3096 let guest = Guest::new(Box::new(focal)); 3097 let mut child = GuestCommand::new(&guest) 3098 .args(&["--cpus", "boot=1"]) 3099 .args(&["--memory", "size=512M"]) 3100 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3101 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3102 .default_disks() 3103 .default_net() 3104 .args(&["--serial", "off"]) 3105 .capture_output() 3106 .spawn() 3107 .unwrap(); 3108 3109 let r = std::panic::catch_unwind(|| { 3110 guest.wait_vm_boot(None).unwrap(); 3111 3112 // Test that there is no ttyS0 3113 assert_eq!( 3114 guest 3115 .ssh_command(GREP_SERIAL_IRQ_CMD) 3116 .unwrap() 3117 .trim() 3118 .parse::<u32>() 3119 .unwrap_or(1), 3120 0 3121 ); 3122 }); 3123 3124 let _ = child.kill(); 3125 let output = child.wait_with_output().unwrap(); 3126 3127 handle_child_output(r, &output); 3128 } 3129 3130 #[test] 3131 fn test_serial_null() { 3132 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3133 let guest = Guest::new(Box::new(focal)); 3134 let mut cmd = GuestCommand::new(&guest); 3135 #[cfg(target_arch = "x86_64")] 3136 let console_str: &str = "console=ttyS0"; 3137 #[cfg(target_arch = "aarch64")] 3138 let console_str: &str = "console=ttyAMA0"; 3139 3140 cmd.args(&["--cpus", "boot=1"]) 3141 .args(&["--memory", "size=512M"]) 3142 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3143 .args(&[ 3144 "--cmdline", 3145 DIRECT_KERNEL_BOOT_CMDLINE 3146 .replace("console=hvc0 ", console_str) 3147 .as_str(), 3148 ]) 3149 .default_disks() 3150 .default_net() 3151 .args(&["--serial", "null"]) 3152 .args(&["--console", "off"]) 3153 .capture_output(); 3154 3155 let mut child = cmd.spawn().unwrap(); 3156 3157 let r = std::panic::catch_unwind(|| { 3158 guest.wait_vm_boot(None).unwrap(); 3159 3160 // Test that there is a ttyS0 3161 assert_eq!( 3162 guest 3163 .ssh_command(GREP_SERIAL_IRQ_CMD) 3164 .unwrap() 3165 .trim() 3166 .parse::<u32>() 3167 .unwrap_or_default(), 3168 1 3169 ); 3170 }); 3171 3172 let _ = child.kill(); 3173 let output = child.wait_with_output().unwrap(); 3174 handle_child_output(r, &output); 3175 3176 let r = std::panic::catch_unwind(|| { 3177 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3178 }); 3179 3180 handle_child_output(r, &output); 3181 } 3182 3183 #[test] 3184 fn test_serial_tty() { 3185 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3186 let guest = Guest::new(Box::new(focal)); 3187 3188 let kernel_path = direct_kernel_boot_path(); 3189 3190 #[cfg(target_arch = "x86_64")] 3191 let console_str: &str = "console=ttyS0"; 3192 #[cfg(target_arch = "aarch64")] 3193 let console_str: &str = "console=ttyAMA0"; 3194 3195 let mut child = GuestCommand::new(&guest) 3196 .args(&["--cpus", "boot=1"]) 3197 .args(&["--memory", "size=512M"]) 3198 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3199 .args(&[ 3200 "--cmdline", 3201 DIRECT_KERNEL_BOOT_CMDLINE 3202 .replace("console=hvc0 ", console_str) 3203 .as_str(), 3204 ]) 3205 .default_disks() 3206 .default_net() 3207 .args(&["--serial", "tty"]) 3208 .args(&["--console", "off"]) 3209 .capture_output() 3210 .spawn() 3211 .unwrap(); 3212 3213 let r = std::panic::catch_unwind(|| { 3214 guest.wait_vm_boot(None).unwrap(); 3215 3216 // Test that there is a ttyS0 3217 assert_eq!( 3218 guest 3219 .ssh_command(GREP_SERIAL_IRQ_CMD) 3220 .unwrap() 3221 .trim() 3222 .parse::<u32>() 3223 .unwrap_or_default(), 3224 1 3225 ); 3226 }); 3227 3228 // This sleep is needed to wait for the login prompt 3229 thread::sleep(std::time::Duration::new(2, 0)); 3230 3231 let _ = child.kill(); 3232 let output = child.wait_with_output().unwrap(); 3233 handle_child_output(r, &output); 3234 3235 let r = std::panic::catch_unwind(|| { 3236 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3237 }); 3238 3239 handle_child_output(r, &output); 3240 } 3241 3242 #[test] 3243 fn test_serial_file() { 3244 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3245 let guest = Guest::new(Box::new(focal)); 3246 3247 let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output"); 3248 #[cfg(target_arch = "x86_64")] 3249 let console_str: &str = "console=ttyS0"; 3250 #[cfg(target_arch = "aarch64")] 3251 let console_str: &str = "console=ttyAMA0"; 3252 3253 let mut child = GuestCommand::new(&guest) 3254 .args(&["--cpus", "boot=1"]) 3255 .args(&["--memory", "size=512M"]) 3256 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3257 .args(&[ 3258 "--cmdline", 3259 DIRECT_KERNEL_BOOT_CMDLINE 3260 .replace("console=hvc0 ", console_str) 3261 .as_str(), 3262 ]) 3263 .default_disks() 3264 .default_net() 3265 .args(&[ 3266 "--serial", 3267 format!("file={}", serial_path.to_str().unwrap()).as_str(), 3268 ]) 3269 .capture_output() 3270 .spawn() 3271 .unwrap(); 3272 3273 let r = std::panic::catch_unwind(|| { 3274 guest.wait_vm_boot(None).unwrap(); 3275 3276 // Test that there is a ttyS0 3277 assert_eq!( 3278 guest 3279 .ssh_command(GREP_SERIAL_IRQ_CMD) 3280 .unwrap() 3281 .trim() 3282 .parse::<u32>() 3283 .unwrap_or_default(), 3284 1 3285 ); 3286 3287 guest.ssh_command("sudo shutdown -h now").unwrap(); 3288 }); 3289 3290 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3291 let _ = child.kill(); 3292 let output = child.wait_with_output().unwrap(); 3293 handle_child_output(r, &output); 3294 3295 let r = std::panic::catch_unwind(|| { 3296 // Check that the cloud-hypervisor binary actually terminated 3297 assert!(output.status.success()); 3298 3299 // Do this check after shutdown of the VM as an easy way to ensure 3300 // all writes are flushed to disk 3301 let mut f = std::fs::File::open(serial_path).unwrap(); 3302 let mut buf = String::new(); 3303 f.read_to_string(&mut buf).unwrap(); 3304 assert!(buf.contains(CONSOLE_TEST_STRING)); 3305 }); 3306 3307 handle_child_output(r, &output); 3308 } 3309 3310 #[test] 3311 fn test_pty_interaction() { 3312 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3313 let guest = Guest::new(Box::new(focal)); 3314 let api_socket = temp_api_path(&guest.tmp_dir); 3315 let serial_option = if cfg!(target_arch = "x86_64") { 3316 " console=ttyS0" 3317 } else { 3318 " console=ttyAMA0" 3319 }; 3320 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 3321 3322 let mut child = GuestCommand::new(&guest) 3323 .args(&["--cpus", "boot=1"]) 3324 .args(&["--memory", "size=512M"]) 3325 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3326 .args(&["--cmdline", &cmdline]) 3327 .default_disks() 3328 .default_net() 3329 .args(&["--serial", "null"]) 3330 .args(&["--console", "pty"]) 3331 .args(&["--api-socket", &api_socket]) 3332 .spawn() 3333 .unwrap(); 3334 3335 let r = std::panic::catch_unwind(|| { 3336 guest.wait_vm_boot(None).unwrap(); 3337 // Get pty fd for console 3338 let console_path = get_pty_path(&api_socket, "console"); 3339 // TODO: Get serial pty test working 3340 let mut cf = std::fs::OpenOptions::new() 3341 .write(true) 3342 .read(true) 3343 .open(console_path) 3344 .unwrap(); 3345 3346 // Some dumb sleeps but we don't want to write 3347 // before the console is up and we don't want 3348 // to try and write the next line before the 3349 // login process is ready. 3350 thread::sleep(std::time::Duration::new(5, 0)); 3351 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 3352 thread::sleep(std::time::Duration::new(2, 0)); 3353 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 3354 thread::sleep(std::time::Duration::new(2, 0)); 3355 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 3356 thread::sleep(std::time::Duration::new(2, 0)); 3357 3358 // read pty and ensure they have a login shell 3359 // some fairly hacky workarounds to avoid looping 3360 // forever in case the channel is blocked getting output 3361 let ptyc = pty_read(cf); 3362 let mut empty = 0; 3363 let mut prev = String::new(); 3364 loop { 3365 thread::sleep(std::time::Duration::new(2, 0)); 3366 match ptyc.try_recv() { 3367 Ok(line) => { 3368 empty = 0; 3369 prev = prev + &line; 3370 if prev.contains("test_pty_console") { 3371 break; 3372 } 3373 } 3374 Err(mpsc::TryRecvError::Empty) => { 3375 empty += 1; 3376 if empty > 5 { 3377 panic!("No login on pty"); 3378 } 3379 } 3380 _ => panic!("No login on pty"), 3381 } 3382 } 3383 3384 guest.ssh_command("sudo shutdown -h now").unwrap(); 3385 }); 3386 3387 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3388 let _ = child.kill(); 3389 let output = child.wait_with_output().unwrap(); 3390 handle_child_output(r, &output); 3391 3392 let r = std::panic::catch_unwind(|| { 3393 // Check that the cloud-hypervisor binary actually terminated 3394 assert!(output.status.success()) 3395 }); 3396 handle_child_output(r, &output); 3397 } 3398 3399 #[test] 3400 fn test_virtio_console() { 3401 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3402 let guest = Guest::new(Box::new(focal)); 3403 3404 let kernel_path = direct_kernel_boot_path(); 3405 3406 let mut child = GuestCommand::new(&guest) 3407 .args(&["--cpus", "boot=1"]) 3408 .args(&["--memory", "size=512M"]) 3409 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3410 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3411 .default_disks() 3412 .default_net() 3413 .args(&["--console", "tty"]) 3414 .args(&["--serial", "null"]) 3415 .capture_output() 3416 .spawn() 3417 .unwrap(); 3418 3419 let text = String::from("On a branch floating down river a cricket, singing."); 3420 let cmd = format!("echo {} | sudo tee /dev/hvc0", text); 3421 3422 let r = std::panic::catch_unwind(|| { 3423 guest.wait_vm_boot(None).unwrap(); 3424 3425 #[cfg(feature = "acpi")] 3426 assert!(guest 3427 .does_device_vendor_pair_match("0x1043", "0x1af4") 3428 .unwrap_or_default()); 3429 3430 guest.ssh_command(&cmd).unwrap(); 3431 }); 3432 3433 let _ = child.kill(); 3434 let output = child.wait_with_output().unwrap(); 3435 handle_child_output(r, &output); 3436 3437 let r = std::panic::catch_unwind(|| { 3438 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 3439 }); 3440 3441 handle_child_output(r, &output); 3442 } 3443 3444 #[test] 3445 fn test_console_file() { 3446 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3447 let guest = Guest::new(Box::new(focal)); 3448 3449 let console_path = guest.tmp_dir.as_path().join("/tmp/console-output"); 3450 let mut child = GuestCommand::new(&guest) 3451 .args(&["--cpus", "boot=1"]) 3452 .args(&["--memory", "size=512M"]) 3453 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3454 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3455 .default_disks() 3456 .default_net() 3457 .args(&[ 3458 "--console", 3459 format!("file={}", console_path.to_str().unwrap()).as_str(), 3460 ]) 3461 .capture_output() 3462 .spawn() 3463 .unwrap(); 3464 3465 guest.wait_vm_boot(None).unwrap(); 3466 3467 guest.ssh_command("sudo shutdown -h now").unwrap(); 3468 3469 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3470 let _ = child.kill(); 3471 let output = child.wait_with_output().unwrap(); 3472 3473 let r = std::panic::catch_unwind(|| { 3474 // Check that the cloud-hypervisor binary actually terminated 3475 assert!(output.status.success()); 3476 3477 // Do this check after shutdown of the VM as an easy way to ensure 3478 // all writes are flushed to disk 3479 let mut f = std::fs::File::open(console_path).unwrap(); 3480 let mut buf = String::new(); 3481 f.read_to_string(&mut buf).unwrap(); 3482 3483 if !buf.contains(CONSOLE_TEST_STRING) { 3484 eprintln!( 3485 "\n\n==== Console file output ====\n\n{}\n\n==== End console file output ====", 3486 buf 3487 ); 3488 } 3489 assert!(buf.contains(CONSOLE_TEST_STRING)); 3490 }); 3491 3492 handle_child_output(r, &output); 3493 } 3494 3495 #[test] 3496 #[cfg(target_arch = "x86_64")] 3497 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 3498 // backed networking interfaces, bound through a simple bridge on the host. 3499 // So if the nested cloud-hypervisor succeeds in getting a directly 3500 // assigned interface from its cloud-hypervisor host, we should be able to 3501 // ssh into it, and verify that it's running with the right kernel command 3502 // line (We tag the command line from cloud-hypervisor for that purpose). 3503 // The third device is added to validate that hotplug works correctly since 3504 // it is being added to the L2 VM through hotplugging mechanism. 3505 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 3506 // vfio device support 3507 fn test_vfio() { 3508 setup_vfio_network_interfaces(); 3509 3510 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3511 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 3512 3513 let mut workload_path = dirs::home_dir().unwrap(); 3514 workload_path.push("workloads"); 3515 3516 let kernel_path = direct_kernel_boot_path(); 3517 3518 let mut vfio_path = workload_path.clone(); 3519 vfio_path.push("vfio"); 3520 3521 let mut cloud_init_vfio_base_path = vfio_path.clone(); 3522 cloud_init_vfio_base_path.push("cloudinit.img"); 3523 3524 // We copy our cloudinit into the vfio mount point, for the nested 3525 // cloud-hypervisor guest to use. 3526 rate_limited_copy( 3527 &guest.disk_config.disk(DiskType::CloudInit).unwrap(), 3528 &cloud_init_vfio_base_path, 3529 ) 3530 .expect("copying of cloud-init disk failed"); 3531 3532 let mut vfio_disk_path = workload_path.clone(); 3533 vfio_disk_path.push("vfio.img"); 3534 3535 // Create the vfio disk image 3536 let output = Command::new("mkfs.ext4") 3537 .arg("-d") 3538 .arg(vfio_path.to_str().unwrap()) 3539 .arg(vfio_disk_path.to_str().unwrap()) 3540 .arg("2g") 3541 .output() 3542 .unwrap(); 3543 if !output.status.success() { 3544 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 3545 panic!("mkfs.ext4 command generated an error"); 3546 } 3547 3548 let mut blk_file_path = workload_path; 3549 blk_file_path.push("blk.img"); 3550 3551 let vfio_tap0 = "vfio-tap0"; 3552 let vfio_tap1 = "vfio-tap1"; 3553 let vfio_tap2 = "vfio-tap2"; 3554 let vfio_tap3 = "vfio-tap3"; 3555 3556 let mut child = GuestCommand::new(&guest) 3557 .args(&["--cpus", "boot=4"]) 3558 .args(&["--memory", "size=2G,hugepages=on,shared=on"]) 3559 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3560 .args(&[ 3561 "--disk", 3562 format!( 3563 "path={}", 3564 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3565 ) 3566 .as_str(), 3567 format!( 3568 "path={}", 3569 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3570 ) 3571 .as_str(), 3572 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 3573 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 3574 ]) 3575 .args(&[ 3576 "--cmdline", 3577 format!( 3578 "{} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts", 3579 DIRECT_KERNEL_BOOT_CMDLINE 3580 ) 3581 .as_str(), 3582 ]) 3583 .args(&[ 3584 "--net", 3585 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 3586 format!( 3587 "tap={},mac={},iommu=on", 3588 vfio_tap1, guest.network.l2_guest_mac1 3589 ) 3590 .as_str(), 3591 format!( 3592 "tap={},mac={},iommu=on", 3593 vfio_tap2, guest.network.l2_guest_mac2 3594 ) 3595 .as_str(), 3596 format!( 3597 "tap={},mac={},iommu=on", 3598 vfio_tap3, guest.network.l2_guest_mac3 3599 ) 3600 .as_str(), 3601 ]) 3602 .capture_output() 3603 .spawn() 3604 .unwrap(); 3605 3606 thread::sleep(std::time::Duration::new(30, 0)); 3607 3608 let r = std::panic::catch_unwind(|| { 3609 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 3610 thread::sleep(std::time::Duration::new(120, 0)); 3611 3612 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 3613 // added to its kernel command line. 3614 // Let's ssh into it and verify that it's there. If it is it means 3615 // we're in the right guest (The L2 one) because the QEMU L1 guest 3616 // does not have this command line tag. 3617 assert_eq!( 3618 guest 3619 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 3620 .unwrap() 3621 .trim() 3622 .parse::<u32>() 3623 .unwrap_or_default(), 3624 1 3625 ); 3626 3627 // Let's also verify from the second virtio-net device passed to 3628 // the L2 VM. 3629 assert_eq!( 3630 guest 3631 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 3632 .unwrap() 3633 .trim() 3634 .parse::<u32>() 3635 .unwrap_or_default(), 3636 1 3637 ); 3638 3639 // Check the amount of PCI devices appearing in L2 VM. 3640 assert_eq!( 3641 guest 3642 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3643 .unwrap() 3644 .trim() 3645 .parse::<u32>() 3646 .unwrap_or_default(), 3647 8, 3648 ); 3649 3650 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 3651 assert_eq!( 3652 guest 3653 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 3654 .unwrap() 3655 .trim() 3656 .parse::<u32>() 3657 .unwrap_or_default(), 3658 1 3659 ); 3660 3661 // Hotplug an extra virtio-net device through L2 VM. 3662 guest.ssh_command_l1( 3663 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 3664 ).unwrap(); 3665 guest 3666 .ssh_command_l1( 3667 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind", 3668 ) 3669 .unwrap(); 3670 let vfio_hotplug_output = guest 3671 .ssh_command_l1( 3672 "sudo /mnt/ch-remote \ 3673 --api-socket=/tmp/ch_api.sock \ 3674 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 3675 ) 3676 .unwrap(); 3677 assert!( 3678 vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}") 3679 ); 3680 3681 thread::sleep(std::time::Duration::new(10, 0)); 3682 3683 // Let's also verify from the third virtio-net device passed to 3684 // the L2 VM. This third device has been hotplugged through the L2 3685 // VM, so this is our way to validate hotplug works for VFIO PCI. 3686 assert_eq!( 3687 guest 3688 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 3689 .unwrap() 3690 .trim() 3691 .parse::<u32>() 3692 .unwrap_or_default(), 3693 1 3694 ); 3695 3696 // Check the amount of PCI devices appearing in L2 VM. 3697 // There should be one more device than before, raising the count 3698 // up to 9 PCI devices. 3699 assert_eq!( 3700 guest 3701 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3702 .unwrap() 3703 .trim() 3704 .parse::<u32>() 3705 .unwrap_or_default(), 3706 9, 3707 ); 3708 3709 // Let's now verify that we can correctly remove the virtio-net 3710 // device through the "remove-device" command responsible for 3711 // unplugging VFIO devices. 3712 guest 3713 .ssh_command_l1( 3714 "sudo /mnt/ch-remote \ 3715 --api-socket=/tmp/ch_api.sock \ 3716 remove-device vfio123", 3717 ) 3718 .unwrap(); 3719 thread::sleep(std::time::Duration::new(10, 0)); 3720 3721 // Check the amount of PCI devices appearing in L2 VM is back down 3722 // to 8 devices. 3723 assert_eq!( 3724 guest 3725 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3726 .unwrap() 3727 .trim() 3728 .parse::<u32>() 3729 .unwrap_or_default(), 3730 8, 3731 ); 3732 3733 // Perform memory hotplug in L2 and validate the memory is showing 3734 // up as expected. In order to check, we will use the virtio-net 3735 // device already passed through L2 as a VFIO device, this will 3736 // verify that VFIO devices are functional with memory hotplug. 3737 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 3738 guest.ssh_command_l2_1( 3739 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 3740 ).unwrap(); 3741 guest 3742 .ssh_command_l1( 3743 "sudo /mnt/ch-remote \ 3744 --api-socket=/tmp/ch_api.sock \ 3745 resize --memory=1073741824", 3746 ) 3747 .unwrap(); 3748 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 3749 }); 3750 3751 let _ = child.kill(); 3752 let output = child.wait_with_output().unwrap(); 3753 3754 cleanup_vfio_network_interfaces(); 3755 3756 handle_child_output(r, &output); 3757 } 3758 3759 #[test] 3760 fn test_direct_kernel_boot_noacpi() { 3761 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3762 let guest = Guest::new(Box::new(focal)); 3763 3764 let kernel_path = direct_kernel_boot_path(); 3765 3766 let mut child = GuestCommand::new(&guest) 3767 .args(&["--cpus", "boot=1"]) 3768 .args(&["--memory", "size=512M"]) 3769 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3770 .args(&[ 3771 "--cmdline", 3772 format!("{} acpi=off", DIRECT_KERNEL_BOOT_CMDLINE).as_str(), 3773 ]) 3774 .default_disks() 3775 .default_net() 3776 .capture_output() 3777 .spawn() 3778 .unwrap(); 3779 3780 let r = std::panic::catch_unwind(|| { 3781 guest.wait_vm_boot(None).unwrap(); 3782 3783 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3784 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3785 assert!(guest.get_entropy().unwrap_or_default() >= 900); 3786 }); 3787 3788 let _ = child.kill(); 3789 let output = child.wait_with_output().unwrap(); 3790 3791 handle_child_output(r, &output); 3792 } 3793 3794 #[test] 3795 fn test_reboot() { 3796 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 3797 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3798 3799 vec![Box::new(bionic), Box::new(focal)] 3800 .drain(..) 3801 .for_each(|disk_config| { 3802 let guest = Guest::new(disk_config); 3803 3804 let mut cmd = GuestCommand::new(&guest); 3805 cmd.args(&["--cpus", "boot=1"]) 3806 .args(&["--memory", "size=512M"]) 3807 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3808 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3809 .default_disks() 3810 .default_net() 3811 .capture_output(); 3812 3813 let mut child = cmd.spawn().unwrap(); 3814 3815 let r = std::panic::catch_unwind(|| { 3816 guest.wait_vm_boot(Some(120)).unwrap(); 3817 3818 let fd_count_1 = get_fd_count(child.id()); 3819 guest.reboot_linux(0, Some(120)); 3820 let fd_count_2 = get_fd_count(child.id()); 3821 assert_eq!(fd_count_1, fd_count_2); 3822 3823 guest.ssh_command("sudo shutdown -h now").unwrap(); 3824 }); 3825 3826 let _ = child.wait_timeout(std::time::Duration::from_secs(40)); 3827 let _ = child.kill(); 3828 let output = child.wait_with_output().unwrap(); 3829 handle_child_output(r, &output); 3830 3831 let r = std::panic::catch_unwind(|| { 3832 // Check that the cloud-hypervisor binary actually terminated 3833 assert!(output.status.success()); 3834 }); 3835 3836 handle_child_output(r, &output); 3837 }); 3838 } 3839 3840 #[test] 3841 fn test_virtio_vsock() { 3842 _test_virtio_vsock(false) 3843 } 3844 3845 #[test] 3846 #[cfg(target_arch = "x86_64")] 3847 fn test_virtio_vsock_hotplug() { 3848 _test_virtio_vsock(true); 3849 } 3850 3851 #[test] 3852 // Start cloud-hypervisor with no VM parameters, only the API server running. 3853 // From the API: Create a VM, boot it and check that it looks as expected. 3854 fn test_api_create_boot() { 3855 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3856 let guest = Guest::new(Box::new(focal)); 3857 3858 let api_socket = temp_api_path(&guest.tmp_dir); 3859 3860 let mut child = GuestCommand::new(&guest) 3861 .args(&["--api-socket", &api_socket]) 3862 .capture_output() 3863 .spawn() 3864 .unwrap(); 3865 3866 thread::sleep(std::time::Duration::new(1, 0)); 3867 3868 // Verify API server is running 3869 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 3870 3871 // Create the VM first 3872 let cpu_count: u8 = 4; 3873 let http_body = guest.api_create_body(cpu_count); 3874 curl_command( 3875 &api_socket, 3876 "PUT", 3877 "http://localhost/api/v1/vm.create", 3878 Some(&http_body), 3879 ); 3880 3881 // Then boot it 3882 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 3883 thread::sleep(std::time::Duration::new(20, 0)); 3884 3885 let r = std::panic::catch_unwind(|| { 3886 // Check that the VM booted as expected 3887 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 3888 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3889 assert!(guest.get_entropy().unwrap_or_default() >= 900); 3890 }); 3891 3892 let _ = child.kill(); 3893 let output = child.wait_with_output().unwrap(); 3894 3895 handle_child_output(r, &output); 3896 } 3897 3898 #[test] 3899 // Start cloud-hypervisor with no VM parameters, only the API server running. 3900 // From the API: Create a VM, boot it and check that it looks as expected. 3901 // Then we pause the VM, check that it's no longer available. 3902 // Finally we resume the VM and check that it's available. 3903 fn test_api_pause_resume() { 3904 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3905 let guest = Guest::new(Box::new(focal)); 3906 3907 let api_socket = temp_api_path(&guest.tmp_dir); 3908 3909 let mut child = GuestCommand::new(&guest) 3910 .args(&["--api-socket", &api_socket]) 3911 .capture_output() 3912 .spawn() 3913 .unwrap(); 3914 3915 thread::sleep(std::time::Duration::new(1, 0)); 3916 3917 // Verify API server is running 3918 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 3919 3920 // Create the VM first 3921 let cpu_count: u8 = 4; 3922 let http_body = guest.api_create_body(cpu_count); 3923 curl_command( 3924 &api_socket, 3925 "PUT", 3926 "http://localhost/api/v1/vm.create", 3927 Some(&http_body), 3928 ); 3929 3930 // Then boot it 3931 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 3932 thread::sleep(std::time::Duration::new(20, 0)); 3933 3934 let r = std::panic::catch_unwind(|| { 3935 // Check that the VM booted as expected 3936 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 3937 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3938 assert!(guest.get_entropy().unwrap_or_default() >= 900); 3939 3940 // We now pause the VM 3941 assert!(remote_command(&api_socket, "pause", None)); 3942 3943 // Check pausing again fails 3944 assert!(!remote_command(&api_socket, "pause", None)); 3945 3946 thread::sleep(std::time::Duration::new(2, 0)); 3947 3948 // SSH into the VM should fail 3949 assert!(ssh_command_ip( 3950 "grep -c processor /proc/cpuinfo", 3951 &guest.network.guest_ip, 3952 2, 3953 5 3954 ) 3955 .is_err()); 3956 3957 // Resume the VM 3958 assert!(remote_command(&api_socket, "resume", None)); 3959 3960 // Check resuming again fails 3961 assert!(!remote_command(&api_socket, "resume", None)); 3962 3963 thread::sleep(std::time::Duration::new(2, 0)); 3964 3965 // Now we should be able to SSH back in and get the right number of CPUs 3966 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 3967 }); 3968 3969 let _ = child.kill(); 3970 let output = child.wait_with_output().unwrap(); 3971 3972 handle_child_output(r, &output); 3973 } 3974 3975 #[test] 3976 #[cfg(target_arch = "x86_64")] 3977 // This test validates that it can find the virtio-iommu device at first. 3978 // It also verifies that both disks and the network card are attached to 3979 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 3980 // The last interesting part of this test is that it exercises the network 3981 // interface attached to the virtual IOMMU since this is the one used to 3982 // send all commands through SSH. 3983 fn test_virtio_iommu() { 3984 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3985 let guest = Guest::new(Box::new(focal)); 3986 3987 let kernel_path = direct_kernel_boot_path(); 3988 3989 let mut child = GuestCommand::new(&guest) 3990 .args(&["--cpus", "boot=1"]) 3991 .args(&["--memory", "size=512M"]) 3992 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3993 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3994 .args(&[ 3995 "--disk", 3996 format!( 3997 "path={},iommu=on", 3998 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3999 ) 4000 .as_str(), 4001 format!( 4002 "path={},iommu=on", 4003 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4004 ) 4005 .as_str(), 4006 ]) 4007 .args(&["--net", guest.default_net_string_w_iommu().as_str()]) 4008 .capture_output() 4009 .spawn() 4010 .unwrap(); 4011 4012 let r = std::panic::catch_unwind(|| { 4013 guest.wait_vm_boot(None).unwrap(); 4014 4015 // Verify the virtio-iommu device is present. 4016 assert!(guest 4017 .does_device_vendor_pair_match("0x1057", "0x1af4") 4018 .unwrap_or_default()); 4019 4020 // Verify the first disk is located under IOMMU group 0. 4021 assert_eq!( 4022 guest 4023 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 4024 .unwrap() 4025 .trim(), 4026 "0000:00:02.0" 4027 ); 4028 4029 // Verify the second disk is located under IOMMU group 1. 4030 assert_eq!( 4031 guest 4032 .ssh_command("ls /sys/kernel/iommu_groups/1/devices") 4033 .unwrap() 4034 .trim(), 4035 "0000:00:03.0" 4036 ); 4037 4038 // Verify the network card is located under IOMMU group 2. 4039 assert_eq!( 4040 guest 4041 .ssh_command("ls /sys/kernel/iommu_groups/2/devices") 4042 .unwrap() 4043 .trim(), 4044 "0000:00:04.0" 4045 ); 4046 }); 4047 4048 let _ = child.kill(); 4049 let output = child.wait_with_output().unwrap(); 4050 4051 handle_child_output(r, &output); 4052 } 4053 4054 #[test] 4055 #[cfg(target_arch = "x86_64")] 4056 // We cannot force the software running in the guest to reprogram the BAR 4057 // with some different addresses, but we have a reliable way of testing it 4058 // with a standard Linux kernel. 4059 // By removing a device from the PCI tree, and then rescanning the tree, 4060 // Linux consistently chooses to reorganize the PCI device BARs to other 4061 // locations in the guest address space. 4062 // This test creates a dedicated PCI network device to be checked as being 4063 // properly probed first, then removing it, and adding it again by doing a 4064 // rescan. 4065 fn test_pci_bar_reprogramming() { 4066 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4067 let guest = Guest::new(Box::new(focal)); 4068 let mut child = GuestCommand::new(&guest) 4069 .args(&["--cpus", "boot=1"]) 4070 .args(&["--memory", "size=512M"]) 4071 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4072 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4073 .default_disks() 4074 .args(&[ 4075 "--net", 4076 guest.default_net_string().as_str(), 4077 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4078 ]) 4079 .capture_output() 4080 .spawn() 4081 .unwrap(); 4082 4083 let r = 4084 std::panic::catch_unwind(|| { 4085 guest.wait_vm_boot(None).unwrap(); 4086 4087 // 2 network interfaces + default localhost ==> 3 interfaces 4088 assert_eq!( 4089 guest 4090 .ssh_command("ip -o link | wc -l") 4091 .unwrap() 4092 .trim() 4093 .parse::<u32>() 4094 .unwrap_or_default(), 4095 3 4096 ); 4097 4098 let init_bar_addr = guest.ssh_command( 4099 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4100 ).unwrap(); 4101 4102 // Remove the PCI device 4103 guest 4104 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4105 .unwrap(); 4106 4107 // Only 1 network interface left + default localhost ==> 2 interfaces 4108 assert_eq!( 4109 guest 4110 .ssh_command("ip -o link | wc -l") 4111 .unwrap() 4112 .trim() 4113 .parse::<u32>() 4114 .unwrap_or_default(), 4115 2 4116 ); 4117 4118 // Remove the PCI device 4119 guest 4120 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4121 .unwrap(); 4122 4123 // Back to 2 network interface + default localhost ==> 3 interfaces 4124 assert_eq!( 4125 guest 4126 .ssh_command("ip -o link | wc -l") 4127 .unwrap() 4128 .trim() 4129 .parse::<u32>() 4130 .unwrap_or_default(), 4131 3 4132 ); 4133 4134 let new_bar_addr = guest.ssh_command( 4135 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4136 ).unwrap(); 4137 4138 // Let's compare the BAR addresses for our virtio-net device. 4139 // They should be different as we expect the BAR reprogramming 4140 // to have happened. 4141 assert_ne!(init_bar_addr, new_bar_addr); 4142 }); 4143 4144 let _ = child.kill(); 4145 let output = child.wait_with_output().unwrap(); 4146 4147 handle_child_output(r, &output); 4148 } 4149 4150 #[test] 4151 fn test_memory_mergeable_off() { 4152 test_memory_mergeable(false) 4153 } 4154 4155 #[test] 4156 #[cfg(target_arch = "x86_64")] 4157 fn test_cpu_hotplug() { 4158 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4159 let guest = Guest::new(Box::new(focal)); 4160 let api_socket = temp_api_path(&guest.tmp_dir); 4161 4162 let kernel_path = direct_kernel_boot_path(); 4163 4164 let mut child = GuestCommand::new(&guest) 4165 .args(&["--cpus", "boot=2,max=4"]) 4166 .args(&["--memory", "size=512M"]) 4167 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4168 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4169 .default_disks() 4170 .default_net() 4171 .args(&["--api-socket", &api_socket]) 4172 .capture_output() 4173 .spawn() 4174 .unwrap(); 4175 4176 let r = std::panic::catch_unwind(|| { 4177 guest.wait_vm_boot(None).unwrap(); 4178 4179 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4180 4181 // Resize the VM 4182 let desired_vcpus = 4; 4183 resize_command(&api_socket, Some(desired_vcpus), None, None); 4184 4185 guest 4186 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4187 .unwrap(); 4188 guest 4189 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4190 .unwrap(); 4191 thread::sleep(std::time::Duration::new(10, 0)); 4192 assert_eq!( 4193 guest.get_cpu_count().unwrap_or_default(), 4194 u32::from(desired_vcpus) 4195 ); 4196 4197 guest.reboot_linux(0, None); 4198 4199 assert_eq!( 4200 guest.get_cpu_count().unwrap_or_default(), 4201 u32::from(desired_vcpus) 4202 ); 4203 4204 // Resize the VM 4205 let desired_vcpus = 2; 4206 resize_command(&api_socket, Some(desired_vcpus), None, None); 4207 4208 thread::sleep(std::time::Duration::new(10, 0)); 4209 assert_eq!( 4210 guest.get_cpu_count().unwrap_or_default(), 4211 u32::from(desired_vcpus) 4212 ); 4213 4214 // Resize the VM back up to 4 4215 let desired_vcpus = 4; 4216 resize_command(&api_socket, Some(desired_vcpus), None, None); 4217 4218 guest 4219 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4220 .unwrap(); 4221 guest 4222 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4223 .unwrap(); 4224 thread::sleep(std::time::Duration::new(10, 0)); 4225 assert_eq!( 4226 guest.get_cpu_count().unwrap_or_default(), 4227 u32::from(desired_vcpus) 4228 ); 4229 }); 4230 4231 let _ = child.kill(); 4232 let output = child.wait_with_output().unwrap(); 4233 4234 handle_child_output(r, &output); 4235 } 4236 4237 #[test] 4238 #[cfg(target_arch = "x86_64")] 4239 fn test_memory_hotplug() { 4240 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4241 let guest = Guest::new(Box::new(focal)); 4242 let api_socket = temp_api_path(&guest.tmp_dir); 4243 4244 let kernel_path = direct_kernel_boot_path(); 4245 4246 let mut child = GuestCommand::new(&guest) 4247 .args(&["--cpus", "boot=2,max=4"]) 4248 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4249 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4250 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4251 .default_disks() 4252 .default_net() 4253 .args(&["--balloon", "size=0"]) 4254 .args(&["--api-socket", &api_socket]) 4255 .capture_output() 4256 .spawn() 4257 .unwrap(); 4258 4259 let r = std::panic::catch_unwind(|| { 4260 guest.wait_vm_boot(None).unwrap(); 4261 4262 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4263 4264 guest.enable_memory_hotplug(); 4265 4266 // Add RAM to the VM 4267 let desired_ram = 1024 << 20; 4268 resize_command(&api_socket, None, Some(desired_ram), None); 4269 4270 thread::sleep(std::time::Duration::new(10, 0)); 4271 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4272 4273 // Use balloon to remove RAM from the VM 4274 let desired_balloon = 512 << 20; 4275 resize_command(&api_socket, None, None, Some(desired_balloon)); 4276 4277 thread::sleep(std::time::Duration::new(10, 0)); 4278 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4279 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4280 4281 guest.reboot_linux(0, None); 4282 4283 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4284 4285 // Use balloon add RAM to the VM 4286 let desired_balloon = 0; 4287 resize_command(&api_socket, None, None, Some(desired_balloon)); 4288 4289 thread::sleep(std::time::Duration::new(10, 0)); 4290 4291 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4292 4293 guest.enable_memory_hotplug(); 4294 4295 // Add RAM to the VM 4296 let desired_ram = 2048 << 20; 4297 resize_command(&api_socket, None, Some(desired_ram), None); 4298 4299 thread::sleep(std::time::Duration::new(10, 0)); 4300 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4301 4302 // Remove RAM to the VM (only applies after reboot) 4303 let desired_ram = 1024 << 20; 4304 resize_command(&api_socket, None, Some(desired_ram), None); 4305 4306 guest.reboot_linux(1, None); 4307 4308 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4309 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4310 }); 4311 4312 let _ = child.kill(); 4313 let output = child.wait_with_output().unwrap(); 4314 4315 handle_child_output(r, &output); 4316 } 4317 4318 #[test] 4319 fn test_virtio_mem() { 4320 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4321 let guest = Guest::new(Box::new(focal)); 4322 let api_socket = temp_api_path(&guest.tmp_dir); 4323 4324 let kernel_path = direct_kernel_boot_path(); 4325 4326 let mut child = GuestCommand::new(&guest) 4327 .args(&["--cpus", "boot=2,max=4"]) 4328 .args(&[ 4329 "--memory", 4330 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4331 ]) 4332 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4333 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4334 .default_disks() 4335 .default_net() 4336 .args(&["--api-socket", &api_socket]) 4337 .capture_output() 4338 .spawn() 4339 .unwrap(); 4340 4341 let r = std::panic::catch_unwind(|| { 4342 guest.wait_vm_boot(None).unwrap(); 4343 4344 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4345 4346 guest.enable_memory_hotplug(); 4347 4348 // Add RAM to the VM 4349 let desired_ram = 1024 << 20; 4350 resize_command(&api_socket, None, Some(desired_ram), None); 4351 4352 thread::sleep(std::time::Duration::new(10, 0)); 4353 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4354 4355 // Add RAM to the VM 4356 let desired_ram = 2048 << 20; 4357 resize_command(&api_socket, None, Some(desired_ram), None); 4358 4359 thread::sleep(std::time::Duration::new(10, 0)); 4360 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4361 4362 // Remove RAM from the VM 4363 let desired_ram = 1024 << 20; 4364 resize_command(&api_socket, None, Some(desired_ram), None); 4365 4366 thread::sleep(std::time::Duration::new(10, 0)); 4367 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4368 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4369 4370 guest.reboot_linux(0, None); 4371 4372 // Check the amount of memory after reboot is 1GiB 4373 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4374 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4375 4376 // Check we can still resize to 512MiB 4377 let desired_ram = 512 << 20; 4378 resize_command(&api_socket, None, Some(desired_ram), None); 4379 thread::sleep(std::time::Duration::new(10, 0)); 4380 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4381 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4382 }); 4383 4384 let _ = child.kill(); 4385 let output = child.wait_with_output().unwrap(); 4386 4387 handle_child_output(r, &output); 4388 } 4389 4390 #[test] 4391 #[cfg(target_arch = "x86_64")] 4392 // Test both vCPU and memory resizing together 4393 fn test_resize() { 4394 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4395 let guest = Guest::new(Box::new(focal)); 4396 let api_socket = temp_api_path(&guest.tmp_dir); 4397 4398 let kernel_path = direct_kernel_boot_path(); 4399 4400 let mut child = GuestCommand::new(&guest) 4401 .args(&["--cpus", "boot=2,max=4"]) 4402 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4403 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4404 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4405 .default_disks() 4406 .default_net() 4407 .args(&["--api-socket", &api_socket]) 4408 .capture_output() 4409 .spawn() 4410 .unwrap(); 4411 4412 let r = std::panic::catch_unwind(|| { 4413 guest.wait_vm_boot(None).unwrap(); 4414 4415 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4416 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4417 4418 guest.enable_memory_hotplug(); 4419 4420 // Resize the VM 4421 let desired_vcpus = 4; 4422 let desired_ram = 1024 << 20; 4423 resize_command(&api_socket, Some(desired_vcpus), Some(desired_ram), None); 4424 4425 guest 4426 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4427 .unwrap(); 4428 guest 4429 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4430 .unwrap(); 4431 thread::sleep(std::time::Duration::new(10, 0)); 4432 assert_eq!( 4433 guest.get_cpu_count().unwrap_or_default(), 4434 u32::from(desired_vcpus) 4435 ); 4436 4437 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4438 }); 4439 4440 let _ = child.kill(); 4441 let output = child.wait_with_output().unwrap(); 4442 4443 handle_child_output(r, &output); 4444 } 4445 4446 #[test] 4447 fn test_memory_overhead() { 4448 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4449 let guest = Guest::new(Box::new(focal)); 4450 4451 let kernel_path = direct_kernel_boot_path(); 4452 4453 let guest_memory_size_kb = 512 * 1024; 4454 4455 let mut child = GuestCommand::new(&guest) 4456 .args(&["--cpus", "boot=1"]) 4457 .args(&[ 4458 "--memory", 4459 format!("size={}K", guest_memory_size_kb).as_str(), 4460 ]) 4461 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4462 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4463 .default_disks() 4464 .capture_output() 4465 .spawn() 4466 .unwrap(); 4467 4468 thread::sleep(std::time::Duration::new(20, 0)); 4469 4470 let r = std::panic::catch_unwind(|| { 4471 // On AArch64 when acpi is enabled, there is a 4 MiB gap between the RAM 4472 // that the VMM gives and the guest can see. 4473 // This is a temporary solution, will be fixed in future. 4474 #[cfg(all(target_arch = "aarch64", feature = "acpi"))] 4475 let guest_memory_size_kb = guest_memory_size_kb - 4 * 1024; 4476 4477 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 4478 eprintln!( 4479 "Guest memory overhead: {} vs {}", 4480 overhead, MAXIMUM_VMM_OVERHEAD_KB 4481 ); 4482 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 4483 }); 4484 4485 let _ = child.kill(); 4486 let output = child.wait_with_output().unwrap(); 4487 4488 handle_child_output(r, &output); 4489 } 4490 4491 #[test] 4492 #[cfg(target_arch = "x86_64")] 4493 fn test_disk_hotplug() { 4494 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4495 let guest = Guest::new(Box::new(focal)); 4496 4497 let kernel_path = direct_kernel_boot_path(); 4498 4499 let api_socket = temp_api_path(&guest.tmp_dir); 4500 4501 let mut child = GuestCommand::new(&guest) 4502 .args(&["--api-socket", &api_socket]) 4503 .args(&["--cpus", "boot=1"]) 4504 .args(&["--memory", "size=512M"]) 4505 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4506 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4507 .default_disks() 4508 .default_net() 4509 .capture_output() 4510 .spawn() 4511 .unwrap(); 4512 4513 let r = std::panic::catch_unwind(|| { 4514 guest.wait_vm_boot(None).unwrap(); 4515 4516 // Check /dev/vdc is not there 4517 assert_eq!( 4518 guest 4519 .ssh_command("lsblk | grep -c vdc.*16M || true") 4520 .unwrap() 4521 .trim() 4522 .parse::<u32>() 4523 .unwrap_or(1), 4524 0 4525 ); 4526 4527 // Now let's add the extra disk. 4528 let mut blk_file_path = dirs::home_dir().unwrap(); 4529 blk_file_path.push("workloads"); 4530 blk_file_path.push("blk.img"); 4531 let (cmd_success, cmd_output) = remote_command_w_output( 4532 &api_socket, 4533 "add-disk", 4534 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4535 ); 4536 assert!(cmd_success); 4537 assert!(String::from_utf8_lossy(&cmd_output) 4538 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4539 4540 thread::sleep(std::time::Duration::new(10, 0)); 4541 4542 // Check that /dev/vdc exists and the block size is 16M. 4543 assert_eq!( 4544 guest 4545 .ssh_command("lsblk | grep vdc | grep -c 16M") 4546 .unwrap() 4547 .trim() 4548 .parse::<u32>() 4549 .unwrap_or_default(), 4550 1 4551 ); 4552 // And check the block device can be read. 4553 guest 4554 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4555 .unwrap(); 4556 4557 // Let's remove it the extra disk. 4558 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4559 thread::sleep(std::time::Duration::new(5, 0)); 4560 // And check /dev/vdc is not there 4561 assert_eq!( 4562 guest 4563 .ssh_command("lsblk | grep -c vdc.*16M || true") 4564 .unwrap() 4565 .trim() 4566 .parse::<u32>() 4567 .unwrap_or(1), 4568 0 4569 ); 4570 4571 // And add it back to validate unplug did work correctly. 4572 let (cmd_success, cmd_output) = remote_command_w_output( 4573 &api_socket, 4574 "add-disk", 4575 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4576 ); 4577 assert!(cmd_success); 4578 assert!(String::from_utf8_lossy(&cmd_output) 4579 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4580 4581 thread::sleep(std::time::Duration::new(10, 0)); 4582 4583 // Check that /dev/vdc exists and the block size is 16M. 4584 assert_eq!( 4585 guest 4586 .ssh_command("lsblk | grep vdc | grep -c 16M") 4587 .unwrap() 4588 .trim() 4589 .parse::<u32>() 4590 .unwrap_or_default(), 4591 1 4592 ); 4593 // And check the block device can be read. 4594 guest 4595 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4596 .unwrap(); 4597 4598 // Reboot the VM. 4599 guest.reboot_linux(0, None); 4600 4601 // Check still there after reboot 4602 assert_eq!( 4603 guest 4604 .ssh_command("lsblk | grep vdc | grep -c 16M") 4605 .unwrap() 4606 .trim() 4607 .parse::<u32>() 4608 .unwrap_or_default(), 4609 1 4610 ); 4611 4612 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4613 4614 thread::sleep(std::time::Duration::new(20, 0)); 4615 4616 // Check device has gone away 4617 assert_eq!( 4618 guest 4619 .ssh_command("lsblk | grep -c vdc.*16M || true") 4620 .unwrap() 4621 .trim() 4622 .parse::<u32>() 4623 .unwrap_or(1), 4624 0 4625 ); 4626 4627 guest.reboot_linux(1, None); 4628 4629 // Check device still absent 4630 assert_eq!( 4631 guest 4632 .ssh_command("lsblk | grep -c vdc.*16M || true") 4633 .unwrap() 4634 .trim() 4635 .parse::<u32>() 4636 .unwrap_or(1), 4637 0 4638 ); 4639 }); 4640 4641 let _ = child.kill(); 4642 let output = child.wait_with_output().unwrap(); 4643 4644 handle_child_output(r, &output); 4645 } 4646 4647 #[test] 4648 #[cfg(target_arch = "x86_64")] 4649 fn test_virtio_balloon() { 4650 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4651 let guest = Guest::new(Box::new(focal)); 4652 4653 let kernel_path = direct_kernel_boot_path(); 4654 4655 let api_socket = temp_api_path(&guest.tmp_dir); 4656 4657 //Let's start a 4G guest with balloon occupied 2G memory 4658 let mut child = GuestCommand::new(&guest) 4659 .args(&["--api-socket", &api_socket]) 4660 .args(&["--cpus", "boot=1"]) 4661 .args(&["--memory", "size=4G"]) 4662 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4663 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4664 .args(&["--balloon", "size=2G,deflate_on_oom=on"]) 4665 .default_disks() 4666 .default_net() 4667 .capture_output() 4668 .spawn() 4669 .unwrap(); 4670 4671 let r = std::panic::catch_unwind(|| { 4672 guest.wait_vm_boot(None).unwrap(); 4673 4674 // Wait for balloon memory's initialization and check its size. 4675 // The virtio-balloon driver might take a few seconds to report the 4676 // balloon effective size back to the VMM. 4677 thread::sleep(std::time::Duration::new(20, 0)); 4678 4679 let orig_balloon = balloon_size(&api_socket); 4680 println!("The original balloon memory size is {} bytes", orig_balloon); 4681 assert!(orig_balloon == 2147483648); 4682 4683 // Two steps to verify if the 'deflate_on_oom' parameter works. 4684 // 1st: run a command in guest to eat up memory heavily, which 4685 // will consume much more memory than $(total_mem - balloon_size) 4686 // to trigger an oom. 4687 guest 4688 .ssh_command("sudo stress --vm 25 --vm-keep --vm-bytes 1G --timeout 20") 4689 .unwrap(); 4690 4691 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 4692 let deflated_balloon = balloon_size(&api_socket); 4693 println!( 4694 "After deflating, balloon memory size is {} bytes", 4695 deflated_balloon 4696 ); 4697 // Verify the balloon size deflated 4698 assert!(deflated_balloon < 2147483648); 4699 }); 4700 4701 let _ = child.kill(); 4702 let output = child.wait_with_output().unwrap(); 4703 4704 handle_child_output(r, &output); 4705 } 4706 4707 #[test] 4708 #[cfg(target_arch = "x86_64")] 4709 fn test_pmem_hotplug() { 4710 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4711 let guest = Guest::new(Box::new(focal)); 4712 4713 let kernel_path = direct_kernel_boot_path(); 4714 4715 let api_socket = temp_api_path(&guest.tmp_dir); 4716 4717 let mut child = GuestCommand::new(&guest) 4718 .args(&["--api-socket", &api_socket]) 4719 .args(&["--cpus", "boot=1"]) 4720 .args(&["--memory", "size=512M"]) 4721 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4722 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4723 .default_disks() 4724 .default_net() 4725 .capture_output() 4726 .spawn() 4727 .unwrap(); 4728 4729 let r = std::panic::catch_unwind(|| { 4730 guest.wait_vm_boot(None).unwrap(); 4731 4732 // Check /dev/pmem0 is not there 4733 assert_eq!( 4734 guest 4735 .ssh_command("lsblk | grep -c pmem0 || true") 4736 .unwrap() 4737 .trim() 4738 .parse::<u32>() 4739 .unwrap_or(1), 4740 0 4741 ); 4742 4743 let pmem_temp_file = TempFile::new().unwrap(); 4744 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 4745 let (cmd_success, cmd_output) = remote_command_w_output( 4746 &api_socket, 4747 "add-pmem", 4748 Some(&format!( 4749 "file={},id=test0", 4750 pmem_temp_file.as_path().to_str().unwrap() 4751 )), 4752 ); 4753 assert!(cmd_success); 4754 assert!(String::from_utf8_lossy(&cmd_output) 4755 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4756 4757 // Check that /dev/pmem0 exists and the block size is 128M 4758 assert_eq!( 4759 guest 4760 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 4761 .unwrap() 4762 .trim() 4763 .parse::<u32>() 4764 .unwrap_or_default(), 4765 1 4766 ); 4767 4768 guest.reboot_linux(0, None); 4769 4770 // Check still there after reboot 4771 assert_eq!( 4772 guest 4773 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 4774 .unwrap() 4775 .trim() 4776 .parse::<u32>() 4777 .unwrap_or_default(), 4778 1 4779 ); 4780 4781 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4782 4783 thread::sleep(std::time::Duration::new(20, 0)); 4784 4785 // Check device has gone away 4786 assert_eq!( 4787 guest 4788 .ssh_command("lsblk | grep -c pmem0.*128M || true") 4789 .unwrap() 4790 .trim() 4791 .parse::<u32>() 4792 .unwrap_or(1), 4793 0 4794 ); 4795 4796 guest.reboot_linux(1, None); 4797 4798 // Check still absent after reboot 4799 assert_eq!( 4800 guest 4801 .ssh_command("lsblk | grep -c pmem0.*128M || true") 4802 .unwrap() 4803 .trim() 4804 .parse::<u32>() 4805 .unwrap_or(1), 4806 0 4807 ); 4808 }); 4809 4810 let _ = child.kill(); 4811 let output = child.wait_with_output().unwrap(); 4812 4813 handle_child_output(r, &output); 4814 } 4815 4816 #[test] 4817 #[cfg(target_arch = "x86_64")] 4818 fn test_net_hotplug() { 4819 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4820 let guest = Guest::new(Box::new(focal)); 4821 4822 let kernel_path = direct_kernel_boot_path(); 4823 4824 let api_socket = temp_api_path(&guest.tmp_dir); 4825 4826 // Boot without network 4827 let mut child = GuestCommand::new(&guest) 4828 .args(&["--api-socket", &api_socket]) 4829 .args(&["--cpus", "boot=1"]) 4830 .args(&["--memory", "size=512M"]) 4831 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4832 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4833 .default_disks() 4834 .capture_output() 4835 .spawn() 4836 .unwrap(); 4837 4838 thread::sleep(std::time::Duration::new(20, 0)); 4839 4840 let r = std::panic::catch_unwind(|| { 4841 // Add network 4842 let (cmd_success, cmd_output) = remote_command_w_output( 4843 &api_socket, 4844 "add-net", 4845 Some(guest.default_net_string().as_str()), 4846 ); 4847 assert!(cmd_success); 4848 assert!(String::from_utf8_lossy(&cmd_output) 4849 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 4850 4851 thread::sleep(std::time::Duration::new(5, 0)); 4852 4853 // 1 network interfaces + default localhost ==> 2 interfaces 4854 assert_eq!( 4855 guest 4856 .ssh_command("ip -o link | wc -l") 4857 .unwrap() 4858 .trim() 4859 .parse::<u32>() 4860 .unwrap_or_default(), 4861 2 4862 ); 4863 4864 // Remove network 4865 assert!(remote_command(&api_socket, "remove-device", Some("_net2"))); 4866 thread::sleep(std::time::Duration::new(5, 0)); 4867 4868 // Add network again 4869 let (cmd_success, cmd_output) = remote_command_w_output( 4870 &api_socket, 4871 "add-net", 4872 Some(guest.default_net_string().as_str()), 4873 ); 4874 assert!(cmd_success); 4875 assert!(String::from_utf8_lossy(&cmd_output) 4876 .contains("{\"id\":\"_net3\",\"bdf\":\"0000:00:05.0\"}")); 4877 4878 thread::sleep(std::time::Duration::new(5, 0)); 4879 4880 // 1 network interfaces + default localhost ==> 2 interfaces 4881 assert_eq!( 4882 guest 4883 .ssh_command("ip -o link | wc -l") 4884 .unwrap() 4885 .trim() 4886 .parse::<u32>() 4887 .unwrap_or_default(), 4888 2 4889 ); 4890 4891 guest.reboot_linux(0, None); 4892 4893 // Check still there after reboot 4894 // 1 network interfaces + default localhost ==> 2 interfaces 4895 assert_eq!( 4896 guest 4897 .ssh_command("ip -o link | wc -l") 4898 .unwrap() 4899 .trim() 4900 .parse::<u32>() 4901 .unwrap_or_default(), 4902 2 4903 ); 4904 }); 4905 4906 let _ = child.kill(); 4907 let output = child.wait_with_output().unwrap(); 4908 4909 handle_child_output(r, &output); 4910 } 4911 4912 #[test] 4913 fn test_initramfs() { 4914 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4915 let guest = Guest::new(Box::new(focal)); 4916 let mut workload_path = dirs::home_dir().unwrap(); 4917 workload_path.push("workloads"); 4918 4919 let mut kernels = vec![direct_kernel_boot_path()]; 4920 4921 #[cfg(target_arch = "x86_64")] 4922 { 4923 let mut pvh_kernel_path = workload_path.clone(); 4924 pvh_kernel_path.push("vmlinux"); 4925 kernels.push(pvh_kernel_path); 4926 } 4927 4928 let mut initramfs_path = workload_path; 4929 initramfs_path.push("alpine_initramfs.img"); 4930 4931 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 4932 let cmdline = format!("console=hvc0 quiet TEST_STRING={}", test_string); 4933 4934 kernels.iter().for_each(|k_path| { 4935 let mut child = GuestCommand::new(&guest) 4936 .args(&["--kernel", k_path.to_str().unwrap()]) 4937 .args(&["--initramfs", initramfs_path.to_str().unwrap()]) 4938 .args(&["--cmdline", &cmdline]) 4939 .capture_output() 4940 .spawn() 4941 .unwrap(); 4942 4943 thread::sleep(std::time::Duration::new(20, 0)); 4944 4945 let _ = child.kill(); 4946 let output = child.wait_with_output().unwrap(); 4947 4948 let r = std::panic::catch_unwind(|| { 4949 let s = String::from_utf8_lossy(&output.stdout); 4950 4951 assert_ne!(s.lines().position(|line| line == test_string), None); 4952 }); 4953 4954 handle_child_output(r, &output); 4955 }); 4956 } 4957 4958 // One thing to note about this test. The virtio-net device is heavily used 4959 // through each ssh command. There's no need to perform a dedicated test to 4960 // verify the migration went well for virtio-net. 4961 #[test] 4962 fn test_snapshot_restore() { 4963 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4964 let guest = Guest::new(Box::new(focal)); 4965 let kernel_path = direct_kernel_boot_path(); 4966 4967 let api_socket = temp_api_path(&guest.tmp_dir); 4968 4969 let net_id = "net123"; 4970 let net_params = format!( 4971 "id={},tap=,mac={},ip={},mask=255.255.255.0", 4972 net_id, guest.network.guest_mac, guest.network.host_ip 4973 ); 4974 4975 let cloudinit_params = format!( 4976 "path={},iommu=on", 4977 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4978 ); 4979 4980 let socket = temp_vsock_path(&guest.tmp_dir); 4981 4982 let mut child = GuestCommand::new(&guest) 4983 .args(&["--api-socket", &api_socket]) 4984 .args(&["--cpus", "boot=4"]) 4985 .args(&["--memory", "size=4G"]) 4986 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4987 .args(&[ 4988 "--disk", 4989 format!( 4990 "path={}", 4991 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4992 ) 4993 .as_str(), 4994 cloudinit_params.as_str(), 4995 ]) 4996 .args(&["--net", net_params.as_str()]) 4997 .args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]) 4998 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4999 .capture_output() 5000 .spawn() 5001 .unwrap(); 5002 5003 let console_text = String::from("On a branch floating down river a cricket, singing."); 5004 // Create the snapshot directory 5005 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 5006 5007 let r = std::panic::catch_unwind(|| { 5008 guest.wait_vm_boot(None).unwrap(); 5009 5010 // Check the number of vCPUs 5011 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5012 // Check the guest RAM 5013 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5014 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 5015 guest.check_devices_common(Some(&socket), Some(&console_text)); 5016 5017 // x86_64: We check that removing and adding back the virtio-net device 5018 // does not break the snapshot/restore support for virtio-pci. 5019 // This is an important thing to test as the hotplug will 5020 // trigger a PCI BAR reprogramming, which is a good way of 5021 // checking if the stored resources are correctly restored. 5022 // Unplug the virtio-net device 5023 // AArch64: Device hotplug is currently not supported, skipping here. 5024 #[cfg(target_arch = "x86_64")] 5025 { 5026 assert!(remote_command(&api_socket, "remove-device", Some(net_id),)); 5027 thread::sleep(std::time::Duration::new(10, 0)); 5028 5029 // Plug the virtio-net device again 5030 assert!(remote_command( 5031 &api_socket, 5032 "add-net", 5033 Some(net_params.as_str()), 5034 )); 5035 thread::sleep(std::time::Duration::new(10, 0)); 5036 } 5037 5038 // Pause the VM 5039 assert!(remote_command(&api_socket, "pause", None)); 5040 5041 // Take a snapshot from the VM 5042 assert!(remote_command( 5043 &api_socket, 5044 "snapshot", 5045 Some(format!("file://{}", snapshot_dir).as_str()), 5046 )); 5047 5048 // Wait to make sure the snapshot is completed 5049 thread::sleep(std::time::Duration::new(10, 0)); 5050 }); 5051 5052 // Shutdown the source VM and check console output 5053 let _ = child.kill(); 5054 let output = child.wait_with_output().unwrap(); 5055 handle_child_output(r, &output); 5056 5057 let r = std::panic::catch_unwind(|| { 5058 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5059 }); 5060 5061 handle_child_output(r, &output); 5062 5063 // Remove the vsock socket file. 5064 Command::new("rm") 5065 .arg("-f") 5066 .arg(socket.as_str()) 5067 .output() 5068 .unwrap(); 5069 5070 // Restore the VM from the snapshot 5071 let mut child = GuestCommand::new(&guest) 5072 .args(&["--api-socket", &api_socket]) 5073 .args(&[ 5074 "--restore", 5075 format!("source_url=file://{}", snapshot_dir).as_str(), 5076 ]) 5077 .capture_output() 5078 .spawn() 5079 .unwrap(); 5080 5081 // Wait for the VM to be restored 5082 thread::sleep(std::time::Duration::new(10, 0)); 5083 5084 let r = std::panic::catch_unwind(|| { 5085 // Resume the VM 5086 assert!(remote_command(&api_socket, "resume", None)); 5087 5088 // Perform same checks to validate VM has been properly restored 5089 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5090 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5091 guest.check_devices_common(Some(&socket), Some(&console_text)); 5092 }); 5093 // Shutdown the target VM and check console output 5094 let _ = child.kill(); 5095 let output = child.wait_with_output().unwrap(); 5096 handle_child_output(r, &output); 5097 5098 let r = std::panic::catch_unwind(|| { 5099 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5100 }); 5101 5102 handle_child_output(r, &output); 5103 } 5104 5105 #[test] 5106 fn test_counters() { 5107 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5108 let guest = Guest::new(Box::new(focal)); 5109 let api_socket = temp_api_path(&guest.tmp_dir); 5110 5111 let mut cmd = GuestCommand::new(&guest); 5112 cmd.args(&["--cpus", "boot=1"]) 5113 .args(&["--memory", "size=512M"]) 5114 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5115 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5116 .default_disks() 5117 .args(&["--net", guest.default_net_string().as_str()]) 5118 .args(&["--api-socket", &api_socket]) 5119 .capture_output(); 5120 5121 let mut child = cmd.spawn().unwrap(); 5122 5123 let r = std::panic::catch_unwind(|| { 5124 guest.wait_vm_boot(None).unwrap(); 5125 5126 let orig_counters = get_counters(&api_socket); 5127 guest 5128 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 5129 .unwrap(); 5130 5131 let new_counters = get_counters(&api_socket); 5132 5133 // Check that all the counters have increased 5134 assert!(new_counters > orig_counters); 5135 }); 5136 5137 let _ = child.kill(); 5138 let output = child.wait_with_output().unwrap(); 5139 5140 handle_child_output(r, &output); 5141 } 5142 5143 #[test] 5144 fn test_watchdog() { 5145 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5146 let guest = Guest::new(Box::new(focal)); 5147 let api_socket = temp_api_path(&guest.tmp_dir); 5148 5149 let kernel_path = direct_kernel_boot_path(); 5150 5151 let mut cmd = GuestCommand::new(&guest); 5152 cmd.args(&["--cpus", "boot=1"]) 5153 .args(&["--memory", "size=512M"]) 5154 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5155 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5156 .default_disks() 5157 .args(&["--net", guest.default_net_string().as_str()]) 5158 .args(&["--watchdog"]) 5159 .args(&["--api-socket", &api_socket]) 5160 .capture_output(); 5161 5162 let mut child = cmd.spawn().unwrap(); 5163 5164 let r = std::panic::catch_unwind(|| { 5165 guest.wait_vm_boot(None).unwrap(); 5166 5167 // Check for PCI device 5168 assert!(guest 5169 .does_device_vendor_pair_match("0x1063", "0x1af4") 5170 .unwrap_or_default()); 5171 5172 // Enable systemd watchdog 5173 guest 5174 .ssh_command( 5175 "echo RuntimeWatchdogSec=15s | sudo tee -a /etc/systemd/system.conf", 5176 ) 5177 .unwrap(); 5178 5179 guest.ssh_command("sudo reboot").unwrap(); 5180 5181 guest.wait_vm_boot(None).unwrap(); 5182 5183 // Check that systemd has activated the watchdog 5184 assert_eq!( 5185 guest 5186 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 5187 .unwrap() 5188 .trim() 5189 .parse::<u32>() 5190 .unwrap_or_default(), 5191 2 5192 ); 5193 5194 // Ensure that the current boot journal is written so reboot counts are valid 5195 guest.ssh_command("sudo journalctl --sync").unwrap(); 5196 5197 let boot_count = guest 5198 .ssh_command("sudo journalctl --list-boots | wc -l") 5199 .unwrap() 5200 .trim() 5201 .parse::<u32>() 5202 .unwrap_or_default(); 5203 assert_eq!(boot_count, 2); 5204 // Allow some normal time to elapse to check we don't get spurious reboots 5205 thread::sleep(std::time::Duration::new(40, 0)); 5206 5207 // Check no reboot 5208 let boot_count = guest 5209 .ssh_command("sudo journalctl --list-boots | wc -l") 5210 .unwrap() 5211 .trim() 5212 .parse::<u32>() 5213 .unwrap_or_default(); 5214 assert_eq!(boot_count, 2); 5215 5216 // Ensure that the current boot journal is written so reboot counts are valid 5217 guest.ssh_command("sudo journalctl --sync").unwrap(); 5218 5219 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 5220 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 5221 5222 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 5223 guest.wait_vm_boot(Some(50)).unwrap(); 5224 5225 // Check that watchdog triggered reboot 5226 let boot_count = guest 5227 .ssh_command("sudo journalctl --list-boots | wc -l") 5228 .unwrap() 5229 .trim() 5230 .parse::<u32>() 5231 .unwrap_or_default(); 5232 assert_eq!(boot_count, 3); 5233 5234 #[cfg(target_arch = "x86_64")] 5235 { 5236 // Now pause the VM and remain offline for 30s 5237 assert!(remote_command(&api_socket, "pause", None)); 5238 thread::sleep(std::time::Duration::new(30, 0)); 5239 assert!(remote_command(&api_socket, "resume", None)); 5240 5241 // Check no reboot 5242 let boot_count = guest 5243 .ssh_command("sudo journalctl --list-boots | wc -l") 5244 .unwrap() 5245 .trim() 5246 .parse::<u32>() 5247 .unwrap_or_default(); 5248 assert_eq!(boot_count, 3); 5249 } 5250 }); 5251 5252 let _ = child.kill(); 5253 let output = child.wait_with_output().unwrap(); 5254 5255 handle_child_output(r, &output); 5256 } 5257 5258 #[test] 5259 fn test_tap_from_fd() { 5260 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5261 let guest = Guest::new(Box::new(focal)); 5262 let kernel_path = direct_kernel_boot_path(); 5263 5264 // Create a TAP interface with multi-queue enabled 5265 let num_queue_pairs: usize = 2; 5266 5267 use std::str::FromStr; 5268 let taps = net_util::open_tap( 5269 Some("chtap0"), 5270 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 5271 None, 5272 &mut None, 5273 num_queue_pairs, 5274 Some(libc::O_RDWR | libc::O_NONBLOCK), 5275 ) 5276 .unwrap(); 5277 5278 let mut child = GuestCommand::new(&guest) 5279 .args(&["--cpus", &format!("boot={}", num_queue_pairs)]) 5280 .args(&["--memory", "size=512M"]) 5281 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5282 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5283 .default_disks() 5284 .args(&[ 5285 "--net", 5286 &format!( 5287 "fd={}:{},mac={},num_queues={}", 5288 taps[0].as_raw_fd(), 5289 taps[1].as_raw_fd(), 5290 guest.network.guest_mac, 5291 num_queue_pairs * 2 5292 ), 5293 ]) 5294 .capture_output() 5295 .spawn() 5296 .unwrap(); 5297 5298 let r = std::panic::catch_unwind(|| { 5299 guest.wait_vm_boot(None).unwrap(); 5300 5301 assert_eq!( 5302 guest 5303 .ssh_command("ip -o link | wc -l") 5304 .unwrap() 5305 .trim() 5306 .parse::<u32>() 5307 .unwrap_or_default(), 5308 2 5309 ); 5310 5311 guest.reboot_linux(0, None); 5312 5313 assert_eq!( 5314 guest 5315 .ssh_command("ip -o link | wc -l") 5316 .unwrap() 5317 .trim() 5318 .parse::<u32>() 5319 .unwrap_or_default(), 5320 2 5321 ); 5322 }); 5323 5324 let _ = child.kill(); 5325 let output = child.wait_with_output().unwrap(); 5326 5327 handle_child_output(r, &output); 5328 } 5329 5330 // By design, a guest VM won't be able to connect to the host 5331 // machine when using a macvtap network interface (while it can 5332 // communicate externally). As a workaround, this integration 5333 // test creates two macvtap interfaces in 'bridge' mode on the 5334 // same physical net interface, one for the guest and one for 5335 // the host. With additional setup on the IP address and the 5336 // routing table, it enables the communications between the 5337 // guest VM and the host machine. 5338 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 5339 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 5340 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5341 let guest = Guest::new(Box::new(focal)); 5342 let api_socket = temp_api_path(&guest.tmp_dir); 5343 let kernel_path = direct_kernel_boot_path(); 5344 let phy_net = "eth0"; 5345 5346 // Create a macvtap interface for the guest VM to use 5347 assert!(exec_host_command_status(&format!( 5348 "sudo ip link add link {} name {} type macvtap mod bridge", 5349 phy_net, guest_macvtap_name 5350 )) 5351 .success()); 5352 assert!(exec_host_command_status(&format!( 5353 "sudo ip link set {} address {} up", 5354 guest_macvtap_name, guest.network.guest_mac 5355 )) 5356 .success()); 5357 assert!( 5358 exec_host_command_status(&format!("sudo ip link show {}", guest_macvtap_name)) 5359 .success() 5360 ); 5361 5362 let tap_index = 5363 fs::read_to_string(&format!("/sys/class/net/{}/ifindex", guest_macvtap_name)) 5364 .unwrap(); 5365 let tap_device = format!("/dev/tap{}", tap_index.trim()); 5366 5367 assert!( 5368 exec_host_command_status(&format!("sudo chown $UID.$UID {}", tap_device)).success() 5369 ); 5370 5371 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 5372 let tap_fd = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 5373 assert!(tap_fd > 0); 5374 5375 // Create a macvtap on the same physical net interface for 5376 // the host machine to use 5377 assert!(exec_host_command_status(&format!( 5378 "sudo ip link add link {} name {} type macvtap mod bridge", 5379 phy_net, host_macvtap_name 5380 )) 5381 .success()); 5382 // Use default mask "255.255.255.0" 5383 assert!(exec_host_command_status(&format!( 5384 "sudo ip address add {}/24 dev {}", 5385 guest.network.host_ip, host_macvtap_name 5386 )) 5387 .success()); 5388 assert!(exec_host_command_status(&format!( 5389 "sudo ip link set dev {} up", 5390 host_macvtap_name 5391 )) 5392 .success()); 5393 5394 let mut guest_command = GuestCommand::new(&guest); 5395 guest_command 5396 .args(&["--cpus", "boot=1"]) 5397 .args(&["--memory", "size=512M"]) 5398 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5399 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5400 .default_disks() 5401 .args(&["--api-socket", &api_socket]); 5402 5403 let net_params = format!("fd={},mac={}", tap_fd, guest.network.guest_mac); 5404 5405 if !hotplug { 5406 guest_command.args(&["--net", &net_params]); 5407 } 5408 5409 let mut child = guest_command.capture_output().spawn().unwrap(); 5410 5411 if hotplug { 5412 // Give some time to the VMM process to listen to the API 5413 // socket. This is the only requirement to avoid the following 5414 // call to ch-remote from failing. 5415 thread::sleep(std::time::Duration::new(10, 0)); 5416 // Hotplug the virtio-net device 5417 let (cmd_success, cmd_output) = 5418 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 5419 assert!(cmd_success); 5420 assert!(String::from_utf8_lossy(&cmd_output) 5421 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 5422 } 5423 5424 // The functional connectivity provided by the virtio-net device 5425 // gets tested through wait_vm_boot() as it expects to receive a 5426 // HTTP request, and through the SSH command as well. 5427 let r = std::panic::catch_unwind(|| { 5428 guest.wait_vm_boot(None).unwrap(); 5429 5430 assert_eq!( 5431 guest 5432 .ssh_command("ip -o link | wc -l") 5433 .unwrap() 5434 .trim() 5435 .parse::<u32>() 5436 .unwrap_or_default(), 5437 2 5438 ); 5439 }); 5440 5441 let _ = child.kill(); 5442 5443 exec_host_command_status(&format!("sudo ip link del {}", guest_macvtap_name)); 5444 exec_host_command_status(&format!("sudo ip link del {}", host_macvtap_name)); 5445 5446 let output = child.wait_with_output().unwrap(); 5447 5448 handle_child_output(r, &output); 5449 } 5450 5451 #[test] 5452 fn test_macvtap() { 5453 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 5454 } 5455 5456 #[test] 5457 #[cfg(target_arch = "x86_64")] 5458 fn test_macvtap_hotplug() { 5459 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 5460 } 5461 5462 #[test] 5463 fn test_ovs_dpdk() { 5464 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5465 let guest1 = Guest::new(Box::new(focal1)); 5466 5467 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5468 let guest2 = Guest::new(Box::new(focal2)); 5469 let api_socket = temp_api_path(&guest2.tmp_dir); 5470 5471 let (mut child1, mut child2) = setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket); 5472 5473 // Create the snapshot directory 5474 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 5475 5476 let r = std::panic::catch_unwind(|| { 5477 // Remove one of the two ports from the OVS bridge 5478 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 5479 5480 // Spawn a new netcat listener in the first VM 5481 let guest_ip = guest1.network.guest_ip.clone(); 5482 thread::spawn(move || { 5483 ssh_command_ip( 5484 "nc -l 12345", 5485 &guest_ip, 5486 DEFAULT_SSH_RETRIES, 5487 DEFAULT_SSH_TIMEOUT, 5488 ) 5489 .unwrap(); 5490 }); 5491 5492 // Wait for the server to be listening 5493 thread::sleep(std::time::Duration::new(5, 0)); 5494 5495 // Check the connection fails this time 5496 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 5497 5498 // Add the OVS port back 5499 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 5500 5501 // And finally check the connection is functional again 5502 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 5503 5504 // Pause the VM 5505 assert!(remote_command(&api_socket, "pause", None)); 5506 5507 // Take a snapshot from the VM 5508 assert!(remote_command( 5509 &api_socket, 5510 "snapshot", 5511 Some(format!("file://{}", snapshot_dir).as_str()), 5512 )); 5513 5514 // Wait to make sure the snapshot is completed 5515 thread::sleep(std::time::Duration::new(10, 0)); 5516 }); 5517 5518 // Shutdown the source VM 5519 let _ = child2.kill(); 5520 let output = child2.wait_with_output().unwrap(); 5521 handle_child_output(r, &output); 5522 5523 // Remove the vhost-user socket file. 5524 Command::new("rm") 5525 .arg("-f") 5526 .arg("/tmp/dpdkvhostclient2") 5527 .output() 5528 .unwrap(); 5529 5530 // Restore the VM from the snapshot 5531 let mut child2 = GuestCommand::new(&guest2) 5532 .args(&["--api-socket", &api_socket]) 5533 .args(&[ 5534 "--restore", 5535 format!("source_url=file://{}", snapshot_dir).as_str(), 5536 ]) 5537 .capture_output() 5538 .spawn() 5539 .unwrap(); 5540 5541 // Wait for the VM to be restored 5542 thread::sleep(std::time::Duration::new(10, 0)); 5543 5544 let r = std::panic::catch_unwind(|| { 5545 // Resume the VM 5546 assert!(remote_command(&api_socket, "resume", None)); 5547 5548 // Spawn a new netcat listener in the first VM 5549 let guest_ip = guest1.network.guest_ip.clone(); 5550 thread::spawn(move || { 5551 ssh_command_ip( 5552 "nc -l 12345", 5553 &guest_ip, 5554 DEFAULT_SSH_RETRIES, 5555 DEFAULT_SSH_TIMEOUT, 5556 ) 5557 .unwrap(); 5558 }); 5559 5560 // Wait for the server to be listening 5561 thread::sleep(std::time::Duration::new(5, 0)); 5562 5563 // And check the connection is still functional after restore 5564 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 5565 }); 5566 5567 cleanup_ovs_dpdk(); 5568 5569 let _ = child1.kill(); 5570 let _ = child2.kill(); 5571 5572 let output = child1.wait_with_output().unwrap(); 5573 child2.wait().unwrap(); 5574 5575 handle_child_output(r, &output); 5576 } 5577 } 5578 5579 mod sequential { 5580 use crate::tests::*; 5581 5582 #[test] 5583 fn test_memory_mergeable_on() { 5584 test_memory_mergeable(true) 5585 } 5586 } 5587 5588 #[cfg(target_arch = "x86_64")] 5589 mod windows { 5590 use crate::tests::*; 5591 5592 lazy_static! { 5593 static ref NEXT_DISK_ID: Mutex<u8> = Mutex::new(1); 5594 } 5595 5596 struct WindowsGuest { 5597 guest: Guest, 5598 auth: PasswordAuth, 5599 } 5600 5601 trait FsType { 5602 const FS_FAT: u8; 5603 const FS_NTFS: u8; 5604 } 5605 impl FsType for WindowsGuest { 5606 const FS_FAT: u8 = 0; 5607 const FS_NTFS: u8 = 1; 5608 } 5609 5610 impl WindowsGuest { 5611 fn new() -> Self { 5612 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 5613 let guest = Guest::new(Box::new(disk)); 5614 let auth = PasswordAuth { 5615 username: String::from("administrator"), 5616 password: String::from("Admin123"), 5617 }; 5618 5619 WindowsGuest { guest, auth } 5620 } 5621 5622 fn guest(&self) -> &Guest { 5623 &self.guest 5624 } 5625 5626 fn ssh_cmd(&self, cmd: &str) -> String { 5627 ssh_command_ip_with_auth( 5628 cmd, 5629 &self.auth, 5630 &self.guest.network.guest_ip, 5631 DEFAULT_SSH_RETRIES, 5632 DEFAULT_SSH_TIMEOUT, 5633 ) 5634 .unwrap() 5635 } 5636 5637 fn cpu_count(&self) -> u8 { 5638 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 5639 .trim() 5640 .parse::<u8>() 5641 .unwrap_or(0) 5642 } 5643 5644 fn ram_size(&self) -> usize { 5645 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 5646 .trim() 5647 .parse::<usize>() 5648 .unwrap_or(0) 5649 } 5650 5651 fn netdev_count(&self) -> u8 { 5652 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 5653 .trim() 5654 .parse::<u8>() 5655 .unwrap_or(0) 5656 } 5657 5658 fn disk_count(&self) -> u8 { 5659 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 5660 .trim() 5661 .parse::<u8>() 5662 .unwrap_or(0) 5663 } 5664 5665 fn reboot(&self) { 5666 let _ = self.ssh_cmd("shutdown /r /t 0"); 5667 } 5668 5669 fn shutdown(&self) { 5670 let _ = self.ssh_cmd("shutdown /s /t 0"); 5671 } 5672 5673 fn run_dnsmasq(&self) -> std::process::Child { 5674 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 5675 let dhcp_host = format!( 5676 "--dhcp-host={},{}", 5677 self.guest.network.guest_mac, self.guest.network.guest_ip 5678 ); 5679 let dhcp_range = format!( 5680 "--dhcp-range=eth,{},{}", 5681 self.guest.network.guest_ip, self.guest.network.guest_ip 5682 ); 5683 5684 Command::new("dnsmasq") 5685 .arg("--no-daemon") 5686 .arg("--log-queries") 5687 .arg(listen_address.as_str()) 5688 .arg("--except-interface=lo") 5689 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 5690 .arg("--conf-file=/dev/null") 5691 .arg(dhcp_host.as_str()) 5692 .arg(dhcp_range.as_str()) 5693 .spawn() 5694 .unwrap() 5695 } 5696 5697 // TODO Cleanup image file explicitly after test, if there's some space issues. 5698 fn disk_new(&self, fs: u8, sz: usize) -> String { 5699 let mut guard = NEXT_DISK_ID.lock().unwrap(); 5700 let id = *guard; 5701 *guard = id + 1; 5702 5703 let img = PathBuf::from(format!("/tmp/test-hotplug-{}.raw", id)); 5704 let _ = fs::remove_file(&img); 5705 5706 // Create an image file 5707 let out = Command::new("qemu-img") 5708 .args(&[ 5709 "create", 5710 "-f", 5711 "raw", 5712 img.to_str().unwrap(), 5713 format!("{}m", sz).as_str(), 5714 ]) 5715 .output() 5716 .expect("qemu-img command failed") 5717 .stdout; 5718 println!("{:?}", out); 5719 5720 // Associate image to a loop device 5721 let out = Command::new("losetup") 5722 .args(&["--show", "-f", img.to_str().unwrap()]) 5723 .output() 5724 .expect("failed to create loop device") 5725 .stdout; 5726 let _tmp = String::from_utf8_lossy(&out); 5727 let loop_dev = _tmp.trim(); 5728 println!("{:?}", out); 5729 5730 // Create a partition table 5731 // echo 'type=7' | sudo sfdisk "${LOOP}" 5732 let mut child = Command::new("sfdisk") 5733 .args(&[loop_dev]) 5734 .stdin(Stdio::piped()) 5735 .spawn() 5736 .unwrap(); 5737 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 5738 let _ = stdin 5739 .write_all("type=7".as_bytes()) 5740 .expect("failed to write stdin"); 5741 let out = child.wait_with_output().expect("sfdisk failed").stdout; 5742 println!("{:?}", out); 5743 5744 // Disengage the loop device 5745 let out = Command::new("losetup") 5746 .args(&["-d", loop_dev]) 5747 .output() 5748 .expect("loop device not found") 5749 .stdout; 5750 println!("{:?}", out); 5751 5752 // Re-associate loop device pointing to the partition only 5753 let out = Command::new("losetup") 5754 .args(&[ 5755 "--show", 5756 "--offset", 5757 (512 * 2048).to_string().as_str(), 5758 "-f", 5759 img.to_str().unwrap(), 5760 ]) 5761 .output() 5762 .expect("failed to create loop device") 5763 .stdout; 5764 let _tmp = String::from_utf8_lossy(&out); 5765 let loop_dev = _tmp.trim(); 5766 println!("{:?}", out); 5767 5768 // Create filesystem. 5769 let fs_cmd = match fs { 5770 WindowsGuest::FS_FAT => "mkfs.msdos", 5771 WindowsGuest::FS_NTFS => "mkfs.ntfs", 5772 _ => panic!("Unknown filesystem type '{}'", fs), 5773 }; 5774 let out = Command::new(fs_cmd) 5775 .args(&[&loop_dev]) 5776 .output() 5777 .unwrap_or_else(|_| panic!("{} failed", fs_cmd)) 5778 .stdout; 5779 println!("{:?}", out); 5780 5781 // Disengage the loop device 5782 let out = Command::new("losetup") 5783 .args(&["-d", loop_dev]) 5784 .output() 5785 .unwrap_or_else(|_| panic!("loop device '{}' not found", loop_dev)) 5786 .stdout; 5787 println!("{:?}", out); 5788 5789 img.to_str().unwrap().to_string() 5790 } 5791 5792 fn disks_set_rw(&self) { 5793 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 5794 } 5795 5796 fn disks_online(&self) { 5797 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 5798 } 5799 5800 fn disk_file_put(&self, fname: &str, data: &str) { 5801 let _ = self.ssh_cmd(&format!( 5802 "powershell -Command \"'{}' | Set-Content -Path {}\"", 5803 data, fname 5804 )); 5805 } 5806 5807 fn disk_file_read(&self, fname: &str) -> String { 5808 self.ssh_cmd(&format!( 5809 "powershell -Command \"Get-Content -Path {}\"", 5810 fname 5811 )) 5812 } 5813 5814 fn wait_for_boot(&self) -> bool { 5815 let cmd = "dir /b c:\\ | find \"Windows\""; 5816 let tmo_max = 180; 5817 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 5818 // interval must be small. 5819 let tmo_int = 2; 5820 let out = ssh_command_ip_with_auth( 5821 cmd, 5822 &self.auth, 5823 &self.guest.network.guest_ip, 5824 { 5825 let mut ret = 1; 5826 let mut tmo_acc = 0; 5827 loop { 5828 tmo_acc += tmo_int * ret; 5829 if tmo_acc >= tmo_max { 5830 break; 5831 } 5832 ret += 1; 5833 } 5834 ret 5835 }, 5836 tmo_int, 5837 ) 5838 .unwrap(); 5839 5840 if "Windows" == out.trim() { 5841 return true; 5842 } 5843 5844 false 5845 } 5846 } 5847 5848 fn vcpu_threads_count(pid: u32) -> u8 { 5849 // ps -T -p 12345 | grep vcpu | wc -l 5850 let out = Command::new("ps") 5851 .args(&["-T", "-p", format!("{}", pid).as_str()]) 5852 .output() 5853 .expect("ps command failed") 5854 .stdout; 5855 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 5856 } 5857 5858 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 5859 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 5860 let out = Command::new("ps") 5861 .args(&["-T", "-p", format!("{}", pid).as_str()]) 5862 .output() 5863 .expect("ps command failed") 5864 .stdout; 5865 let mut n = 0; 5866 String::from_utf8_lossy(&out) 5867 .split_whitespace() 5868 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 5869 n 5870 } 5871 5872 fn disk_ctrl_threads_count(pid: u32) -> u8 { 5873 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 5874 let out = Command::new("ps") 5875 .args(&["-T", "-p", format!("{}", pid).as_str()]) 5876 .output() 5877 .expect("ps command failed") 5878 .stdout; 5879 let mut n = 0; 5880 String::from_utf8_lossy(&out) 5881 .split_whitespace() 5882 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 5883 n 5884 } 5885 5886 #[test] 5887 fn test_windows_guest() { 5888 let windows_guest = WindowsGuest::new(); 5889 5890 let mut ovmf_path = dirs::home_dir().unwrap(); 5891 ovmf_path.push("workloads"); 5892 ovmf_path.push(OVMF_NAME); 5893 5894 let mut child = GuestCommand::new(windows_guest.guest()) 5895 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 5896 .args(&["--memory", "size=4G"]) 5897 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 5898 .args(&["--serial", "tty"]) 5899 .args(&["--console", "off"]) 5900 .default_disks() 5901 .default_net() 5902 .capture_output() 5903 .spawn() 5904 .unwrap(); 5905 5906 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 5907 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5908 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 5909 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5910 5911 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 5912 5913 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 5914 5915 let r = std::panic::catch_unwind(|| { 5916 // Wait to make sure Windows boots up 5917 assert!(windows_guest.wait_for_boot()); 5918 5919 windows_guest.shutdown(); 5920 }); 5921 5922 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 5923 let _ = child.kill(); 5924 let output = child.wait_with_output().unwrap(); 5925 5926 let _ = child_dnsmasq.kill(); 5927 let _ = child_dnsmasq.wait(); 5928 5929 handle_child_output(r, &output); 5930 } 5931 5932 #[test] 5933 fn test_windows_guest_multiple_queues() { 5934 let windows_guest = WindowsGuest::new(); 5935 5936 let mut ovmf_path = dirs::home_dir().unwrap(); 5937 ovmf_path.push("workloads"); 5938 ovmf_path.push(OVMF_NAME); 5939 5940 let mut child = GuestCommand::new(windows_guest.guest()) 5941 .args(&["--cpus", "boot=4,kvm_hyperv=on"]) 5942 .args(&["--memory", "size=4G"]) 5943 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 5944 .args(&["--serial", "tty"]) 5945 .args(&["--console", "off"]) 5946 .args(&[ 5947 "--disk", 5948 format!( 5949 "path={},num_queues=4", 5950 windows_guest 5951 .guest() 5952 .disk_config 5953 .disk(DiskType::OperatingSystem) 5954 .unwrap() 5955 ) 5956 .as_str(), 5957 ]) 5958 .args(&[ 5959 "--net", 5960 format!( 5961 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 5962 windows_guest.guest().network.guest_mac, 5963 windows_guest.guest().network.host_ip 5964 ) 5965 .as_str(), 5966 ]) 5967 .capture_output() 5968 .spawn() 5969 .unwrap(); 5970 5971 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 5972 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5973 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 5974 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5975 5976 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 5977 5978 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 5979 5980 let r = std::panic::catch_unwind(|| { 5981 // Wait to make sure Windows boots up 5982 assert!(windows_guest.wait_for_boot()); 5983 5984 windows_guest.shutdown(); 5985 }); 5986 5987 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 5988 let _ = child.kill(); 5989 let output = child.wait_with_output().unwrap(); 5990 5991 let _ = child_dnsmasq.kill(); 5992 let _ = child_dnsmasq.wait(); 5993 5994 handle_child_output(r, &output); 5995 } 5996 5997 #[test] 5998 #[cfg(not(feature = "mshv"))] 5999 fn test_windows_guest_snapshot_restore() { 6000 let windows_guest = WindowsGuest::new(); 6001 6002 let mut ovmf_path = dirs::home_dir().unwrap(); 6003 ovmf_path.push("workloads"); 6004 ovmf_path.push(OVMF_NAME); 6005 6006 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6007 let api_socket = temp_api_path(&tmp_dir); 6008 6009 let mut child = GuestCommand::new(windows_guest.guest()) 6010 .args(&["--api-socket", &api_socket]) 6011 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6012 .args(&["--memory", "size=4G"]) 6013 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6014 .args(&["--serial", "tty"]) 6015 .args(&["--console", "off"]) 6016 .default_disks() 6017 .default_net() 6018 .capture_output() 6019 .spawn() 6020 .unwrap(); 6021 6022 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 6023 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6024 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 6025 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6026 6027 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 6028 6029 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6030 6031 // Wait to make sure Windows boots up 6032 assert!(windows_guest.wait_for_boot()); 6033 6034 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 6035 6036 // Pause the VM 6037 assert!(remote_command(&api_socket, "pause", None)); 6038 6039 // Take a snapshot from the VM 6040 assert!(remote_command( 6041 &api_socket, 6042 "snapshot", 6043 Some(format!("file://{}", snapshot_dir).as_str()), 6044 )); 6045 6046 // Wait to make sure the snapshot is completed 6047 thread::sleep(std::time::Duration::new(30, 0)); 6048 6049 let _ = child.kill(); 6050 child.wait().unwrap(); 6051 6052 // Restore the VM from the snapshot 6053 let mut child = GuestCommand::new(windows_guest.guest()) 6054 .args(&["--api-socket", &api_socket]) 6055 .args(&[ 6056 "--restore", 6057 format!("source_url=file://{}", snapshot_dir).as_str(), 6058 ]) 6059 .capture_output() 6060 .spawn() 6061 .unwrap(); 6062 6063 // Wait for the VM to be restored 6064 thread::sleep(std::time::Duration::new(20, 0)); 6065 6066 let r = std::panic::catch_unwind(|| { 6067 // Resume the VM 6068 assert!(remote_command(&api_socket, "resume", None)); 6069 6070 windows_guest.shutdown(); 6071 }); 6072 6073 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6074 let _ = child.kill(); 6075 let output = child.wait_with_output().unwrap(); 6076 6077 let _ = child_dnsmasq.kill(); 6078 let _ = child_dnsmasq.wait(); 6079 6080 handle_child_output(r, &output); 6081 } 6082 6083 #[test] 6084 #[cfg(not(feature = "mshv"))] 6085 fn test_windows_guest_cpu_hotplug() { 6086 let windows_guest = WindowsGuest::new(); 6087 6088 let mut ovmf_path = dirs::home_dir().unwrap(); 6089 ovmf_path.push("workloads"); 6090 ovmf_path.push(OVMF_NAME); 6091 6092 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6093 let api_socket = temp_api_path(&tmp_dir); 6094 6095 let mut child = GuestCommand::new(windows_guest.guest()) 6096 .args(&["--api-socket", &api_socket]) 6097 .args(&["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 6098 .args(&["--memory", "size=4G"]) 6099 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6100 .args(&["--serial", "tty"]) 6101 .args(&["--console", "off"]) 6102 .default_disks() 6103 .default_net() 6104 .capture_output() 6105 .spawn() 6106 .unwrap(); 6107 6108 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6109 6110 let r = std::panic::catch_unwind(|| { 6111 // Wait to make sure Windows boots up 6112 assert!(windows_guest.wait_for_boot()); 6113 6114 let vcpu_num = 2; 6115 // Check the initial number of CPUs the guest sees 6116 assert_eq!(windows_guest.cpu_count(), vcpu_num); 6117 // Check the initial number of vcpu threads in the CH process 6118 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 6119 6120 let vcpu_num = 6; 6121 // Hotplug some CPUs 6122 resize_command(&api_socket, Some(vcpu_num), None, None); 6123 // Wait to make sure CPUs are added 6124 thread::sleep(std::time::Duration::new(10, 0)); 6125 // Check the guest sees the correct number 6126 assert_eq!(windows_guest.cpu_count(), vcpu_num); 6127 // Check the CH process has the correct number of vcpu threads 6128 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 6129 6130 let vcpu_num = 4; 6131 // Remove some CPUs. Note that Windows doesn't support hot-remove. 6132 resize_command(&api_socket, Some(vcpu_num), None, None); 6133 // Wait to make sure CPUs are removed 6134 thread::sleep(std::time::Duration::new(10, 0)); 6135 // Reboot to let Windows catch up 6136 windows_guest.reboot(); 6137 // Wait to make sure Windows completely rebooted 6138 thread::sleep(std::time::Duration::new(60, 0)); 6139 // Check the guest sees the correct number 6140 assert_eq!(windows_guest.cpu_count(), vcpu_num); 6141 // Check the CH process has the correct number of vcpu threads 6142 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 6143 6144 windows_guest.shutdown(); 6145 }); 6146 6147 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6148 let _ = child.kill(); 6149 let output = child.wait_with_output().unwrap(); 6150 6151 let _ = child_dnsmasq.kill(); 6152 let _ = child_dnsmasq.wait(); 6153 6154 handle_child_output(r, &output); 6155 } 6156 6157 #[test] 6158 #[cfg(not(feature = "mshv"))] 6159 fn test_windows_guest_ram_hotplug() { 6160 let windows_guest = WindowsGuest::new(); 6161 6162 let mut ovmf_path = dirs::home_dir().unwrap(); 6163 ovmf_path.push("workloads"); 6164 ovmf_path.push(OVMF_NAME); 6165 6166 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6167 let api_socket = temp_api_path(&tmp_dir); 6168 6169 let mut child = GuestCommand::new(windows_guest.guest()) 6170 .args(&["--api-socket", &api_socket]) 6171 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6172 .args(&["--memory", "size=2G,hotplug_size=5G"]) 6173 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6174 .args(&["--serial", "tty"]) 6175 .args(&["--console", "off"]) 6176 .default_disks() 6177 .default_net() 6178 .capture_output() 6179 .spawn() 6180 .unwrap(); 6181 6182 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6183 6184 let r = std::panic::catch_unwind(|| { 6185 // Wait to make sure Windows boots up 6186 assert!(windows_guest.wait_for_boot()); 6187 6188 let ram_size = 2 * 1024 * 1024 * 1024; 6189 // Check the initial number of RAM the guest sees 6190 let current_ram_size = windows_guest.ram_size(); 6191 // This size seems to be reserved by the system and thus the 6192 // reported amount differs by this constant value. 6193 let reserved_ram_size = ram_size - current_ram_size; 6194 // Verify that there's not more than 4mb constant diff wasted 6195 // by the reserved ram. 6196 assert!(reserved_ram_size < 4 * 1024 * 1024); 6197 6198 let ram_size = 4 * 1024 * 1024 * 1024; 6199 // Hotplug some RAM 6200 resize_command(&api_socket, None, Some(ram_size), None); 6201 // Wait to make sure RAM has been added 6202 thread::sleep(std::time::Duration::new(10, 0)); 6203 // Check the guest sees the correct number 6204 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 6205 6206 let ram_size = 3 * 1024 * 1024 * 1024; 6207 // Unplug some RAM. Note that hot-remove most likely won't work. 6208 resize_command(&api_socket, None, Some(ram_size), None); 6209 // Wait to make sure RAM has been added 6210 thread::sleep(std::time::Duration::new(10, 0)); 6211 // Reboot to let Windows catch up 6212 windows_guest.reboot(); 6213 // Wait to make sure guest completely rebooted 6214 thread::sleep(std::time::Duration::new(60, 0)); 6215 // Check the guest sees the correct number 6216 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 6217 6218 windows_guest.shutdown(); 6219 }); 6220 6221 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6222 let _ = child.kill(); 6223 let output = child.wait_with_output().unwrap(); 6224 6225 let _ = child_dnsmasq.kill(); 6226 let _ = child_dnsmasq.wait(); 6227 6228 handle_child_output(r, &output); 6229 } 6230 6231 #[test] 6232 #[cfg(not(feature = "mshv"))] 6233 fn test_windows_guest_netdev_hotplug() { 6234 let windows_guest = WindowsGuest::new(); 6235 6236 let mut ovmf_path = dirs::home_dir().unwrap(); 6237 ovmf_path.push("workloads"); 6238 ovmf_path.push(OVMF_NAME); 6239 6240 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6241 let api_socket = temp_api_path(&tmp_dir); 6242 6243 let mut child = GuestCommand::new(windows_guest.guest()) 6244 .args(&["--api-socket", &api_socket]) 6245 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6246 .args(&["--memory", "size=4G"]) 6247 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6248 .args(&["--serial", "tty"]) 6249 .args(&["--console", "off"]) 6250 .default_disks() 6251 .default_net() 6252 .capture_output() 6253 .spawn() 6254 .unwrap(); 6255 6256 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6257 6258 let r = std::panic::catch_unwind(|| { 6259 // Wait to make sure Windows boots up 6260 assert!(windows_guest.wait_for_boot()); 6261 6262 // Initially present network device 6263 let netdev_num = 1; 6264 assert_eq!(windows_guest.netdev_count(), netdev_num); 6265 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6266 6267 // Hotplug network device 6268 let (cmd_success, cmd_output) = remote_command_w_output( 6269 &api_socket, 6270 "add-net", 6271 Some(windows_guest.guest().default_net_string().as_str()), 6272 ); 6273 assert!(cmd_success); 6274 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 6275 thread::sleep(std::time::Duration::new(5, 0)); 6276 // Verify the device is on the system 6277 let netdev_num = 2; 6278 assert_eq!(windows_guest.netdev_count(), netdev_num); 6279 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6280 6281 // Remove network device 6282 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 6283 assert!(cmd_success); 6284 thread::sleep(std::time::Duration::new(5, 0)); 6285 // Verify the device has been removed 6286 let netdev_num = 1; 6287 assert_eq!(windows_guest.netdev_count(), netdev_num); 6288 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6289 6290 windows_guest.shutdown(); 6291 }); 6292 6293 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6294 let _ = child.kill(); 6295 let output = child.wait_with_output().unwrap(); 6296 6297 let _ = child_dnsmasq.kill(); 6298 let _ = child_dnsmasq.wait(); 6299 6300 handle_child_output(r, &output); 6301 } 6302 6303 #[test] 6304 #[cfg(not(feature = "mshv"))] 6305 fn test_windows_guest_disk_hotplug() { 6306 let windows_guest = WindowsGuest::new(); 6307 6308 let mut ovmf_path = dirs::home_dir().unwrap(); 6309 ovmf_path.push("workloads"); 6310 ovmf_path.push(OVMF_NAME); 6311 6312 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6313 let api_socket = temp_api_path(&tmp_dir); 6314 6315 let mut child = GuestCommand::new(windows_guest.guest()) 6316 .args(&["--api-socket", &api_socket]) 6317 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6318 .args(&["--memory", "size=4G"]) 6319 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6320 .args(&["--serial", "tty"]) 6321 .args(&["--console", "off"]) 6322 .default_disks() 6323 .default_net() 6324 .capture_output() 6325 .spawn() 6326 .unwrap(); 6327 6328 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6329 6330 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 6331 6332 let r = std::panic::catch_unwind(|| { 6333 // Wait to make sure Windows boots up 6334 assert!(windows_guest.wait_for_boot()); 6335 6336 // Initially present disk device 6337 let disk_num = 1; 6338 assert_eq!(windows_guest.disk_count(), disk_num); 6339 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6340 6341 // Hotplug disk device 6342 let (cmd_success, cmd_output) = remote_command_w_output( 6343 &api_socket, 6344 "add-disk", 6345 Some(format!("path={},readonly=off", disk).as_str()), 6346 ); 6347 assert!(cmd_success); 6348 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 6349 thread::sleep(std::time::Duration::new(5, 0)); 6350 // Online disk device 6351 windows_guest.disks_set_rw(); 6352 windows_guest.disks_online(); 6353 // Verify the device is on the system 6354 let disk_num = 2; 6355 assert_eq!(windows_guest.disk_count(), disk_num); 6356 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6357 6358 let data = "hello"; 6359 let fname = "d:\\world"; 6360 windows_guest.disk_file_put(fname, data); 6361 6362 // Unmount disk device 6363 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 6364 assert!(cmd_success); 6365 thread::sleep(std::time::Duration::new(5, 0)); 6366 // Verify the device has been removed 6367 let disk_num = 1; 6368 assert_eq!(windows_guest.disk_count(), disk_num); 6369 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6370 6371 // Remount and check the file exists with the expected contents 6372 let (cmd_success, _cmd_output) = remote_command_w_output( 6373 &api_socket, 6374 "add-disk", 6375 Some(format!("path={},readonly=off", disk).as_str()), 6376 ); 6377 assert!(cmd_success); 6378 thread::sleep(std::time::Duration::new(5, 0)); 6379 let out = windows_guest.disk_file_read(fname); 6380 assert_eq!(data, out.trim()); 6381 6382 // Intentionally no unmount, it'll happen at shutdown. 6383 6384 windows_guest.shutdown(); 6385 }); 6386 6387 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6388 let _ = child.kill(); 6389 let output = child.wait_with_output().unwrap(); 6390 6391 let _ = child_dnsmasq.kill(); 6392 let _ = child_dnsmasq.wait(); 6393 6394 handle_child_output(r, &output); 6395 } 6396 6397 #[test] 6398 #[cfg(not(feature = "mshv"))] 6399 fn test_windows_guest_disk_hotplug_multi() { 6400 let windows_guest = WindowsGuest::new(); 6401 6402 let mut ovmf_path = dirs::home_dir().unwrap(); 6403 ovmf_path.push("workloads"); 6404 ovmf_path.push(OVMF_NAME); 6405 6406 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6407 let api_socket = temp_api_path(&tmp_dir); 6408 6409 let mut child = GuestCommand::new(windows_guest.guest()) 6410 .args(&["--api-socket", &api_socket]) 6411 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6412 .args(&["--memory", "size=2G"]) 6413 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6414 .args(&["--serial", "tty"]) 6415 .args(&["--console", "off"]) 6416 .default_disks() 6417 .default_net() 6418 .capture_output() 6419 .spawn() 6420 .unwrap(); 6421 6422 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6423 6424 // Predefined data to used at various test stages 6425 let disk_test_data: [[String; 4]; 2] = [ 6426 [ 6427 "_disk2".to_string(), 6428 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 6429 "d:\\world".to_string(), 6430 "hello".to_string(), 6431 ], 6432 [ 6433 "_disk3".to_string(), 6434 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 6435 "e:\\hello".to_string(), 6436 "world".to_string(), 6437 ], 6438 ]; 6439 6440 let r = std::panic::catch_unwind(|| { 6441 // Wait to make sure Windows boots up 6442 assert!(windows_guest.wait_for_boot()); 6443 6444 // Initially present disk device 6445 let disk_num = 1; 6446 assert_eq!(windows_guest.disk_count(), disk_num); 6447 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6448 6449 for it in &disk_test_data { 6450 let disk_id = it[0].as_str(); 6451 let disk = it[1].as_str(); 6452 // Hotplug disk device 6453 let (cmd_success, cmd_output) = remote_command_w_output( 6454 &api_socket, 6455 "add-disk", 6456 Some(format!("path={},readonly=off", disk).as_str()), 6457 ); 6458 assert!(cmd_success); 6459 assert!(String::from_utf8_lossy(&cmd_output) 6460 .contains(format!("\"id\":\"{}\"", disk_id).as_str())); 6461 thread::sleep(std::time::Duration::new(5, 0)); 6462 // Online disk devices 6463 windows_guest.disks_set_rw(); 6464 windows_guest.disks_online(); 6465 } 6466 // Verify the devices are on the system 6467 let disk_num = (disk_test_data.len() + 1) as u8; 6468 assert_eq!(windows_guest.disk_count(), disk_num); 6469 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6470 6471 // Put test data 6472 for it in &disk_test_data { 6473 let fname = it[2].as_str(); 6474 let data = it[3].as_str(); 6475 windows_guest.disk_file_put(fname, data); 6476 } 6477 6478 // Unmount disk devices 6479 for it in &disk_test_data { 6480 let disk_id = it[0].as_str(); 6481 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 6482 assert!(cmd_success); 6483 thread::sleep(std::time::Duration::new(5, 0)); 6484 } 6485 6486 // Verify the devices have been removed 6487 let disk_num = 1; 6488 assert_eq!(windows_guest.disk_count(), disk_num); 6489 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6490 6491 // Remount 6492 for it in &disk_test_data { 6493 let disk = it[1].as_str(); 6494 let (cmd_success, _cmd_output) = remote_command_w_output( 6495 &api_socket, 6496 "add-disk", 6497 Some(format!("path={},readonly=off", disk).as_str()), 6498 ); 6499 assert!(cmd_success); 6500 thread::sleep(std::time::Duration::new(5, 0)); 6501 } 6502 6503 // Check the files exists with the expected contents 6504 for it in &disk_test_data { 6505 let fname = it[2].as_str(); 6506 let data = it[3].as_str(); 6507 let out = windows_guest.disk_file_read(fname); 6508 assert_eq!(data, out.trim()); 6509 } 6510 6511 // Intentionally no unmount, it'll happen at shutdown. 6512 6513 windows_guest.shutdown(); 6514 }); 6515 6516 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6517 let _ = child.kill(); 6518 let output = child.wait_with_output().unwrap(); 6519 6520 let _ = child_dnsmasq.kill(); 6521 let _ = child_dnsmasq.wait(); 6522 6523 handle_child_output(r, &output); 6524 } 6525 6526 #[test] 6527 #[cfg(not(feature = "mshv"))] 6528 fn test_windows_guest_netdev_multi() { 6529 let windows_guest = WindowsGuest::new(); 6530 6531 let mut ovmf_path = dirs::home_dir().unwrap(); 6532 ovmf_path.push("workloads"); 6533 ovmf_path.push(OVMF_NAME); 6534 6535 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6536 let api_socket = temp_api_path(&tmp_dir); 6537 6538 let mut child = GuestCommand::new(windows_guest.guest()) 6539 .args(&["--api-socket", &api_socket]) 6540 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6541 .args(&["--memory", "size=4G"]) 6542 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6543 .args(&["--serial", "tty"]) 6544 .args(&["--console", "off"]) 6545 .default_disks() 6546 // The multi net dev config is borrowed from test_multiple_network_interfaces 6547 .args(&[ 6548 "--net", 6549 windows_guest.guest().default_net_string().as_str(), 6550 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 6551 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 6552 ]) 6553 .capture_output() 6554 .spawn() 6555 .unwrap(); 6556 6557 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6558 6559 let r = std::panic::catch_unwind(|| { 6560 // Wait to make sure Windows boots up 6561 assert!(windows_guest.wait_for_boot()); 6562 6563 let netdev_num = 3; 6564 assert_eq!(windows_guest.netdev_count(), netdev_num); 6565 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6566 6567 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 6568 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 6569 6570 windows_guest.shutdown(); 6571 }); 6572 6573 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6574 let _ = child.kill(); 6575 let output = child.wait_with_output().unwrap(); 6576 6577 let _ = child_dnsmasq.kill(); 6578 let _ = child_dnsmasq.wait(); 6579 6580 handle_child_output(r, &output); 6581 } 6582 } 6583 6584 #[cfg(target_arch = "x86_64")] 6585 mod sgx { 6586 use crate::tests::*; 6587 6588 #[test] 6589 fn test_sgx() { 6590 let focal = UbuntuDiskConfig::new(FOCAL_SGX_IMAGE_NAME.to_string()); 6591 let guest = Guest::new(Box::new(focal)); 6592 let mut workload_path = dirs::home_dir().unwrap(); 6593 workload_path.push("workloads"); 6594 6595 let mut kernel_path = workload_path; 6596 kernel_path.push("vmlinux_w_sgx"); 6597 6598 let mut child = GuestCommand::new(&guest) 6599 .args(&["--cpus", "boot=1"]) 6600 .args(&["--memory", "size=512M"]) 6601 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6602 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6603 .default_disks() 6604 .default_net() 6605 .args(&["--sgx-epc", "id=epc0,size=64M"]) 6606 .capture_output() 6607 .spawn() 6608 .unwrap(); 6609 6610 let r = std::panic::catch_unwind(|| { 6611 guest.wait_vm_boot(None).unwrap(); 6612 6613 // Check if SGX is correctly detected in the guest. 6614 guest.check_sgx_support().unwrap(); 6615 6616 // Validate the SGX EPC section is 64MiB. 6617 assert_eq!( 6618 guest 6619 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 6620 .unwrap() 6621 .trim(), 6622 "0x0000000004000000" 6623 ); 6624 6625 // Run a test relying on SGX enclaves and check if it runs 6626 // successfully. 6627 assert!(guest 6628 .ssh_command("cd /linux-sgx/SampleCode/LocalAttestation/bin/ && sudo ./app") 6629 .unwrap() 6630 .trim() 6631 .contains( 6632 "succeed to load enclaves.\nsucceed to \ 6633 establish secure channel.\nSucceed to exchange \ 6634 secure message...\nSucceed to close Session..." 6635 )); 6636 }); 6637 6638 let _ = child.kill(); 6639 let output = child.wait_with_output().unwrap(); 6640 6641 handle_child_output(r, &output); 6642 } 6643 } 6644 6645 #[cfg(target_arch = "x86_64")] 6646 mod vfio { 6647 use crate::tests::*; 6648 6649 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 6650 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 6651 let guest = Guest::new(Box::new(hirsute)); 6652 let api_socket = temp_api_path(&guest.tmp_dir); 6653 6654 let mut child = GuestCommand::new(&guest) 6655 .args(&["--cpus", "boot=4"]) 6656 .args(&[ 6657 "--memory", 6658 format!("size=4G,hotplug_size=4G,hotplug_method={}", hotplug_method).as_str(), 6659 ]) 6660 .args(&["--kernel", guest.fw_path.as_str()]) 6661 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 6662 .args(&["--api-socket", &api_socket]) 6663 .default_disks() 6664 .default_net() 6665 .capture_output() 6666 .spawn() 6667 .unwrap(); 6668 6669 let r = std::panic::catch_unwind(|| { 6670 guest.wait_vm_boot(None).unwrap(); 6671 6672 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6673 6674 guest.enable_memory_hotplug(); 6675 6676 // Add RAM to the VM 6677 let desired_ram = 6 << 30; 6678 resize_command(&api_socket, None, Some(desired_ram), None); 6679 thread::sleep(std::time::Duration::new(30, 0)); 6680 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6681 6682 // Check the VFIO device works when RAM is increased to 6GiB 6683 guest.check_nvidia_gpu(); 6684 }); 6685 6686 let _ = child.kill(); 6687 let output = child.wait_with_output().unwrap(); 6688 6689 handle_child_output(r, &output); 6690 } 6691 6692 #[test] 6693 fn test_nvidia_card_memory_hotplug_acpi() { 6694 test_nvidia_card_memory_hotplug("acpi") 6695 } 6696 6697 #[test] 6698 fn test_nvidia_card_memory_hotplug_virtio_mem() { 6699 test_nvidia_card_memory_hotplug("virtio-mem") 6700 } 6701 6702 #[test] 6703 fn test_nvidia_card_pci_hotplug() { 6704 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 6705 let guest = Guest::new(Box::new(hirsute)); 6706 let api_socket = temp_api_path(&guest.tmp_dir); 6707 6708 let mut child = GuestCommand::new(&guest) 6709 .args(&["--cpus", "boot=4"]) 6710 .args(&["--memory", "size=4G"]) 6711 .args(&["--kernel", guest.fw_path.as_str()]) 6712 .args(&["--api-socket", &api_socket]) 6713 .default_disks() 6714 .default_net() 6715 .capture_output() 6716 .spawn() 6717 .unwrap(); 6718 6719 let r = std::panic::catch_unwind(|| { 6720 guest.wait_vm_boot(None).unwrap(); 6721 6722 // Hotplug the card to the VM 6723 let (cmd_success, cmd_output) = remote_command_w_output( 6724 &api_socket, 6725 "add-device", 6726 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 6727 ); 6728 assert!(cmd_success); 6729 assert!(String::from_utf8_lossy(&cmd_output) 6730 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 6731 6732 thread::sleep(std::time::Duration::new(10, 0)); 6733 6734 // Check the VFIO device works after hotplug 6735 guest.check_nvidia_gpu(); 6736 }); 6737 6738 let _ = child.kill(); 6739 let output = child.wait_with_output().unwrap(); 6740 6741 handle_child_output(r, &output); 6742 } 6743 6744 #[test] 6745 fn test_nvidia_card_reboot() { 6746 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 6747 let guest = Guest::new(Box::new(hirsute)); 6748 let api_socket = temp_api_path(&guest.tmp_dir); 6749 6750 let mut child = GuestCommand::new(&guest) 6751 .args(&["--cpus", "boot=4"]) 6752 .args(&["--memory", "size=4G"]) 6753 .args(&["--kernel", guest.fw_path.as_str()]) 6754 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 6755 .args(&["--api-socket", &api_socket]) 6756 .default_disks() 6757 .default_net() 6758 .capture_output() 6759 .spawn() 6760 .unwrap(); 6761 6762 let r = std::panic::catch_unwind(|| { 6763 guest.wait_vm_boot(None).unwrap(); 6764 6765 // Check the VFIO device works after boot 6766 guest.check_nvidia_gpu(); 6767 6768 guest.reboot_linux(0, None); 6769 6770 // Check the VFIO device works after reboot 6771 guest.check_nvidia_gpu(); 6772 }); 6773 6774 let _ = child.kill(); 6775 let output = child.wait_with_output().unwrap(); 6776 6777 handle_child_output(r, &output); 6778 } 6779 } 6780 6781 mod live_migration { 6782 use crate::tests::*; 6783 6784 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 6785 // same host. It ensures the following behaviors: 6786 // 1. The source VM is up and functional (including various virtio-devices are working properly); 6787 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 6788 // 3. The source VM terminated gracefully after live migration; 6789 // 4. The destination VM is functional (including various virtio-devices are working properly) after 6790 // live migration; 6791 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 6792 fn _test_live_migration(numa: bool) { 6793 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6794 let guest = Guest::new(Box::new(focal)); 6795 let kernel_path = direct_kernel_boot_path(); 6796 let console_text = String::from("On a branch floating down river a cricket, singing."); 6797 let net_id = "net123"; 6798 let net_params = format!( 6799 "id={},tap=,mac={},ip={},mask=255.255.255.0", 6800 net_id, guest.network.guest_mac, guest.network.host_ip 6801 ); 6802 6803 let memory_param: &[&str] = if numa { 6804 &[ 6805 "--memory", 6806 "size=0", 6807 "--memory-zone", 6808 "id=mem0,size=1G", 6809 "id=mem1,size=1G", 6810 "id=mem2,size=2G", 6811 "--numa", 6812 "guest_numa_id=0,cpus=0-2:9,distances=1@15:2@20,memory_zones=mem0", 6813 "guest_numa_id=1,cpus=3-4:6-8,distances=0@20:2@25,memory_zones=mem1", 6814 "guest_numa_id=2,cpus=5:10-11,distances=0@25:1@30,memory_zones=mem2", 6815 ] 6816 } else { 6817 &["--memory", "size=4G"] 6818 }; 6819 6820 // Start the source VM 6821 let src_api_socket = temp_api_path(&guest.tmp_dir); 6822 let mut src_child = GuestCommand::new(&guest) 6823 .args(&["--cpus", "boot=6,max=12"]) 6824 .args(memory_param) 6825 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6826 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6827 .default_disks() 6828 .args(&["--net", net_params.as_str()]) 6829 .args(&["--api-socket", &src_api_socket]) 6830 .capture_output() 6831 .spawn() 6832 .unwrap(); 6833 6834 // Start the destination VM 6835 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 6836 dest_api_socket.push_str(".dest"); 6837 let mut dest_child = GuestCommand::new(&guest) 6838 .args(&["--api-socket", &dest_api_socket]) 6839 .capture_output() 6840 .spawn() 6841 .unwrap(); 6842 6843 let r = std::panic::catch_unwind(|| { 6844 guest.wait_vm_boot(Some(30)).unwrap(); 6845 6846 // Make sure the source VM is functaionl 6847 // Check the number of vCPUs 6848 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 6); 6849 // Check the guest RAM 6850 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6851 // Check the guest virtio-devices, e.g. block, rng, console, and net 6852 guest.check_devices_common(None, Some(&console_text)); 6853 6854 // x86_64: Following what's done in the `test_snapshot_restore`, we need 6855 // to make sure that removing and adding back the virtio-net device does 6856 // not break the live-migration support for virtio-pci. 6857 #[cfg(target_arch = "x86_64")] 6858 { 6859 assert!(remote_command( 6860 &src_api_socket, 6861 "remove-device", 6862 Some(net_id), 6863 )); 6864 thread::sleep(std::time::Duration::new(10, 0)); 6865 6866 // Plug the virtio-net device again 6867 assert!(remote_command( 6868 &src_api_socket, 6869 "add-net", 6870 Some(net_params.as_str()), 6871 )); 6872 thread::sleep(std::time::Duration::new(10, 0)); 6873 } 6874 6875 // Start the live-migration 6876 let migration_socket = String::from( 6877 guest 6878 .tmp_dir 6879 .as_path() 6880 .join("live-migration.sock") 6881 .to_str() 6882 .unwrap(), 6883 ); 6884 // Start to receive migration from the destintion VM 6885 let mut receive_migration = Command::new(clh_command("ch-remote")) 6886 .args(&[ 6887 &format!("--api-socket={}", &dest_api_socket), 6888 "receive-migration", 6889 &format! {"unix:{}", migration_socket}, 6890 ]) 6891 .stderr(Stdio::piped()) 6892 .stdout(Stdio::piped()) 6893 .spawn() 6894 .unwrap(); 6895 // Give it '1s' to make sure the 'migration_socket' file is properly created 6896 thread::sleep(std::time::Duration::new(1, 0)); 6897 // Start to send migration from the source VM 6898 let mut send_migration = Command::new(clh_command("ch-remote")) 6899 .args(&[ 6900 &format!("--api-socket={}", &src_api_socket), 6901 "send-migration", 6902 &format! {"unix:{}", migration_socket}, 6903 ]) 6904 .stderr(Stdio::piped()) 6905 .stdout(Stdio::piped()) 6906 .spawn() 6907 .unwrap(); 6908 6909 let mut success; 6910 // The 'send-migration' command should be executed successfully within the given timeout 6911 if let Some(status) = send_migration 6912 .wait_timeout(std::time::Duration::from_secs(30)) 6913 .unwrap() 6914 { 6915 success = status.success(); 6916 } else { 6917 success = false; 6918 } 6919 if !success { 6920 let _ = send_migration.kill(); 6921 let output = send_migration.wait_with_output().unwrap(); 6922 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 6923 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 6924 } 6925 // The 'receive-migration' command should be executed successfully within the given timeout 6926 if let Some(status) = receive_migration 6927 .wait_timeout(std::time::Duration::from_secs(30)) 6928 .unwrap() 6929 { 6930 success = status.success(); 6931 } else { 6932 success = false; 6933 } 6934 if !success { 6935 let _ = receive_migration.kill(); 6936 let output = receive_migration.wait_with_output().unwrap(); 6937 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 6938 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 6939 } 6940 assert!( 6941 success, 6942 "Unsuccessful command: 'send-migration' or 'receive-migration'." 6943 ); 6944 }); 6945 6946 let print_and_panic = |src_vm: Child, dest_vm: Child, message: &str| -> ! { 6947 let mut src_vm = src_vm; 6948 let mut dest_vm = dest_vm; 6949 6950 let _ = src_vm.kill(); 6951 let src_output = src_vm.wait_with_output().unwrap(); 6952 eprintln!( 6953 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 6954 String::from_utf8_lossy(&src_output.stdout) 6955 ); 6956 eprintln!( 6957 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 6958 String::from_utf8_lossy(&src_output.stderr) 6959 ); 6960 let _ = dest_vm.kill(); 6961 let dest_output = dest_vm.wait_with_output().unwrap(); 6962 eprintln!( 6963 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 6964 String::from_utf8_lossy(&dest_output.stdout) 6965 ); 6966 eprintln!( 6967 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 6968 String::from_utf8_lossy(&dest_output.stderr) 6969 ); 6970 6971 panic!("Test failed: {}", message) 6972 }; 6973 6974 // Check and report any errors occured during the live-migration 6975 if r.is_err() { 6976 print_and_panic(src_child, dest_child, "Error occured during live-migration"); 6977 } 6978 6979 // Check the source vm has been terminated successful (give it '3s' to settle) 6980 thread::sleep(std::time::Duration::new(3, 0)); 6981 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 6982 print_and_panic( 6983 src_child, 6984 dest_child, 6985 "source VM was not terminated successfully.", 6986 ); 6987 }; 6988 6989 // Post live-migration check to make sure the destination VM is funcational 6990 let r = std::panic::catch_unwind(|| { 6991 // Perform same checks to validate VM has been properly migrated 6992 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 6); 6993 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6994 guest.check_devices_common(None, Some(&console_text)); 6995 6996 // Perform NUMA related checks 6997 if numa { 6998 guest.check_numa_common( 6999 Some(&[960_000, 960_000, 1_920_000]), 7000 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 7001 Some(&["10 15 20", "20 10 25", "25 30 10"]), 7002 ); 7003 7004 // AArch64 currently does not support hotplug, and therefore we only 7005 // test hotplug-related function on x86_64 here. 7006 #[cfg(target_arch = "x86_64")] 7007 { 7008 guest.enable_memory_hotplug(); 7009 7010 // Resize to the maximum amount of CPUs and check each NUMA 7011 // node has been assigned the right CPUs set. 7012 resize_command(&dest_api_socket, Some(12), None, None); 7013 thread::sleep(std::time::Duration::new(5, 0)); 7014 7015 guest.check_numa_common( 7016 None, 7017 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 7018 None, 7019 ); 7020 } 7021 } 7022 }); 7023 7024 // Clean-up the destination VM and make sure it terminated correctly 7025 let _ = dest_child.kill(); 7026 let dest_output = dest_child.wait_with_output().unwrap(); 7027 handle_child_output(r, &dest_output); 7028 7029 // Check the destination VM has the expected 'concole_text' from its output 7030 let r = std::panic::catch_unwind(|| { 7031 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 7032 }); 7033 handle_child_output(r, &dest_output); 7034 } 7035 7036 #[test] 7037 fn test_live_migration_basic() { 7038 _test_live_migration(false) 7039 } 7040 7041 #[test] 7042 fn test_live_migration_numa() { 7043 _test_live_migration(true) 7044 } 7045 #[test] 7046 fn test_live_migration_ovs_dpdk() { 7047 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7048 let ovs_guest = Guest::new(Box::new(ovs_focal)); 7049 7050 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7051 let migration_guest = Guest::new(Box::new(migration_focal)); 7052 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 7053 7054 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 7055 let (mut ovs_child, mut src_child) = 7056 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket); 7057 7058 // Start the destination VM 7059 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 7060 dest_api_socket.push_str(".dest"); 7061 let mut dest_child = GuestCommand::new(&migration_guest) 7062 .args(&["--api-socket", &dest_api_socket]) 7063 .capture_output() 7064 .spawn() 7065 .unwrap(); 7066 7067 let r = std::panic::catch_unwind(|| { 7068 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 7069 thread::sleep(std::time::Duration::new(1, 0)); 7070 7071 // Start the live-migration 7072 let migration_socket = String::from( 7073 migration_guest 7074 .tmp_dir 7075 .as_path() 7076 .join("live-migration.sock") 7077 .to_str() 7078 .unwrap(), 7079 ); 7080 // Start to receive migration from the destintion VM 7081 let mut receive_migration = Command::new(clh_command("ch-remote")) 7082 .args(&[ 7083 &format!("--api-socket={}", &dest_api_socket), 7084 "receive-migration", 7085 &format! {"unix:{}", migration_socket}, 7086 ]) 7087 .stderr(Stdio::piped()) 7088 .stdout(Stdio::piped()) 7089 .spawn() 7090 .unwrap(); 7091 // Give it '1s' to make sure the 'migration_socket' file is properly created 7092 thread::sleep(std::time::Duration::new(1, 0)); 7093 // Start to send migration from the source VM 7094 let mut send_migration = Command::new(clh_command("ch-remote")) 7095 .args(&[ 7096 &format!("--api-socket={}", &src_api_socket), 7097 "send-migration", 7098 &format! {"unix:{}", migration_socket}, 7099 ]) 7100 .stderr(Stdio::piped()) 7101 .stdout(Stdio::piped()) 7102 .spawn() 7103 .unwrap(); 7104 7105 let mut success; 7106 // The 'send-migration' command should be executed successfully within the given timeout 7107 if let Some(status) = send_migration 7108 .wait_timeout(std::time::Duration::from_secs(30)) 7109 .unwrap() 7110 { 7111 success = status.success(); 7112 } else { 7113 success = false; 7114 } 7115 if !success { 7116 let _ = send_migration.kill(); 7117 let output = send_migration.wait_with_output().unwrap(); 7118 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 7119 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 7120 } 7121 // The 'receive-migration' command should be executed successfully within the given timeout 7122 if let Some(status) = receive_migration 7123 .wait_timeout(std::time::Duration::from_secs(30)) 7124 .unwrap() 7125 { 7126 success = status.success(); 7127 } else { 7128 success = false; 7129 } 7130 if !success { 7131 let _ = receive_migration.kill(); 7132 let output = receive_migration.wait_with_output().unwrap(); 7133 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 7134 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 7135 } 7136 assert!( 7137 success, 7138 "Unsuccessful command: 'send-migration' or 'receive-migration'." 7139 ); 7140 }); 7141 7142 let print_and_panic = |src_vm: Child, 7143 dest_vm: Child, 7144 ovs_vm: Child, 7145 message: &str| 7146 -> ! { 7147 let mut src_vm = src_vm; 7148 let mut dest_vm = dest_vm; 7149 let mut ovs_vm = ovs_vm; 7150 7151 let _ = src_vm.kill(); 7152 let src_output = src_vm.wait_with_output().unwrap(); 7153 eprintln!( 7154 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 7155 String::from_utf8_lossy(&src_output.stdout) 7156 ); 7157 eprintln!( 7158 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 7159 String::from_utf8_lossy(&src_output.stderr) 7160 ); 7161 let _ = dest_vm.kill(); 7162 let dest_output = dest_vm.wait_with_output().unwrap(); 7163 eprintln!( 7164 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 7165 String::from_utf8_lossy(&dest_output.stdout) 7166 ); 7167 eprintln!( 7168 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 7169 String::from_utf8_lossy(&dest_output.stderr) 7170 ); 7171 let _ = ovs_vm.kill(); 7172 let ovs_output = ovs_vm.wait_with_output().unwrap(); 7173 eprintln!( 7174 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 7175 String::from_utf8_lossy(&ovs_output.stdout) 7176 ); 7177 eprintln!( 7178 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 7179 String::from_utf8_lossy(&ovs_output.stderr) 7180 ); 7181 7182 cleanup_ovs_dpdk(); 7183 7184 panic!("Test failed: {}", message) 7185 }; 7186 7187 // Check and report any errors occured during the live-migration 7188 if r.is_err() { 7189 print_and_panic( 7190 src_child, 7191 dest_child, 7192 ovs_child, 7193 "Error occured during live-migration", 7194 ); 7195 } 7196 7197 // Check the source vm has been terminated successful (give it '3s' to settle) 7198 thread::sleep(std::time::Duration::new(3, 0)); 7199 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 7200 print_and_panic( 7201 src_child, 7202 dest_child, 7203 ovs_child, 7204 "source VM was not terminated successfully.", 7205 ); 7206 }; 7207 7208 // Post live-migration check to make sure the destination VM is funcational 7209 let r = std::panic::catch_unwind(|| { 7210 // Perform same checks to validate VM has been properly migrated 7211 // Spawn a new netcat listener in the OVS VM 7212 let guest_ip = ovs_guest.network.guest_ip.clone(); 7213 thread::spawn(move || { 7214 ssh_command_ip( 7215 "nc -l 12345", 7216 &guest_ip, 7217 DEFAULT_SSH_RETRIES, 7218 DEFAULT_SSH_TIMEOUT, 7219 ) 7220 .unwrap(); 7221 }); 7222 7223 // Wait for the server to be listening 7224 thread::sleep(std::time::Duration::new(5, 0)); 7225 7226 // And check the connection is still functional after live-migration 7227 migration_guest 7228 .ssh_command("nc -vz 172.100.0.1 12345") 7229 .unwrap(); 7230 7231 cleanup_ovs_dpdk(); 7232 }); 7233 7234 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 7235 let _ = dest_child.kill(); 7236 let _ = ovs_child.kill(); 7237 let dest_output = dest_child.wait_with_output().unwrap(); 7238 handle_child_output(r, &dest_output); 7239 let ovs_output = ovs_child.wait_with_output().unwrap(); 7240 handle_child_output(Ok(()), &ovs_output); 7241 } 7242 } 7243 7244 #[cfg(all(target_arch = "aarch64", feature = "acpi"))] 7245 mod aarch64_acpi { 7246 use crate::tests::*; 7247 7248 #[test] 7249 fn test_simple_launch_acpi() { 7250 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7251 7252 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 7253 let guest = Guest::new(disk_config); 7254 7255 let mut child = GuestCommand::new(&guest) 7256 .args(&["--cpus", "boot=1"]) 7257 .args(&["--memory", "size=512M"]) 7258 .args(&["--kernel", edk2_path().to_str().unwrap()]) 7259 .default_disks() 7260 .default_net() 7261 .args(&["--serial", "tty", "--console", "off"]) 7262 .capture_output() 7263 .spawn() 7264 .unwrap(); 7265 7266 let r = std::panic::catch_unwind(|| { 7267 guest.wait_vm_boot(Some(120)).unwrap(); 7268 7269 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 7270 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 7271 assert!(guest.get_entropy().unwrap_or_default() >= 900); 7272 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 7273 }); 7274 7275 let _ = child.kill(); 7276 let output = child.wait_with_output().unwrap(); 7277 7278 handle_child_output(r, &output); 7279 }); 7280 } 7281 7282 #[test] 7283 fn test_guest_numa_nodes_acpi() { 7284 _test_guest_numa_nodes(true); 7285 } 7286 7287 #[test] 7288 fn test_cpu_topology_421_acpi() { 7289 test_cpu_topology(4, 2, 1, true); 7290 } 7291 7292 #[test] 7293 fn test_cpu_topology_142_acpi() { 7294 test_cpu_topology(1, 4, 2, true); 7295 } 7296 7297 #[test] 7298 fn test_cpu_topology_262_acpi() { 7299 test_cpu_topology(2, 6, 2, true); 7300 } 7301 7302 #[test] 7303 fn test_power_button_acpi() { 7304 _test_power_button(true); 7305 } 7306 } 7307 } 7308