1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 use std::sync::Mutex; 28 use std::thread; 29 use test_infra::*; 30 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 31 use wait_timeout::ChildExt; 32 33 #[cfg(target_arch = "x86_64")] 34 mod x86_64 { 35 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw"; 36 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 37 pub const FOCAL_SGX_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-sgx.raw"; 38 pub const JAMMY_NVIDIA_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-nvidia.raw"; 39 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 40 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 41 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 42 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20221118-1.raw"; 43 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2019.raw"; 44 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 45 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 46 } 47 48 #[cfg(target_arch = "x86_64")] 49 use x86_64::*; 50 51 #[cfg(target_arch = "aarch64")] 52 mod aarch64 { 53 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 54 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 55 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 56 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 57 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 58 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 59 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 60 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 61 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 62 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 63 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 64 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 65 } 66 67 #[cfg(target_arch = "aarch64")] 68 use aarch64::*; 69 70 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 71 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 72 73 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 74 75 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 76 let mut workload_path = dirs::home_dir().unwrap(); 77 workload_path.push("workloads"); 78 79 let mut virtiofsd_path = workload_path; 80 virtiofsd_path.push("virtiofsd"); 81 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 82 83 let virtiofsd_socket_path = 84 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 85 86 // Start the daemon 87 let child = Command::new(virtiofsd_path.as_str()) 88 .args(["--shared-dir", shared_dir]) 89 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 90 .args(["--cache", "never"]) 91 .spawn() 92 .unwrap(); 93 94 thread::sleep(std::time::Duration::new(10, 0)); 95 96 (child, virtiofsd_socket_path) 97 } 98 99 fn prepare_vubd( 100 tmp_dir: &TempDir, 101 blk_img: &str, 102 num_queues: usize, 103 rdonly: bool, 104 direct: bool, 105 ) -> (std::process::Child, String) { 106 let mut workload_path = dirs::home_dir().unwrap(); 107 workload_path.push("workloads"); 108 109 let mut blk_file_path = workload_path; 110 blk_file_path.push(blk_img); 111 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 112 113 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 114 115 // Start the daemon 116 let child = Command::new(clh_command("vhost_user_block")) 117 .args([ 118 "--block-backend", 119 format!( 120 "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" 121 ) 122 .as_str(), 123 ]) 124 .spawn() 125 .unwrap(); 126 127 thread::sleep(std::time::Duration::new(10, 0)); 128 129 (child, vubd_socket_path) 130 } 131 132 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 133 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 134 } 135 136 fn temp_api_path(tmp_dir: &TempDir) -> String { 137 String::from( 138 tmp_dir 139 .as_path() 140 .join("cloud-hypervisor.sock") 141 .to_str() 142 .unwrap(), 143 ) 144 } 145 146 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 147 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 148 } 149 150 // Creates the directory and returns the path. 151 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 152 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 153 std::fs::create_dir(&snapshot_dir).unwrap(); 154 snapshot_dir 155 } 156 157 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 158 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 159 vmcore_file 160 } 161 162 // Creates the path for direct kernel boot and return the path. 163 // For x86_64, this function returns the vmlinux kernel path. 164 // For AArch64, this function returns the PE kernel path. 165 fn direct_kernel_boot_path() -> PathBuf { 166 let mut workload_path = dirs::home_dir().unwrap(); 167 workload_path.push("workloads"); 168 169 let mut kernel_path = workload_path; 170 #[cfg(target_arch = "x86_64")] 171 kernel_path.push("vmlinux"); 172 #[cfg(target_arch = "aarch64")] 173 kernel_path.push("Image"); 174 175 kernel_path 176 } 177 178 fn edk2_path() -> PathBuf { 179 let mut workload_path = dirs::home_dir().unwrap(); 180 workload_path.push("workloads"); 181 let mut edk2_path = workload_path; 182 edk2_path.push(OVMF_NAME); 183 184 edk2_path 185 } 186 187 fn cloud_hypervisor_release_path() -> String { 188 let mut workload_path = dirs::home_dir().unwrap(); 189 workload_path.push("workloads"); 190 191 let mut ch_release_path = workload_path; 192 #[cfg(target_arch = "x86_64")] 193 ch_release_path.push("cloud-hypervisor-static"); 194 #[cfg(target_arch = "aarch64")] 195 ch_release_path.push("cloud-hypervisor-static-aarch64"); 196 197 ch_release_path.into_os_string().into_string().unwrap() 198 } 199 200 fn prepare_vhost_user_net_daemon( 201 tmp_dir: &TempDir, 202 ip: &str, 203 tap: Option<&str>, 204 mtu: Option<u16>, 205 num_queues: usize, 206 client_mode: bool, 207 ) -> (std::process::Command, String) { 208 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 209 210 // Start the daemon 211 let mut net_params = format!( 212 "ip={ip},mask=255.255.255.0,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" 213 ); 214 215 if let Some(tap) = tap { 216 net_params.push_str(format!(",tap={tap}").as_str()); 217 } 218 219 if let Some(mtu) = mtu { 220 net_params.push_str(format!(",mtu={mtu}").as_str()); 221 } 222 223 let mut command = Command::new(clh_command("vhost_user_net")); 224 command.args(["--net-backend", net_params.as_str()]); 225 226 (command, vunet_socket_path) 227 } 228 229 fn curl_command(api_socket: &str, method: &str, url: &str, http_body: Option<&str>) { 230 let mut curl_args: Vec<&str> = ["--unix-socket", api_socket, "-i", "-X", method, url].to_vec(); 231 232 if let Some(body) = http_body { 233 curl_args.push("-H"); 234 curl_args.push("Accept: application/json"); 235 curl_args.push("-H"); 236 curl_args.push("Content-Type: application/json"); 237 curl_args.push("-d"); 238 curl_args.push(body); 239 } 240 241 let status = Command::new("curl") 242 .args(curl_args) 243 .status() 244 .expect("Failed to launch curl command"); 245 246 assert!(status.success()); 247 } 248 249 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 250 let mut cmd = Command::new(clh_command("ch-remote")); 251 cmd.args([&format!("--api-socket={api_socket}"), command]); 252 253 if let Some(arg) = arg { 254 cmd.arg(arg); 255 } 256 let output = cmd.output().unwrap(); 257 if output.status.success() { 258 true 259 } else { 260 eprintln!("Error running ch-remote command: {:?}", &cmd); 261 let stderr = String::from_utf8_lossy(&output.stderr); 262 eprintln!("stderr: {stderr}"); 263 false 264 } 265 } 266 267 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 268 let mut cmd = Command::new(clh_command("ch-remote")); 269 cmd.args([&format!("--api-socket={api_socket}"), command]); 270 271 if let Some(arg) = arg { 272 cmd.arg(arg); 273 } 274 275 let output = cmd.output().expect("Failed to launch ch-remote"); 276 277 (output.status.success(), output.stdout) 278 } 279 280 fn resize_command( 281 api_socket: &str, 282 desired_vcpus: Option<u8>, 283 desired_ram: Option<usize>, 284 desired_balloon: Option<usize>, 285 event_file: Option<&str>, 286 ) -> bool { 287 let mut cmd = Command::new(clh_command("ch-remote")); 288 cmd.args([&format!("--api-socket={api_socket}"), "resize"]); 289 290 if let Some(desired_vcpus) = desired_vcpus { 291 cmd.arg(format!("--cpus={desired_vcpus}")); 292 } 293 294 if let Some(desired_ram) = desired_ram { 295 cmd.arg(format!("--memory={desired_ram}")); 296 } 297 298 if let Some(desired_balloon) = desired_balloon { 299 cmd.arg(format!("--balloon={desired_balloon}")); 300 } 301 302 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 303 304 if let Some(event_path) = event_file { 305 let latest_events = [ 306 &MetaEvent { 307 event: "resizing".to_string(), 308 device_id: None, 309 }, 310 &MetaEvent { 311 event: "resized".to_string(), 312 device_id: None, 313 }, 314 ]; 315 assert!(check_latest_events_exact(&latest_events, event_path)); 316 } 317 318 ret 319 } 320 321 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 322 let mut cmd = Command::new(clh_command("ch-remote")); 323 cmd.args([ 324 &format!("--api-socket={api_socket}"), 325 "resize-zone", 326 &format!("--id={id}"), 327 &format!("--size={desired_size}"), 328 ]); 329 330 cmd.status().expect("Failed to launch ch-remote").success() 331 } 332 333 // setup OVS-DPDK bridge and ports 334 fn setup_ovs_dpdk() { 335 // setup OVS-DPDK 336 assert!(exec_host_command_status("service openvswitch-switch start").success()); 337 assert!(exec_host_command_status("ovs-vsctl init").success()); 338 assert!( 339 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 340 .success() 341 ); 342 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 343 344 // Create OVS-DPDK bridge and ports 345 assert!(exec_host_command_status( 346 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 347 ) 348 .success()); 349 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 350 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 351 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 352 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 353 } 354 fn cleanup_ovs_dpdk() { 355 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 356 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 357 } 358 // Setup two guests and ensure they are connected through ovs-dpdk 359 fn setup_ovs_dpdk_guests( 360 guest1: &Guest, 361 guest2: &Guest, 362 api_socket: &str, 363 release_binary: bool, 364 ) -> (Child, Child) { 365 setup_ovs_dpdk(); 366 367 let clh_path = if !release_binary { 368 clh_command("cloud-hypervisor") 369 } else { 370 cloud_hypervisor_release_path() 371 }; 372 373 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 374 .args(["--cpus", "boot=2"]) 375 .args(["--memory", "size=0,shared=on"]) 376 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 377 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 378 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 379 .default_disks() 380 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 381 .capture_output() 382 .spawn() 383 .unwrap(); 384 385 #[cfg(target_arch = "x86_64")] 386 let guest_net_iface = "ens5"; 387 #[cfg(target_arch = "aarch64")] 388 let guest_net_iface = "enp0s5"; 389 390 let r = std::panic::catch_unwind(|| { 391 guest1.wait_vm_boot(None).unwrap(); 392 393 guest1 394 .ssh_command(&format!( 395 "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" 396 )) 397 .unwrap(); 398 guest1 399 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 400 .unwrap(); 401 402 let guest_ip = guest1.network.guest_ip.clone(); 403 thread::spawn(move || { 404 ssh_command_ip( 405 "nc -l 12345", 406 &guest_ip, 407 DEFAULT_SSH_RETRIES, 408 DEFAULT_SSH_TIMEOUT, 409 ) 410 .unwrap(); 411 }); 412 }); 413 if r.is_err() { 414 cleanup_ovs_dpdk(); 415 416 let _ = child1.kill(); 417 let output = child1.wait_with_output().unwrap(); 418 handle_child_output(r, &output); 419 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 420 } 421 422 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 423 .args(["--api-socket", api_socket]) 424 .args(["--cpus", "boot=2"]) 425 .args(["--memory", "size=0,shared=on"]) 426 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 427 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 428 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 429 .default_disks() 430 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 431 .capture_output() 432 .spawn() 433 .unwrap(); 434 435 let r = std::panic::catch_unwind(|| { 436 guest2.wait_vm_boot(None).unwrap(); 437 438 guest2 439 .ssh_command(&format!( 440 "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" 441 )) 442 .unwrap(); 443 guest2 444 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 445 .unwrap(); 446 447 // Check the connection works properly between the two VMs 448 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 449 }); 450 if r.is_err() { 451 cleanup_ovs_dpdk(); 452 453 let _ = child1.kill(); 454 let _ = child2.kill(); 455 let output = child2.wait_with_output().unwrap(); 456 handle_child_output(r, &output); 457 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 458 } 459 460 (child1, child2) 461 } 462 463 enum FwType { 464 Ovmf, 465 RustHypervisorFirmware, 466 } 467 468 fn fw_path(_fw_type: FwType) -> String { 469 let mut workload_path = dirs::home_dir().unwrap(); 470 workload_path.push("workloads"); 471 472 let mut fw_path = workload_path; 473 #[cfg(target_arch = "aarch64")] 474 fw_path.push("CLOUDHV_EFI.fd"); 475 #[cfg(target_arch = "x86_64")] 476 { 477 match _fw_type { 478 FwType::Ovmf => fw_path.push(OVMF_NAME), 479 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 480 } 481 } 482 483 fw_path.to_str().unwrap().to_string() 484 } 485 486 struct MetaEvent { 487 event: String, 488 device_id: Option<String>, 489 } 490 491 impl MetaEvent { 492 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 493 let mut matched = false; 494 if v["event"].as_str().unwrap() == self.event { 495 if let Some(device_id) = &self.device_id { 496 if v["properties"]["id"].as_str().unwrap() == device_id { 497 matched = true 498 } 499 } else { 500 matched = true; 501 } 502 } 503 matched 504 } 505 } 506 507 // Parse the event_monitor file based on the format that each event 508 // is followed by a double newline 509 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 510 let content = fs::read(event_file).unwrap(); 511 let mut ret = Vec::new(); 512 for entry in String::from_utf8_lossy(&content) 513 .trim() 514 .split("\n\n") 515 .collect::<Vec<&str>>() 516 { 517 ret.push(serde_json::from_str(entry).unwrap()); 518 } 519 520 ret 521 } 522 523 // Return true if all events from the input 'expected_events' are matched sequentially 524 // with events from the 'event_file' 525 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 526 let json_events = parse_event_file(event_file); 527 let len = expected_events.len(); 528 let mut idx = 0; 529 for e in &json_events { 530 if idx == len { 531 break; 532 } 533 if expected_events[idx].match_with_json_event(e) { 534 idx += 1; 535 } 536 } 537 538 idx == len 539 } 540 541 // Return true if all events from the input 'expected_events' are matched exactly 542 // with events from the 'event_file' 543 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 544 let json_events = parse_event_file(event_file); 545 assert!(expected_events.len() <= json_events.len()); 546 let json_events = &json_events[..expected_events.len()]; 547 548 for (idx, e) in json_events.iter().enumerate() { 549 if !expected_events[idx].match_with_json_event(e) { 550 return false; 551 } 552 } 553 554 true 555 } 556 557 // Return true if events from the input 'expected_events' are matched exactly 558 // with the most recent events from the 'event_file' 559 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 560 let json_events = parse_event_file(event_file); 561 assert!(latest_events.len() <= json_events.len()); 562 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 563 564 for (idx, e) in json_events.iter().enumerate() { 565 if !latest_events[idx].match_with_json_event(e) { 566 return false; 567 } 568 } 569 570 true 571 } 572 573 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 574 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 575 let guest = Guest::new(Box::new(focal)); 576 let total_vcpus = threads_per_core * cores_per_package * packages; 577 let direct_kernel_boot_path = direct_kernel_boot_path(); 578 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 579 let fw_path = fw_path(FwType::RustHypervisorFirmware); 580 if use_fw { 581 kernel_path = fw_path.as_str(); 582 } 583 584 let mut child = GuestCommand::new(&guest) 585 .args([ 586 "--cpus", 587 &format!( 588 "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" 589 ), 590 ]) 591 .args(["--memory", "size=512M"]) 592 .args(["--kernel", kernel_path]) 593 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 594 .default_disks() 595 .default_net() 596 .capture_output() 597 .spawn() 598 .unwrap(); 599 600 let r = std::panic::catch_unwind(|| { 601 guest.wait_vm_boot(None).unwrap(); 602 assert_eq!( 603 guest.get_cpu_count().unwrap_or_default(), 604 u32::from(total_vcpus) 605 ); 606 assert_eq!( 607 guest 608 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 609 .unwrap() 610 .trim() 611 .parse::<u8>() 612 .unwrap_or(0), 613 threads_per_core 614 ); 615 616 assert_eq!( 617 guest 618 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 619 .unwrap() 620 .trim() 621 .parse::<u8>() 622 .unwrap_or(0), 623 cores_per_package 624 ); 625 626 assert_eq!( 627 guest 628 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 629 .unwrap() 630 .trim() 631 .parse::<u8>() 632 .unwrap_or(0), 633 packages 634 ); 635 }); 636 637 let _ = child.kill(); 638 let output = child.wait_with_output().unwrap(); 639 640 handle_child_output(r, &output); 641 } 642 643 #[allow(unused_variables)] 644 fn _test_guest_numa_nodes(acpi: bool) { 645 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 646 let guest = Guest::new(Box::new(focal)); 647 let api_socket = temp_api_path(&guest.tmp_dir); 648 #[cfg(target_arch = "x86_64")] 649 let kernel_path = direct_kernel_boot_path(); 650 #[cfg(target_arch = "aarch64")] 651 let kernel_path = if acpi { 652 edk2_path() 653 } else { 654 direct_kernel_boot_path() 655 }; 656 657 let mut child = GuestCommand::new(&guest) 658 .args(["--cpus", "boot=6,max=12"]) 659 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 660 .args([ 661 "--memory-zone", 662 "id=mem0,size=1G,hotplug_size=3G", 663 "id=mem1,size=2G,hotplug_size=3G", 664 "id=mem2,size=3G,hotplug_size=3G", 665 ]) 666 .args([ 667 "--numa", 668 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 669 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 670 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 671 ]) 672 .args(["--kernel", kernel_path.to_str().unwrap()]) 673 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 674 .args(["--api-socket", &api_socket]) 675 .capture_output() 676 .default_disks() 677 .default_net() 678 .spawn() 679 .unwrap(); 680 681 let r = std::panic::catch_unwind(|| { 682 guest.wait_vm_boot(None).unwrap(); 683 684 guest.check_numa_common( 685 Some(&[960_000, 1_920_000, 2_880_000]), 686 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 687 Some(&["10 15 20", "20 10 25", "25 30 10"]), 688 ); 689 690 // AArch64 currently does not support hotplug, and therefore we only 691 // test hotplug-related function on x86_64 here. 692 #[cfg(target_arch = "x86_64")] 693 { 694 guest.enable_memory_hotplug(); 695 696 // Resize every memory zone and check each associated NUMA node 697 // has been assigned the right amount of memory. 698 resize_zone_command(&api_socket, "mem0", "4G"); 699 resize_zone_command(&api_socket, "mem1", "4G"); 700 resize_zone_command(&api_socket, "mem2", "4G"); 701 // Resize to the maximum amount of CPUs and check each NUMA 702 // node has been assigned the right CPUs set. 703 resize_command(&api_socket, Some(12), None, None, None); 704 thread::sleep(std::time::Duration::new(5, 0)); 705 706 guest.check_numa_common( 707 Some(&[3_840_000, 3_840_000, 3_840_000]), 708 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 709 None, 710 ); 711 } 712 }); 713 714 let _ = child.kill(); 715 let output = child.wait_with_output().unwrap(); 716 717 handle_child_output(r, &output); 718 } 719 720 #[allow(unused_variables)] 721 fn _test_power_button(acpi: bool) { 722 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 723 let guest = Guest::new(Box::new(focal)); 724 let mut cmd = GuestCommand::new(&guest); 725 let api_socket = temp_api_path(&guest.tmp_dir); 726 727 #[cfg(target_arch = "x86_64")] 728 let kernel_path = direct_kernel_boot_path(); 729 #[cfg(target_arch = "aarch64")] 730 let kernel_path = if acpi { 731 edk2_path() 732 } else { 733 direct_kernel_boot_path() 734 }; 735 736 cmd.args(["--cpus", "boot=1"]) 737 .args(["--memory", "size=512M"]) 738 .args(["--kernel", kernel_path.to_str().unwrap()]) 739 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 740 .capture_output() 741 .default_disks() 742 .default_net() 743 .args(["--api-socket", &api_socket]); 744 745 let child = cmd.spawn().unwrap(); 746 747 let r = std::panic::catch_unwind(|| { 748 guest.wait_vm_boot(None).unwrap(); 749 assert!(remote_command(&api_socket, "power-button", None)); 750 }); 751 752 let output = child.wait_with_output().unwrap(); 753 assert!(output.status.success()); 754 handle_child_output(r, &output); 755 } 756 757 type PrepareNetDaemon = dyn Fn( 758 &TempDir, 759 &str, 760 Option<&str>, 761 Option<u16>, 762 usize, 763 bool, 764 ) -> (std::process::Command, String); 765 766 fn test_vhost_user_net( 767 tap: Option<&str>, 768 num_queues: usize, 769 prepare_daemon: &PrepareNetDaemon, 770 generate_host_mac: bool, 771 client_mode_daemon: bool, 772 ) { 773 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 774 let guest = Guest::new(Box::new(focal)); 775 let api_socket = temp_api_path(&guest.tmp_dir); 776 777 let kernel_path = direct_kernel_boot_path(); 778 779 let host_mac = if generate_host_mac { 780 Some(MacAddr::local_random()) 781 } else { 782 None 783 }; 784 785 let mtu = Some(3000); 786 787 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 788 &guest.tmp_dir, 789 &guest.network.host_ip, 790 tap, 791 mtu, 792 num_queues, 793 client_mode_daemon, 794 ); 795 796 let net_params = format!( 797 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 798 guest.network.guest_mac, 799 vunet_socket_path, 800 num_queues, 801 if let Some(host_mac) = host_mac { 802 format!(",host_mac={host_mac}") 803 } else { 804 "".to_owned() 805 }, 806 if client_mode_daemon { 807 "server" 808 } else { 809 "client" 810 }, 811 ); 812 813 let mut ch_command = GuestCommand::new(&guest); 814 ch_command 815 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 816 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 817 .args(["--kernel", kernel_path.to_str().unwrap()]) 818 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 819 .default_disks() 820 .args(["--net", net_params.as_str()]) 821 .args(["--api-socket", &api_socket]) 822 .capture_output(); 823 824 let mut daemon_child: std::process::Child; 825 let mut child: std::process::Child; 826 827 if client_mode_daemon { 828 child = ch_command.spawn().unwrap(); 829 // Make sure the VMM is waiting for the backend to connect 830 thread::sleep(std::time::Duration::new(10, 0)); 831 daemon_child = daemon_command.spawn().unwrap(); 832 } else { 833 daemon_child = daemon_command.spawn().unwrap(); 834 // Make sure the backend is waiting for the VMM to connect 835 thread::sleep(std::time::Duration::new(10, 0)); 836 child = ch_command.spawn().unwrap(); 837 } 838 839 let r = std::panic::catch_unwind(|| { 840 guest.wait_vm_boot(None).unwrap(); 841 842 if let Some(tap_name) = tap { 843 let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); 844 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 845 } 846 847 if let Some(host_mac) = tap { 848 let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); 849 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 850 } 851 852 #[cfg(target_arch = "aarch64")] 853 let iface = "enp0s4"; 854 #[cfg(target_arch = "x86_64")] 855 let iface = "ens4"; 856 857 assert_eq!( 858 guest 859 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 860 .unwrap() 861 .trim(), 862 "3000" 863 ); 864 865 // 1 network interface + default localhost ==> 2 interfaces 866 // It's important to note that this test is fully exercising the 867 // vhost-user-net implementation and the associated backend since 868 // it does not define any --net network interface. That means all 869 // the ssh communication in that test happens through the network 870 // interface backed by vhost-user-net. 871 assert_eq!( 872 guest 873 .ssh_command("ip -o link | wc -l") 874 .unwrap() 875 .trim() 876 .parse::<u32>() 877 .unwrap_or_default(), 878 2 879 ); 880 881 // The following pci devices will appear on guest with PCI-MSI 882 // interrupt vectors assigned. 883 // 1 virtio-console with 3 vectors: config, Rx, Tx 884 // 1 virtio-blk with 2 vectors: config, Request 885 // 1 virtio-blk with 2 vectors: config, Request 886 // 1 virtio-rng with 2 vectors: config, Request 887 // Since virtio-net has 2 queue pairs, its vectors is as follows: 888 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 889 // Based on the above, the total vectors should 14. 890 #[cfg(target_arch = "x86_64")] 891 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 892 #[cfg(target_arch = "aarch64")] 893 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 894 assert_eq!( 895 guest 896 .ssh_command(grep_cmd) 897 .unwrap() 898 .trim() 899 .parse::<u32>() 900 .unwrap_or_default(), 901 10 + (num_queues as u32) 902 ); 903 904 // ACPI feature is needed. 905 #[cfg(target_arch = "x86_64")] 906 { 907 guest.enable_memory_hotplug(); 908 909 // Add RAM to the VM 910 let desired_ram = 1024 << 20; 911 resize_command(&api_socket, None, Some(desired_ram), None, None); 912 913 thread::sleep(std::time::Duration::new(10, 0)); 914 915 // Here by simply checking the size (through ssh), we validate 916 // the connection is still working, which means vhost-user-net 917 // keeps working after the resize. 918 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 919 } 920 }); 921 922 let _ = child.kill(); 923 let output = child.wait_with_output().unwrap(); 924 925 thread::sleep(std::time::Duration::new(5, 0)); 926 let _ = daemon_child.kill(); 927 let _ = daemon_child.wait(); 928 929 handle_child_output(r, &output); 930 } 931 932 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 933 934 fn test_vhost_user_blk( 935 num_queues: usize, 936 readonly: bool, 937 direct: bool, 938 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 939 ) { 940 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 941 let guest = Guest::new(Box::new(focal)); 942 let api_socket = temp_api_path(&guest.tmp_dir); 943 944 let kernel_path = direct_kernel_boot_path(); 945 946 let (blk_params, daemon_child) = { 947 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 948 // Start the daemon 949 let (daemon_child, vubd_socket_path) = 950 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 951 952 ( 953 format!( 954 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 955 ), 956 Some(daemon_child), 957 ) 958 }; 959 960 let mut child = GuestCommand::new(&guest) 961 .args(["--cpus", format!("boot={num_queues}").as_str()]) 962 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 963 .args(["--kernel", kernel_path.to_str().unwrap()]) 964 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 965 .args([ 966 "--disk", 967 format!( 968 "path={}", 969 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 970 ) 971 .as_str(), 972 format!( 973 "path={}", 974 guest.disk_config.disk(DiskType::CloudInit).unwrap() 975 ) 976 .as_str(), 977 blk_params.as_str(), 978 ]) 979 .default_net() 980 .args(["--api-socket", &api_socket]) 981 .capture_output() 982 .spawn() 983 .unwrap(); 984 985 let r = std::panic::catch_unwind(|| { 986 guest.wait_vm_boot(None).unwrap(); 987 988 // Check both if /dev/vdc exists and if the block size is 16M. 989 assert_eq!( 990 guest 991 .ssh_command("lsblk | grep vdc | grep -c 16M") 992 .unwrap() 993 .trim() 994 .parse::<u32>() 995 .unwrap_or_default(), 996 1 997 ); 998 999 // Check if this block is RO or RW. 1000 assert_eq!( 1001 guest 1002 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1003 .unwrap() 1004 .trim() 1005 .parse::<u32>() 1006 .unwrap_or_default(), 1007 readonly as u32 1008 ); 1009 1010 // Check if the number of queues in /sys/block/vdc/mq matches the 1011 // expected num_queues. 1012 assert_eq!( 1013 guest 1014 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1015 .unwrap() 1016 .trim() 1017 .parse::<u32>() 1018 .unwrap_or_default(), 1019 num_queues as u32 1020 ); 1021 1022 // Mount the device 1023 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1024 guest.ssh_command("mkdir mount_image").unwrap(); 1025 guest 1026 .ssh_command( 1027 format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), 1028 ) 1029 .unwrap(); 1030 1031 // Check the content of the block device. The file "foo" should 1032 // contain "bar". 1033 assert_eq!( 1034 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1035 "bar" 1036 ); 1037 1038 // ACPI feature is needed. 1039 #[cfg(target_arch = "x86_64")] 1040 { 1041 guest.enable_memory_hotplug(); 1042 1043 // Add RAM to the VM 1044 let desired_ram = 1024 << 20; 1045 resize_command(&api_socket, None, Some(desired_ram), None, None); 1046 1047 thread::sleep(std::time::Duration::new(10, 0)); 1048 1049 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1050 1051 // Check again the content of the block device after the resize 1052 // has been performed. 1053 assert_eq!( 1054 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1055 "bar" 1056 ); 1057 } 1058 1059 // Unmount the device 1060 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1061 guest.ssh_command("rm -r mount_image").unwrap(); 1062 }); 1063 1064 let _ = child.kill(); 1065 let output = child.wait_with_output().unwrap(); 1066 1067 if let Some(mut daemon_child) = daemon_child { 1068 thread::sleep(std::time::Duration::new(5, 0)); 1069 let _ = daemon_child.kill(); 1070 let _ = daemon_child.wait(); 1071 } 1072 1073 handle_child_output(r, &output); 1074 } 1075 1076 fn test_boot_from_vhost_user_blk( 1077 num_queues: usize, 1078 readonly: bool, 1079 direct: bool, 1080 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1081 ) { 1082 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1083 let guest = Guest::new(Box::new(focal)); 1084 1085 let kernel_path = direct_kernel_boot_path(); 1086 1087 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1088 1089 let (blk_boot_params, daemon_child) = { 1090 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1091 // Start the daemon 1092 let (daemon_child, vubd_socket_path) = prepare_daemon( 1093 &guest.tmp_dir, 1094 disk_path.as_str(), 1095 num_queues, 1096 readonly, 1097 direct, 1098 ); 1099 1100 ( 1101 format!( 1102 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1103 ), 1104 Some(daemon_child), 1105 ) 1106 }; 1107 1108 let mut child = GuestCommand::new(&guest) 1109 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1110 .args(["--memory", "size=512M,shared=on"]) 1111 .args(["--kernel", kernel_path.to_str().unwrap()]) 1112 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1113 .args([ 1114 "--disk", 1115 blk_boot_params.as_str(), 1116 format!( 1117 "path={}", 1118 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1119 ) 1120 .as_str(), 1121 ]) 1122 .default_net() 1123 .capture_output() 1124 .spawn() 1125 .unwrap(); 1126 1127 let r = std::panic::catch_unwind(|| { 1128 guest.wait_vm_boot(None).unwrap(); 1129 1130 // Just check the VM booted correctly. 1131 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1132 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1133 }); 1134 let _ = child.kill(); 1135 let output = child.wait_with_output().unwrap(); 1136 1137 if let Some(mut daemon_child) = daemon_child { 1138 thread::sleep(std::time::Duration::new(5, 0)); 1139 let _ = daemon_child.kill(); 1140 let _ = daemon_child.wait(); 1141 } 1142 1143 handle_child_output(r, &output); 1144 } 1145 1146 fn _test_virtio_fs( 1147 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1148 hotplug: bool, 1149 pci_segment: Option<u16>, 1150 ) { 1151 #[cfg(target_arch = "aarch64")] 1152 let focal_image = if hotplug { 1153 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1154 } else { 1155 FOCAL_IMAGE_NAME.to_string() 1156 }; 1157 #[cfg(target_arch = "x86_64")] 1158 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1159 let focal = UbuntuDiskConfig::new(focal_image); 1160 let guest = Guest::new(Box::new(focal)); 1161 let api_socket = temp_api_path(&guest.tmp_dir); 1162 1163 let mut workload_path = dirs::home_dir().unwrap(); 1164 workload_path.push("workloads"); 1165 1166 let mut shared_dir = workload_path; 1167 shared_dir.push("shared_dir"); 1168 1169 #[cfg(target_arch = "x86_64")] 1170 let kernel_path = direct_kernel_boot_path(); 1171 #[cfg(target_arch = "aarch64")] 1172 let kernel_path = if hotplug { 1173 edk2_path() 1174 } else { 1175 direct_kernel_boot_path() 1176 }; 1177 1178 let (mut daemon_child, virtiofsd_socket_path) = 1179 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1180 1181 let mut guest_command = GuestCommand::new(&guest); 1182 guest_command 1183 .args(["--cpus", "boot=1"]) 1184 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1185 .args(["--kernel", kernel_path.to_str().unwrap()]) 1186 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1187 .default_disks() 1188 .default_net() 1189 .args(["--api-socket", &api_socket]); 1190 if pci_segment.is_some() { 1191 guest_command.args(["--platform", "num_pci_segments=16"]); 1192 } 1193 1194 let fs_params = format!( 1195 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1196 virtiofsd_socket_path, 1197 if let Some(pci_segment) = pci_segment { 1198 format!(",pci_segment={pci_segment}") 1199 } else { 1200 "".to_owned() 1201 } 1202 ); 1203 1204 if !hotplug { 1205 guest_command.args(["--fs", fs_params.as_str()]); 1206 } 1207 1208 let mut child = guest_command.capture_output().spawn().unwrap(); 1209 1210 let r = std::panic::catch_unwind(|| { 1211 guest.wait_vm_boot(None).unwrap(); 1212 1213 if hotplug { 1214 // Add fs to the VM 1215 let (cmd_success, cmd_output) = 1216 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1217 assert!(cmd_success); 1218 1219 if let Some(pci_segment) = pci_segment { 1220 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1221 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1222 ))); 1223 } else { 1224 assert!(String::from_utf8_lossy(&cmd_output) 1225 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1226 } 1227 1228 thread::sleep(std::time::Duration::new(10, 0)); 1229 } 1230 1231 // Mount shared directory through virtio_fs filesystem 1232 guest 1233 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1234 .unwrap(); 1235 1236 // Check file1 exists and its content is "foo" 1237 assert_eq!( 1238 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1239 "foo" 1240 ); 1241 // Check file2 does not exist 1242 guest 1243 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1244 .unwrap(); 1245 1246 // Check file3 exists and its content is "bar" 1247 assert_eq!( 1248 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1249 "bar" 1250 ); 1251 1252 // ACPI feature is needed. 1253 #[cfg(target_arch = "x86_64")] 1254 { 1255 guest.enable_memory_hotplug(); 1256 1257 // Add RAM to the VM 1258 let desired_ram = 1024 << 20; 1259 resize_command(&api_socket, None, Some(desired_ram), None, None); 1260 1261 thread::sleep(std::time::Duration::new(30, 0)); 1262 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1263 1264 // After the resize, check again that file1 exists and its 1265 // content is "foo". 1266 assert_eq!( 1267 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1268 "foo" 1269 ); 1270 } 1271 1272 if hotplug { 1273 // Remove from VM 1274 guest.ssh_command("sudo umount mount_dir").unwrap(); 1275 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1276 } 1277 }); 1278 1279 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1280 thread::sleep(std::time::Duration::new(10, 0)); 1281 let (daemon_child, virtiofsd_socket_path) = 1282 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1283 1284 let r = std::panic::catch_unwind(|| { 1285 thread::sleep(std::time::Duration::new(10, 0)); 1286 let fs_params = format!( 1287 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1288 virtiofsd_socket_path, 1289 if let Some(pci_segment) = pci_segment { 1290 format!(",pci_segment={pci_segment}") 1291 } else { 1292 "".to_owned() 1293 } 1294 ); 1295 1296 // Add back and check it works 1297 let (cmd_success, cmd_output) = 1298 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1299 assert!(cmd_success); 1300 if let Some(pci_segment) = pci_segment { 1301 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1302 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1303 ))); 1304 } else { 1305 assert!(String::from_utf8_lossy(&cmd_output) 1306 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1307 } 1308 1309 thread::sleep(std::time::Duration::new(10, 0)); 1310 // Mount shared directory through virtio_fs filesystem 1311 guest 1312 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1313 .unwrap(); 1314 1315 // Check file1 exists and its content is "foo" 1316 assert_eq!( 1317 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1318 "foo" 1319 ); 1320 }); 1321 1322 (r, Some(daemon_child)) 1323 } else { 1324 (r, None) 1325 }; 1326 1327 let _ = child.kill(); 1328 let output = child.wait_with_output().unwrap(); 1329 1330 let _ = daemon_child.kill(); 1331 let _ = daemon_child.wait(); 1332 1333 if let Some(mut daemon_child) = hotplug_daemon_child { 1334 let _ = daemon_child.kill(); 1335 let _ = daemon_child.wait(); 1336 } 1337 1338 handle_child_output(r, &output); 1339 } 1340 1341 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1342 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1343 let guest = Guest::new(Box::new(focal)); 1344 1345 let kernel_path = direct_kernel_boot_path(); 1346 1347 let pmem_temp_file = TempFile::new().unwrap(); 1348 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1349 1350 std::process::Command::new("mkfs.ext4") 1351 .arg(pmem_temp_file.as_path()) 1352 .output() 1353 .expect("Expect creating disk image to succeed"); 1354 1355 let mut child = GuestCommand::new(&guest) 1356 .args(["--cpus", "boot=1"]) 1357 .args(["--memory", "size=512M"]) 1358 .args(["--kernel", kernel_path.to_str().unwrap()]) 1359 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1360 .default_disks() 1361 .default_net() 1362 .args([ 1363 "--pmem", 1364 format!( 1365 "file={}{}{}", 1366 pmem_temp_file.as_path().to_str().unwrap(), 1367 if specify_size { ",size=128M" } else { "" }, 1368 if discard_writes { 1369 ",discard_writes=on" 1370 } else { 1371 "" 1372 } 1373 ) 1374 .as_str(), 1375 ]) 1376 .capture_output() 1377 .spawn() 1378 .unwrap(); 1379 1380 let r = std::panic::catch_unwind(|| { 1381 guest.wait_vm_boot(None).unwrap(); 1382 1383 // Check for the presence of /dev/pmem0 1384 assert_eq!( 1385 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1386 "/dev/pmem0" 1387 ); 1388 1389 // Check changes persist after reboot 1390 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1391 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1392 guest 1393 .ssh_command("echo test123 | sudo tee /mnt/test") 1394 .unwrap(); 1395 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1396 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1397 1398 guest.reboot_linux(0, None); 1399 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1400 assert_eq!( 1401 guest 1402 .ssh_command("sudo cat /mnt/test || true") 1403 .unwrap() 1404 .trim(), 1405 if discard_writes { "" } else { "test123" } 1406 ); 1407 }); 1408 1409 let _ = child.kill(); 1410 let output = child.wait_with_output().unwrap(); 1411 1412 handle_child_output(r, &output); 1413 } 1414 1415 fn get_fd_count(pid: u32) -> usize { 1416 fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() 1417 } 1418 1419 fn _test_virtio_vsock(hotplug: bool) { 1420 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1421 let guest = Guest::new(Box::new(focal)); 1422 1423 #[cfg(target_arch = "x86_64")] 1424 let kernel_path = direct_kernel_boot_path(); 1425 #[cfg(target_arch = "aarch64")] 1426 let kernel_path = if hotplug { 1427 edk2_path() 1428 } else { 1429 direct_kernel_boot_path() 1430 }; 1431 1432 let socket = temp_vsock_path(&guest.tmp_dir); 1433 let api_socket = temp_api_path(&guest.tmp_dir); 1434 1435 let mut cmd = GuestCommand::new(&guest); 1436 cmd.args(["--api-socket", &api_socket]); 1437 cmd.args(["--cpus", "boot=1"]); 1438 cmd.args(["--memory", "size=512M"]); 1439 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1440 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1441 cmd.default_disks(); 1442 cmd.default_net(); 1443 1444 if !hotplug { 1445 cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); 1446 } 1447 1448 let mut child = cmd.capture_output().spawn().unwrap(); 1449 1450 let r = std::panic::catch_unwind(|| { 1451 guest.wait_vm_boot(None).unwrap(); 1452 1453 if hotplug { 1454 let (cmd_success, cmd_output) = remote_command_w_output( 1455 &api_socket, 1456 "add-vsock", 1457 Some(format!("cid=3,socket={socket},id=test0").as_str()), 1458 ); 1459 assert!(cmd_success); 1460 assert!(String::from_utf8_lossy(&cmd_output) 1461 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1462 thread::sleep(std::time::Duration::new(10, 0)); 1463 // Check adding a second one fails 1464 assert!(!remote_command( 1465 &api_socket, 1466 "add-vsock", 1467 Some("cid=1234,socket=/tmp/fail") 1468 )); 1469 } 1470 1471 // Validate vsock works as expected. 1472 guest.check_vsock(socket.as_str()); 1473 guest.reboot_linux(0, None); 1474 // Validate vsock still works after a reboot. 1475 guest.check_vsock(socket.as_str()); 1476 1477 if hotplug { 1478 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1479 } 1480 }); 1481 1482 let _ = child.kill(); 1483 let output = child.wait_with_output().unwrap(); 1484 1485 handle_child_output(r, &output); 1486 } 1487 1488 fn get_ksm_pages_shared() -> u32 { 1489 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1490 .unwrap() 1491 .trim() 1492 .parse::<u32>() 1493 .unwrap() 1494 } 1495 1496 fn test_memory_mergeable(mergeable: bool) { 1497 let memory_param = if mergeable { 1498 "mergeable=on" 1499 } else { 1500 "mergeable=off" 1501 }; 1502 1503 // We are assuming the rest of the system in our CI is not using mergeable memeory 1504 let ksm_ps_init = get_ksm_pages_shared(); 1505 assert!(ksm_ps_init == 0); 1506 1507 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1508 let guest1 = Guest::new(Box::new(focal1)); 1509 let mut child1 = GuestCommand::new(&guest1) 1510 .args(["--cpus", "boot=1"]) 1511 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1512 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1513 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1514 .default_disks() 1515 .args(["--net", guest1.default_net_string().as_str()]) 1516 .args(["--serial", "tty", "--console", "off"]) 1517 .capture_output() 1518 .spawn() 1519 .unwrap(); 1520 1521 let r = std::panic::catch_unwind(|| { 1522 guest1.wait_vm_boot(None).unwrap(); 1523 }); 1524 if r.is_err() { 1525 let _ = child1.kill(); 1526 let output = child1.wait_with_output().unwrap(); 1527 handle_child_output(r, &output); 1528 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1529 } 1530 1531 let ksm_ps_guest1 = get_ksm_pages_shared(); 1532 1533 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1534 let guest2 = Guest::new(Box::new(focal2)); 1535 let mut child2 = GuestCommand::new(&guest2) 1536 .args(["--cpus", "boot=1"]) 1537 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1538 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1539 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1540 .default_disks() 1541 .args(["--net", guest2.default_net_string().as_str()]) 1542 .args(["--serial", "tty", "--console", "off"]) 1543 .capture_output() 1544 .spawn() 1545 .unwrap(); 1546 1547 let r = std::panic::catch_unwind(|| { 1548 guest2.wait_vm_boot(None).unwrap(); 1549 let ksm_ps_guest2 = get_ksm_pages_shared(); 1550 1551 if mergeable { 1552 println!( 1553 "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" 1554 ); 1555 // We are expecting the number of shared pages to increase as the number of VM increases 1556 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1557 } else { 1558 assert!(ksm_ps_guest1 == 0); 1559 assert!(ksm_ps_guest2 == 0); 1560 } 1561 }); 1562 1563 let _ = child1.kill(); 1564 let _ = child2.kill(); 1565 1566 let output = child1.wait_with_output().unwrap(); 1567 child2.wait().unwrap(); 1568 1569 handle_child_output(r, &output); 1570 } 1571 1572 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 1573 let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); 1574 let reader = io::BufReader::new(smaps); 1575 1576 let mut skip_map: bool = false; 1577 let mut region_name: String = "".to_string(); 1578 let mut region_maps = HashMap::new(); 1579 for line in reader.lines() { 1580 let l = line.unwrap(); 1581 1582 if l.contains('-') { 1583 let values: Vec<&str> = l.split_whitespace().collect(); 1584 region_name = values.last().unwrap().trim().to_string(); 1585 if region_name == "0" { 1586 region_name = "anonymous".to_string() 1587 } 1588 } 1589 1590 // Each section begins with something that looks like: 1591 // Size: 2184 kB 1592 if l.starts_with("Size:") { 1593 let values: Vec<&str> = l.split_whitespace().collect(); 1594 let map_size = values[1].parse::<u32>().unwrap(); 1595 // We skip the assigned guest RAM map, its RSS is only 1596 // dependent on the guest actual memory usage. 1597 // Everything else can be added to the VMM overhead. 1598 skip_map = map_size >= guest_memory_size; 1599 continue; 1600 } 1601 1602 // If this is a map we're taking into account, then we only 1603 // count the RSS. The sum of all counted RSS is the VMM overhead. 1604 if !skip_map && l.starts_with("Rss:") { 1605 let values: Vec<&str> = l.split_whitespace().collect(); 1606 let value = values[1].trim().parse::<u32>().unwrap(); 1607 *region_maps.entry(region_name.clone()).or_insert(0) += value; 1608 } 1609 } 1610 1611 region_maps 1612 } 1613 1614 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 1615 let mut total = 0; 1616 1617 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 1618 eprintln!("{region_name}: {value}"); 1619 total += value; 1620 } 1621 1622 total 1623 } 1624 1625 fn process_rss_kib(pid: u32) -> usize { 1626 let command = format!("ps -q {pid} -o rss="); 1627 let rss = exec_host_command_output(&command); 1628 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 1629 } 1630 1631 // 10MB is our maximum accepted overhead. 1632 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 1633 1634 #[derive(PartialEq, Eq, PartialOrd)] 1635 struct Counters { 1636 rx_bytes: u64, 1637 rx_frames: u64, 1638 tx_bytes: u64, 1639 tx_frames: u64, 1640 read_bytes: u64, 1641 write_bytes: u64, 1642 read_ops: u64, 1643 write_ops: u64, 1644 } 1645 1646 fn get_counters(api_socket: &str) -> Counters { 1647 // Get counters 1648 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 1649 assert!(cmd_success); 1650 1651 let counters: HashMap<&str, HashMap<&str, u64>> = 1652 serde_json::from_slice(&cmd_output).unwrap_or_default(); 1653 1654 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 1655 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 1656 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 1657 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 1658 1659 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 1660 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 1661 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 1662 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 1663 1664 Counters { 1665 rx_bytes, 1666 rx_frames, 1667 tx_bytes, 1668 tx_frames, 1669 read_bytes, 1670 write_bytes, 1671 read_ops, 1672 write_ops, 1673 } 1674 } 1675 1676 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 1677 let (tx, rx) = mpsc::channel::<String>(); 1678 thread::spawn(move || loop { 1679 thread::sleep(std::time::Duration::new(1, 0)); 1680 let mut buf = [0; 512]; 1681 match pty.read(&mut buf) { 1682 Ok(_) => { 1683 let output = std::str::from_utf8(&buf).unwrap().to_string(); 1684 match tx.send(output) { 1685 Ok(_) => (), 1686 Err(_) => break, 1687 } 1688 } 1689 Err(_) => break, 1690 } 1691 }); 1692 rx 1693 } 1694 1695 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 1696 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1697 assert!(cmd_success); 1698 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1699 assert_eq!("Pty", info["config"][pty_type]["mode"]); 1700 PathBuf::from( 1701 info["config"][pty_type]["file"] 1702 .as_str() 1703 .expect("Missing pty path"), 1704 ) 1705 } 1706 1707 // VFIO test network setup. 1708 // We reserve a different IP class for it: 172.18.0.0/24. 1709 #[cfg(target_arch = "x86_64")] 1710 fn setup_vfio_network_interfaces() { 1711 // 'vfio-br0' 1712 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 1713 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 1714 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 1715 // 'vfio-tap0' 1716 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 1717 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 1718 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 1719 // 'vfio-tap1' 1720 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 1721 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 1722 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 1723 // 'vfio-tap2' 1724 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 1725 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 1726 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 1727 // 'vfio-tap3' 1728 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 1729 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 1730 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 1731 } 1732 1733 // Tear VFIO test network down 1734 #[cfg(target_arch = "x86_64")] 1735 fn cleanup_vfio_network_interfaces() { 1736 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 1737 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 1738 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 1739 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 1740 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 1741 } 1742 1743 fn balloon_size(api_socket: &str) -> u64 { 1744 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1745 assert!(cmd_success); 1746 1747 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1748 let total_mem = &info["config"]["memory"]["size"] 1749 .to_string() 1750 .parse::<u64>() 1751 .unwrap(); 1752 let actual_mem = &info["memory_actual_size"] 1753 .to_string() 1754 .parse::<u64>() 1755 .unwrap(); 1756 total_mem - actual_mem 1757 } 1758 1759 // This test validates that it can find the virtio-iommu device at first. 1760 // It also verifies that both disks and the network card are attached to 1761 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 1762 // The last interesting part of this test is that it exercises the network 1763 // interface attached to the virtual IOMMU since this is the one used to 1764 // send all commands through SSH. 1765 fn _test_virtio_iommu(acpi: bool) { 1766 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 1767 // Focal image is still old. 1768 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 1769 // the kernel binary has been updated. 1770 #[cfg(target_arch = "aarch64")] 1771 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 1772 #[cfg(target_arch = "x86_64")] 1773 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1774 let focal = UbuntuDiskConfig::new(focal_image); 1775 let guest = Guest::new(Box::new(focal)); 1776 1777 #[cfg(target_arch = "x86_64")] 1778 let kernel_path = direct_kernel_boot_path(); 1779 #[cfg(target_arch = "aarch64")] 1780 let kernel_path = if acpi { 1781 edk2_path() 1782 } else { 1783 direct_kernel_boot_path() 1784 }; 1785 1786 let mut child = GuestCommand::new(&guest) 1787 .args(["--cpus", "boot=1"]) 1788 .args(["--memory", "size=512M"]) 1789 .args(["--kernel", kernel_path.to_str().unwrap()]) 1790 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1791 .args([ 1792 "--disk", 1793 format!( 1794 "path={},iommu=on", 1795 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1796 ) 1797 .as_str(), 1798 format!( 1799 "path={},iommu=on", 1800 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1801 ) 1802 .as_str(), 1803 ]) 1804 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 1805 .capture_output() 1806 .spawn() 1807 .unwrap(); 1808 1809 let r = std::panic::catch_unwind(|| { 1810 guest.wait_vm_boot(None).unwrap(); 1811 1812 // Verify the virtio-iommu device is present. 1813 assert!(guest 1814 .does_device_vendor_pair_match("0x1057", "0x1af4") 1815 .unwrap_or_default()); 1816 1817 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 1818 // different with ACPI. 1819 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 1820 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 1821 // and appear under folder '/sys/kernel/iommu_groups/'. 1822 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 1823 // which is the console. The first disk "0000:00:02.0" is in group '1'. 1824 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 1825 // contains "0000:00:02.0" which is the first disk. 1826 // 1827 // Verify the iommu group of the first disk. 1828 let iommu_group = !acpi as i32; 1829 assert_eq!( 1830 guest 1831 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 1832 .unwrap() 1833 .trim(), 1834 "0000:00:02.0" 1835 ); 1836 1837 // Verify the iommu group of the second disk. 1838 let iommu_group = if acpi { 1 } else { 2 }; 1839 assert_eq!( 1840 guest 1841 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 1842 .unwrap() 1843 .trim(), 1844 "0000:00:03.0" 1845 ); 1846 1847 // Verify the iommu group of the network card. 1848 let iommu_group = if acpi { 2 } else { 3 }; 1849 assert_eq!( 1850 guest 1851 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 1852 .unwrap() 1853 .trim(), 1854 "0000:00:04.0" 1855 ); 1856 }); 1857 1858 let _ = child.kill(); 1859 let output = child.wait_with_output().unwrap(); 1860 1861 handle_child_output(r, &output); 1862 } 1863 1864 fn get_reboot_count(guest: &Guest) -> u32 { 1865 guest 1866 .ssh_command("sudo last | grep -c reboot") 1867 .unwrap() 1868 .trim() 1869 .parse::<u32>() 1870 .unwrap_or_default() 1871 } 1872 1873 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 1874 // Check for PCI device 1875 assert!(guest 1876 .does_device_vendor_pair_match("0x1063", "0x1af4") 1877 .unwrap_or_default()); 1878 1879 // Enable systemd watchdog 1880 guest 1881 .ssh_command(&format!( 1882 "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" 1883 )) 1884 .unwrap(); 1885 } 1886 1887 mod common_parallel { 1888 use std::{fs::OpenOptions, io::SeekFrom}; 1889 1890 use crate::*; 1891 1892 #[test] 1893 #[cfg(target_arch = "x86_64")] 1894 fn test_bionic_hypervisor_fw() { 1895 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), BIONIC_IMAGE_NAME) 1896 } 1897 1898 #[test] 1899 #[cfg(target_arch = "x86_64")] 1900 fn test_focal_hypervisor_fw() { 1901 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 1902 } 1903 1904 #[test] 1905 #[cfg(target_arch = "x86_64")] 1906 fn test_bionic_ovmf() { 1907 test_simple_launch(fw_path(FwType::Ovmf), BIONIC_IMAGE_NAME) 1908 } 1909 1910 #[test] 1911 #[cfg(target_arch = "x86_64")] 1912 fn test_focal_ovmf() { 1913 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 1914 } 1915 1916 #[cfg(target_arch = "x86_64")] 1917 fn test_simple_launch(fw_path: String, disk_path: &str) { 1918 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 1919 let guest = Guest::new(disk_config); 1920 let event_path = temp_event_monitor_path(&guest.tmp_dir); 1921 1922 let mut child = GuestCommand::new(&guest) 1923 .args(["--cpus", "boot=1"]) 1924 .args(["--memory", "size=512M"]) 1925 .args(["--kernel", fw_path.as_str()]) 1926 .default_disks() 1927 .default_net() 1928 .args(["--serial", "tty", "--console", "off"]) 1929 .args(["--event-monitor", format!("path={event_path}").as_str()]) 1930 .capture_output() 1931 .spawn() 1932 .unwrap(); 1933 1934 let r = std::panic::catch_unwind(|| { 1935 guest.wait_vm_boot(Some(120)).unwrap(); 1936 1937 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 1938 assert_eq!(guest.get_initial_apicid().unwrap_or(1), 0); 1939 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1940 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 1941 1942 let expected_sequential_events = [ 1943 &MetaEvent { 1944 event: "starting".to_string(), 1945 device_id: None, 1946 }, 1947 &MetaEvent { 1948 event: "booting".to_string(), 1949 device_id: None, 1950 }, 1951 &MetaEvent { 1952 event: "booted".to_string(), 1953 device_id: None, 1954 }, 1955 &MetaEvent { 1956 event: "activated".to_string(), 1957 device_id: Some("_disk0".to_string()), 1958 }, 1959 &MetaEvent { 1960 event: "reset".to_string(), 1961 device_id: Some("_disk0".to_string()), 1962 }, 1963 ]; 1964 assert!(check_sequential_events( 1965 &expected_sequential_events, 1966 &event_path 1967 )); 1968 1969 // It's been observed on the Bionic image that udev and snapd 1970 // services can cause some delay in the VM's shutdown. Disabling 1971 // them improves the reliability of this test. 1972 let _ = guest.ssh_command("sudo systemctl disable udev"); 1973 let _ = guest.ssh_command("sudo systemctl stop udev"); 1974 let _ = guest.ssh_command("sudo systemctl disable snapd"); 1975 let _ = guest.ssh_command("sudo systemctl stop snapd"); 1976 1977 guest.ssh_command("sudo poweroff").unwrap(); 1978 thread::sleep(std::time::Duration::new(20, 0)); 1979 let latest_events = [ 1980 &MetaEvent { 1981 event: "shutdown".to_string(), 1982 device_id: None, 1983 }, 1984 &MetaEvent { 1985 event: "deleted".to_string(), 1986 device_id: None, 1987 }, 1988 &MetaEvent { 1989 event: "shutdown".to_string(), 1990 device_id: None, 1991 }, 1992 ]; 1993 assert!(check_latest_events_exact(&latest_events, &event_path)); 1994 }); 1995 1996 let _ = child.kill(); 1997 let output = child.wait_with_output().unwrap(); 1998 1999 handle_child_output(r, &output); 2000 } 2001 2002 #[test] 2003 fn test_multi_cpu() { 2004 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2005 let jammy = UbuntuDiskConfig::new(jammy_image); 2006 let guest = Guest::new(Box::new(jammy)); 2007 2008 let mut cmd = GuestCommand::new(&guest); 2009 cmd.args(["--cpus", "boot=2,max=4"]) 2010 .args(["--memory", "size=512M"]) 2011 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2012 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2013 .capture_output() 2014 .default_disks() 2015 .default_net(); 2016 2017 let mut child = cmd.spawn().unwrap(); 2018 2019 let r = std::panic::catch_unwind(|| { 2020 guest.wait_vm_boot(Some(120)).unwrap(); 2021 2022 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2023 2024 assert_eq!( 2025 guest 2026 .ssh_command( 2027 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2028 ) 2029 .unwrap() 2030 .trim(), 2031 "smp: Brought up 1 node, 2 CPUs" 2032 ); 2033 }); 2034 2035 let _ = child.kill(); 2036 let output = child.wait_with_output().unwrap(); 2037 2038 handle_child_output(r, &output); 2039 } 2040 2041 #[test] 2042 #[cfg(not(feature = "mshv"))] 2043 fn test_cpu_topology_421() { 2044 test_cpu_topology(4, 2, 1, false); 2045 } 2046 2047 #[test] 2048 #[cfg(not(feature = "mshv"))] 2049 fn test_cpu_topology_142() { 2050 test_cpu_topology(1, 4, 2, false); 2051 } 2052 2053 #[test] 2054 #[cfg(not(feature = "mshv"))] 2055 fn test_cpu_topology_262() { 2056 test_cpu_topology(2, 6, 2, false); 2057 } 2058 2059 #[test] 2060 #[cfg(target_arch = "x86_64")] 2061 #[cfg(not(feature = "mshv"))] 2062 fn test_cpu_physical_bits() { 2063 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2064 let guest = Guest::new(Box::new(focal)); 2065 let max_phys_bits: u8 = 36; 2066 let mut child = GuestCommand::new(&guest) 2067 .args(["--cpus", &format!("max_phys_bits={max_phys_bits}")]) 2068 .args(["--memory", "size=512M"]) 2069 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2070 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2071 .default_disks() 2072 .default_net() 2073 .capture_output() 2074 .spawn() 2075 .unwrap(); 2076 2077 let r = std::panic::catch_unwind(|| { 2078 guest.wait_vm_boot(None).unwrap(); 2079 2080 assert!( 2081 guest 2082 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2083 .unwrap() 2084 .trim() 2085 .parse::<u8>() 2086 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2087 ); 2088 }); 2089 2090 let _ = child.kill(); 2091 let output = child.wait_with_output().unwrap(); 2092 2093 handle_child_output(r, &output); 2094 } 2095 2096 #[test] 2097 fn test_cpu_affinity() { 2098 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2099 let guest = Guest::new(Box::new(focal)); 2100 2101 // We need the host to have at least 4 CPUs if we want to be able 2102 // to run this test. 2103 let host_cpus_count = exec_host_command_output("nproc"); 2104 assert!( 2105 String::from_utf8_lossy(&host_cpus_count.stdout) 2106 .trim() 2107 .parse::<u8>() 2108 .unwrap_or(0) 2109 >= 4 2110 ); 2111 2112 let mut child = GuestCommand::new(&guest) 2113 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2114 .args(["--memory", "size=512M"]) 2115 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2116 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2117 .default_disks() 2118 .default_net() 2119 .capture_output() 2120 .spawn() 2121 .unwrap(); 2122 2123 let r = std::panic::catch_unwind(|| { 2124 guest.wait_vm_boot(None).unwrap(); 2125 let pid = child.id(); 2126 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2127 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2128 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2129 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2130 }); 2131 2132 let _ = child.kill(); 2133 let output = child.wait_with_output().unwrap(); 2134 2135 handle_child_output(r, &output); 2136 } 2137 2138 #[test] 2139 #[cfg(not(feature = "mshv"))] 2140 fn test_large_vm() { 2141 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2142 let guest = Guest::new(Box::new(focal)); 2143 let mut cmd = GuestCommand::new(&guest); 2144 cmd.args(["--cpus", "boot=48"]) 2145 .args(["--memory", "size=5120M"]) 2146 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2147 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2148 .args(["--serial", "tty"]) 2149 .args(["--console", "off"]) 2150 .capture_output() 2151 .default_disks() 2152 .default_net(); 2153 2154 let mut child = cmd.spawn().unwrap(); 2155 2156 guest.wait_vm_boot(None).unwrap(); 2157 2158 let r = std::panic::catch_unwind(|| { 2159 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2160 assert_eq!( 2161 guest 2162 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2163 .unwrap() 2164 .trim(), 2165 "0-47" 2166 ); 2167 2168 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2169 }); 2170 2171 let _ = child.kill(); 2172 let output = child.wait_with_output().unwrap(); 2173 2174 handle_child_output(r, &output); 2175 } 2176 2177 #[test] 2178 #[cfg(not(feature = "mshv"))] 2179 fn test_huge_memory() { 2180 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2181 let guest = Guest::new(Box::new(focal)); 2182 let mut cmd = GuestCommand::new(&guest); 2183 cmd.args(["--cpus", "boot=1"]) 2184 .args(["--memory", "size=128G"]) 2185 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2186 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2187 .capture_output() 2188 .default_disks() 2189 .default_net(); 2190 2191 let mut child = cmd.spawn().unwrap(); 2192 2193 guest.wait_vm_boot(Some(120)).unwrap(); 2194 2195 let r = std::panic::catch_unwind(|| { 2196 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2197 }); 2198 2199 let _ = child.kill(); 2200 let output = child.wait_with_output().unwrap(); 2201 2202 handle_child_output(r, &output); 2203 } 2204 2205 #[test] 2206 fn test_power_button() { 2207 _test_power_button(false); 2208 } 2209 2210 #[test] 2211 #[cfg(not(feature = "mshv"))] 2212 fn test_user_defined_memory_regions() { 2213 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2214 let guest = Guest::new(Box::new(focal)); 2215 let api_socket = temp_api_path(&guest.tmp_dir); 2216 2217 let kernel_path = direct_kernel_boot_path(); 2218 2219 let mut child = GuestCommand::new(&guest) 2220 .args(["--cpus", "boot=1"]) 2221 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2222 .args([ 2223 "--memory-zone", 2224 "id=mem0,size=1G,hotplug_size=2G", 2225 "id=mem1,size=1G,file=/dev/shm", 2226 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2227 ]) 2228 .args(["--kernel", kernel_path.to_str().unwrap()]) 2229 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2230 .args(["--api-socket", &api_socket]) 2231 .capture_output() 2232 .default_disks() 2233 .default_net() 2234 .spawn() 2235 .unwrap(); 2236 2237 let r = std::panic::catch_unwind(|| { 2238 guest.wait_vm_boot(None).unwrap(); 2239 2240 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2241 2242 guest.enable_memory_hotplug(); 2243 2244 resize_zone_command(&api_socket, "mem0", "3G"); 2245 thread::sleep(std::time::Duration::new(5, 0)); 2246 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2247 resize_zone_command(&api_socket, "mem2", "3G"); 2248 thread::sleep(std::time::Duration::new(5, 0)); 2249 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2250 resize_zone_command(&api_socket, "mem0", "2G"); 2251 thread::sleep(std::time::Duration::new(5, 0)); 2252 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2253 resize_zone_command(&api_socket, "mem2", "2G"); 2254 thread::sleep(std::time::Duration::new(5, 0)); 2255 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2256 2257 guest.reboot_linux(0, None); 2258 2259 // Check the amount of RAM after reboot 2260 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2261 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2262 2263 // Check if we can still resize down to the initial 'boot'size 2264 resize_zone_command(&api_socket, "mem0", "1G"); 2265 thread::sleep(std::time::Duration::new(5, 0)); 2266 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2267 resize_zone_command(&api_socket, "mem2", "1G"); 2268 thread::sleep(std::time::Duration::new(5, 0)); 2269 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2270 }); 2271 2272 let _ = child.kill(); 2273 let output = child.wait_with_output().unwrap(); 2274 2275 handle_child_output(r, &output); 2276 } 2277 2278 #[test] 2279 #[cfg(not(feature = "mshv"))] 2280 fn test_guest_numa_nodes() { 2281 _test_guest_numa_nodes(false); 2282 } 2283 2284 #[test] 2285 #[cfg(target_arch = "x86_64")] 2286 fn test_iommu_segments() { 2287 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2288 let focal = UbuntuDiskConfig::new(focal_image); 2289 let guest = Guest::new(Box::new(focal)); 2290 2291 // Prepare another disk file for the virtio-disk device 2292 let test_disk_path = String::from( 2293 guest 2294 .tmp_dir 2295 .as_path() 2296 .join("test-disk.raw") 2297 .to_str() 2298 .unwrap(), 2299 ); 2300 assert!( 2301 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2302 ); 2303 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2304 2305 let api_socket = temp_api_path(&guest.tmp_dir); 2306 let mut cmd = GuestCommand::new(&guest); 2307 2308 cmd.args(["--cpus", "boot=1"]) 2309 .args(["--api-socket", &api_socket]) 2310 .args(["--memory", "size=512M"]) 2311 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2312 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2313 .args(["--platform", "num_pci_segments=16,iommu_segments=[1]"]) 2314 .default_disks() 2315 .capture_output() 2316 .default_net(); 2317 2318 let mut child = cmd.spawn().unwrap(); 2319 2320 guest.wait_vm_boot(None).unwrap(); 2321 2322 let r = std::panic::catch_unwind(|| { 2323 let (cmd_success, cmd_output) = remote_command_w_output( 2324 &api_socket, 2325 "add-disk", 2326 Some( 2327 format!( 2328 "path={},id=test0,pci_segment=1,iommu=on", 2329 test_disk_path.as_str() 2330 ) 2331 .as_str(), 2332 ), 2333 ); 2334 assert!(cmd_success); 2335 assert!(String::from_utf8_lossy(&cmd_output) 2336 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2337 2338 // Check IOMMU setup 2339 assert!(guest 2340 .does_device_vendor_pair_match("0x1057", "0x1af4") 2341 .unwrap_or_default()); 2342 assert_eq!( 2343 guest 2344 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2345 .unwrap() 2346 .trim(), 2347 "0001:00:01.0" 2348 ); 2349 }); 2350 2351 let _ = child.kill(); 2352 let output = child.wait_with_output().unwrap(); 2353 2354 handle_child_output(r, &output); 2355 } 2356 2357 #[test] 2358 fn test_pci_msi() { 2359 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2360 let guest = Guest::new(Box::new(focal)); 2361 let mut cmd = GuestCommand::new(&guest); 2362 cmd.args(["--cpus", "boot=1"]) 2363 .args(["--memory", "size=512M"]) 2364 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2365 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2366 .capture_output() 2367 .default_disks() 2368 .default_net(); 2369 2370 let mut child = cmd.spawn().unwrap(); 2371 2372 guest.wait_vm_boot(None).unwrap(); 2373 2374 #[cfg(target_arch = "x86_64")] 2375 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2376 #[cfg(target_arch = "aarch64")] 2377 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2378 2379 let r = std::panic::catch_unwind(|| { 2380 assert_eq!( 2381 guest 2382 .ssh_command(grep_cmd) 2383 .unwrap() 2384 .trim() 2385 .parse::<u32>() 2386 .unwrap_or_default(), 2387 12 2388 ); 2389 }); 2390 2391 let _ = child.kill(); 2392 let output = child.wait_with_output().unwrap(); 2393 2394 handle_child_output(r, &output); 2395 } 2396 2397 #[test] 2398 fn test_virtio_net_ctrl_queue() { 2399 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2400 let guest = Guest::new(Box::new(focal)); 2401 let mut cmd = GuestCommand::new(&guest); 2402 cmd.args(["--cpus", "boot=1"]) 2403 .args(["--memory", "size=512M"]) 2404 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2405 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2406 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2407 .capture_output() 2408 .default_disks(); 2409 2410 let mut child = cmd.spawn().unwrap(); 2411 2412 guest.wait_vm_boot(None).unwrap(); 2413 2414 #[cfg(target_arch = "aarch64")] 2415 let iface = "enp0s4"; 2416 #[cfg(target_arch = "x86_64")] 2417 let iface = "ens4"; 2418 2419 let r = std::panic::catch_unwind(|| { 2420 assert_eq!( 2421 guest 2422 .ssh_command( 2423 format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() 2424 ) 2425 .unwrap() 2426 .trim(), 2427 "success" 2428 ); 2429 assert_eq!( 2430 guest 2431 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 2432 .unwrap() 2433 .trim(), 2434 "3000" 2435 ); 2436 }); 2437 2438 let _ = child.kill(); 2439 let output = child.wait_with_output().unwrap(); 2440 2441 handle_child_output(r, &output); 2442 } 2443 2444 #[test] 2445 #[cfg(not(feature = "mshv"))] 2446 fn test_pci_multiple_segments() { 2447 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2448 let guest = Guest::new(Box::new(focal)); 2449 2450 // Prepare another disk file for the virtio-disk device 2451 let test_disk_path = String::from( 2452 guest 2453 .tmp_dir 2454 .as_path() 2455 .join("test-disk.raw") 2456 .to_str() 2457 .unwrap(), 2458 ); 2459 assert!( 2460 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2461 ); 2462 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2463 2464 let mut cmd = GuestCommand::new(&guest); 2465 cmd.args(["--cpus", "boot=1"]) 2466 .args(["--memory", "size=512M"]) 2467 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2468 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2469 .args(["--platform", "num_pci_segments=16"]) 2470 .args([ 2471 "--disk", 2472 format!( 2473 "path={}", 2474 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2475 ) 2476 .as_str(), 2477 format!( 2478 "path={}", 2479 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2480 ) 2481 .as_str(), 2482 format!("path={test_disk_path},pci_segment=15").as_str(), 2483 ]) 2484 .capture_output() 2485 .default_net(); 2486 2487 let mut child = cmd.spawn().unwrap(); 2488 2489 guest.wait_vm_boot(None).unwrap(); 2490 2491 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2492 2493 let r = std::panic::catch_unwind(|| { 2494 // There should be 16 PCI host bridges in the guest. 2495 assert_eq!( 2496 guest 2497 .ssh_command(grep_cmd) 2498 .unwrap() 2499 .trim() 2500 .parse::<u32>() 2501 .unwrap_or_default(), 2502 16 2503 ); 2504 2505 // Check both if /dev/vdc exists and if the block size is 4M. 2506 assert_eq!( 2507 guest 2508 .ssh_command("lsblk | grep vdc | grep -c 4M") 2509 .unwrap() 2510 .trim() 2511 .parse::<u32>() 2512 .unwrap_or_default(), 2513 1 2514 ); 2515 2516 // Mount the device. 2517 guest.ssh_command("mkdir mount_image").unwrap(); 2518 guest 2519 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 2520 .unwrap(); 2521 // Grant all users with write permission. 2522 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 2523 2524 // Write something to the device. 2525 guest 2526 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 2527 .unwrap(); 2528 2529 // Check the content of the block device. The file "foo" should 2530 // contain "bar". 2531 assert_eq!( 2532 guest 2533 .ssh_command("sudo cat mount_image/foo") 2534 .unwrap() 2535 .trim(), 2536 "bar" 2537 ); 2538 }); 2539 2540 let _ = child.kill(); 2541 let output = child.wait_with_output().unwrap(); 2542 2543 handle_child_output(r, &output); 2544 } 2545 2546 #[test] 2547 fn test_direct_kernel_boot() { 2548 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2549 let guest = Guest::new(Box::new(focal)); 2550 2551 let kernel_path = direct_kernel_boot_path(); 2552 2553 let mut child = GuestCommand::new(&guest) 2554 .args(["--cpus", "boot=1"]) 2555 .args(["--memory", "size=512M"]) 2556 .args(["--kernel", kernel_path.to_str().unwrap()]) 2557 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2558 .default_disks() 2559 .default_net() 2560 .capture_output() 2561 .spawn() 2562 .unwrap(); 2563 2564 let r = std::panic::catch_unwind(|| { 2565 guest.wait_vm_boot(None).unwrap(); 2566 2567 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2568 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2569 2570 let grep_cmd = if cfg!(target_arch = "x86_64") { 2571 "grep -c PCI-MSI /proc/interrupts" 2572 } else { 2573 "grep -c ITS-MSI /proc/interrupts" 2574 }; 2575 assert_eq!( 2576 guest 2577 .ssh_command(grep_cmd) 2578 .unwrap() 2579 .trim() 2580 .parse::<u32>() 2581 .unwrap_or_default(), 2582 12 2583 ); 2584 }); 2585 2586 let _ = child.kill(); 2587 let output = child.wait_with_output().unwrap(); 2588 2589 handle_child_output(r, &output); 2590 } 2591 2592 fn _test_virtio_block(image_name: &str, disable_io_uring: bool) { 2593 let focal = UbuntuDiskConfig::new(image_name.to_string()); 2594 let guest = Guest::new(Box::new(focal)); 2595 2596 let mut workload_path = dirs::home_dir().unwrap(); 2597 workload_path.push("workloads"); 2598 2599 let mut blk_file_path = workload_path; 2600 blk_file_path.push("blk.img"); 2601 2602 let kernel_path = direct_kernel_boot_path(); 2603 2604 let mut cloud_child = GuestCommand::new(&guest) 2605 .args(["--cpus", "boot=4"]) 2606 .args(["--memory", "size=512M,shared=on"]) 2607 .args(["--kernel", kernel_path.to_str().unwrap()]) 2608 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2609 .args([ 2610 "--disk", 2611 format!( 2612 "path={}", 2613 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2614 ) 2615 .as_str(), 2616 format!( 2617 "path={}", 2618 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2619 ) 2620 .as_str(), 2621 format!( 2622 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={}", 2623 blk_file_path.to_str().unwrap(), 2624 disable_io_uring 2625 ) 2626 .as_str(), 2627 ]) 2628 .default_net() 2629 .capture_output() 2630 .spawn() 2631 .unwrap(); 2632 2633 let r = std::panic::catch_unwind(|| { 2634 guest.wait_vm_boot(None).unwrap(); 2635 2636 // Check both if /dev/vdc exists and if the block size is 16M. 2637 assert_eq!( 2638 guest 2639 .ssh_command("lsblk | grep vdc | grep -c 16M") 2640 .unwrap() 2641 .trim() 2642 .parse::<u32>() 2643 .unwrap_or_default(), 2644 1 2645 ); 2646 2647 // Check both if /dev/vdc exists and if this block is RO. 2648 assert_eq!( 2649 guest 2650 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 2651 .unwrap() 2652 .trim() 2653 .parse::<u32>() 2654 .unwrap_or_default(), 2655 1 2656 ); 2657 2658 // Check if the number of queues is 4. 2659 assert_eq!( 2660 guest 2661 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 2662 .unwrap() 2663 .trim() 2664 .parse::<u32>() 2665 .unwrap_or_default(), 2666 4 2667 ); 2668 }); 2669 2670 let _ = cloud_child.kill(); 2671 let output = cloud_child.wait_with_output().unwrap(); 2672 2673 handle_child_output(r, &output); 2674 } 2675 2676 #[test] 2677 fn test_virtio_block() { 2678 _test_virtio_block(FOCAL_IMAGE_NAME, false) 2679 } 2680 2681 #[test] 2682 fn test_virtio_block_disable_io_uring() { 2683 _test_virtio_block(FOCAL_IMAGE_NAME, true) 2684 } 2685 2686 #[test] 2687 fn test_virtio_block_qcow2() { 2688 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false) 2689 } 2690 2691 #[test] 2692 fn test_virtio_block_vhd() { 2693 let mut workload_path = dirs::home_dir().unwrap(); 2694 workload_path.push("workloads"); 2695 2696 let mut raw_file_path = workload_path.clone(); 2697 let mut vhd_file_path = workload_path; 2698 raw_file_path.push(FOCAL_IMAGE_NAME); 2699 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 2700 2701 // Generate VHD file from RAW file 2702 std::process::Command::new("qemu-img") 2703 .arg("convert") 2704 .arg("-p") 2705 .args(["-f", "raw"]) 2706 .args(["-O", "vpc"]) 2707 .args(["-o", "subformat=fixed"]) 2708 .arg(raw_file_path.to_str().unwrap()) 2709 .arg(vhd_file_path.to_str().unwrap()) 2710 .output() 2711 .expect("Expect generating VHD image from RAW image"); 2712 2713 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false) 2714 } 2715 2716 #[test] 2717 fn test_virtio_block_vhdx() { 2718 let mut workload_path = dirs::home_dir().unwrap(); 2719 workload_path.push("workloads"); 2720 2721 let mut raw_file_path = workload_path.clone(); 2722 let mut vhdx_file_path = workload_path; 2723 raw_file_path.push(FOCAL_IMAGE_NAME); 2724 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 2725 2726 // Generate dynamic VHDX file from RAW file 2727 std::process::Command::new("qemu-img") 2728 .arg("convert") 2729 .arg("-p") 2730 .args(["-f", "raw"]) 2731 .args(["-O", "vhdx"]) 2732 .arg(raw_file_path.to_str().unwrap()) 2733 .arg(vhdx_file_path.to_str().unwrap()) 2734 .output() 2735 .expect("Expect generating dynamic VHDx image from RAW image"); 2736 2737 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false) 2738 } 2739 2740 #[test] 2741 fn test_virtio_block_dynamic_vhdx_expand() { 2742 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 2743 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 2744 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 2745 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 2746 2747 let mut workload_path = dirs::home_dir().unwrap(); 2748 workload_path.push("workloads"); 2749 2750 let mut vhdx_file_path = workload_path; 2751 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 2752 let vhdx_path = vhdx_file_path.to_str().unwrap(); 2753 2754 // Generate a 100 MiB dynamic VHDX file 2755 std::process::Command::new("qemu-img") 2756 .arg("create") 2757 .args(["-f", "vhdx"]) 2758 .arg(vhdx_path) 2759 .arg(VIRTUAL_DISK_SIZE.to_string()) 2760 .output() 2761 .expect("Expect generating dynamic VHDx image from RAW image"); 2762 2763 // Check if the size matches with empty VHDx file size 2764 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 2765 2766 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2767 let guest = Guest::new(Box::new(focal)); 2768 let kernel_path = direct_kernel_boot_path(); 2769 2770 let mut cloud_child = GuestCommand::new(&guest) 2771 .args(["--cpus", "boot=1"]) 2772 .args(["--memory", "size=512M"]) 2773 .args(["--kernel", kernel_path.to_str().unwrap()]) 2774 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2775 .args([ 2776 "--disk", 2777 format!( 2778 "path={}", 2779 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2780 ) 2781 .as_str(), 2782 format!( 2783 "path={}", 2784 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2785 ) 2786 .as_str(), 2787 format!("path={vhdx_path}").as_str(), 2788 ]) 2789 .default_net() 2790 .capture_output() 2791 .spawn() 2792 .unwrap(); 2793 2794 let r = std::panic::catch_unwind(|| { 2795 guest.wait_vm_boot(None).unwrap(); 2796 2797 // Check both if /dev/vdc exists and if the block size is 100 MiB. 2798 assert_eq!( 2799 guest 2800 .ssh_command("lsblk | grep vdc | grep -c 100M") 2801 .unwrap() 2802 .trim() 2803 .parse::<u32>() 2804 .unwrap_or_default(), 2805 1 2806 ); 2807 2808 // Write 100 MB of data to the VHDx disk 2809 guest 2810 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 2811 .unwrap(); 2812 }); 2813 2814 // Check if the size matches with expected expanded VHDx file size 2815 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 2816 2817 let _ = cloud_child.kill(); 2818 let output = cloud_child.wait_with_output().unwrap(); 2819 2820 handle_child_output(r, &output); 2821 } 2822 2823 fn vhdx_image_size(disk_name: &str) -> u64 { 2824 std::fs::File::open(disk_name) 2825 .unwrap() 2826 .seek(SeekFrom::End(0)) 2827 .unwrap() 2828 } 2829 2830 #[test] 2831 fn test_virtio_block_direct_and_firmware() { 2832 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2833 let guest = Guest::new(Box::new(focal)); 2834 2835 // The OS disk must be copied to a location that is not backed by 2836 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 2837 // with EINVAL because tmpfs doesn't support this flag. 2838 let mut workloads_path = dirs::home_dir().unwrap(); 2839 workloads_path.push("workloads"); 2840 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 2841 let mut os_path = os_dir.as_path().to_path_buf(); 2842 os_path.push("osdisk.img"); 2843 rate_limited_copy( 2844 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 2845 os_path.as_path(), 2846 ) 2847 .expect("copying of OS disk failed"); 2848 2849 let mut child = GuestCommand::new(&guest) 2850 .args(["--cpus", "boot=1"]) 2851 .args(["--memory", "size=512M"]) 2852 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 2853 .args([ 2854 "--disk", 2855 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 2856 format!( 2857 "path={}", 2858 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2859 ) 2860 .as_str(), 2861 ]) 2862 .default_net() 2863 .capture_output() 2864 .spawn() 2865 .unwrap(); 2866 2867 let r = std::panic::catch_unwind(|| { 2868 guest.wait_vm_boot(Some(120)).unwrap(); 2869 }); 2870 2871 let _ = child.kill(); 2872 let output = child.wait_with_output().unwrap(); 2873 2874 handle_child_output(r, &output); 2875 } 2876 2877 #[test] 2878 fn test_vhost_user_net_default() { 2879 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 2880 } 2881 2882 #[test] 2883 fn test_vhost_user_net_named_tap() { 2884 test_vhost_user_net( 2885 Some("mytap0"), 2886 2, 2887 &prepare_vhost_user_net_daemon, 2888 false, 2889 false, 2890 ) 2891 } 2892 2893 #[test] 2894 fn test_vhost_user_net_existing_tap() { 2895 test_vhost_user_net( 2896 Some("vunet-tap0"), 2897 2, 2898 &prepare_vhost_user_net_daemon, 2899 false, 2900 false, 2901 ) 2902 } 2903 2904 #[test] 2905 fn test_vhost_user_net_multiple_queues() { 2906 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 2907 } 2908 2909 #[test] 2910 fn test_vhost_user_net_tap_multiple_queues() { 2911 test_vhost_user_net( 2912 Some("vunet-tap1"), 2913 4, 2914 &prepare_vhost_user_net_daemon, 2915 false, 2916 false, 2917 ) 2918 } 2919 2920 #[test] 2921 fn test_vhost_user_net_host_mac() { 2922 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 2923 } 2924 2925 #[test] 2926 fn test_vhost_user_net_client_mode() { 2927 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 2928 } 2929 2930 #[test] 2931 fn test_vhost_user_blk_default() { 2932 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 2933 } 2934 2935 #[test] 2936 fn test_vhost_user_blk_readonly() { 2937 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 2938 } 2939 2940 #[test] 2941 fn test_vhost_user_blk_direct() { 2942 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 2943 } 2944 2945 #[test] 2946 fn test_boot_from_vhost_user_blk_default() { 2947 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 2948 } 2949 2950 #[test] 2951 #[cfg(target_arch = "x86_64")] 2952 fn test_split_irqchip() { 2953 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2954 let guest = Guest::new(Box::new(focal)); 2955 2956 let mut child = GuestCommand::new(&guest) 2957 .args(["--cpus", "boot=1"]) 2958 .args(["--memory", "size=512M"]) 2959 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2960 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2961 .default_disks() 2962 .default_net() 2963 .capture_output() 2964 .spawn() 2965 .unwrap(); 2966 2967 let r = std::panic::catch_unwind(|| { 2968 guest.wait_vm_boot(None).unwrap(); 2969 2970 assert_eq!( 2971 guest 2972 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 2973 .unwrap() 2974 .trim() 2975 .parse::<u32>() 2976 .unwrap_or(1), 2977 0 2978 ); 2979 assert_eq!( 2980 guest 2981 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 2982 .unwrap() 2983 .trim() 2984 .parse::<u32>() 2985 .unwrap_or(1), 2986 0 2987 ); 2988 }); 2989 2990 let _ = child.kill(); 2991 let output = child.wait_with_output().unwrap(); 2992 2993 handle_child_output(r, &output); 2994 } 2995 2996 #[test] 2997 #[cfg(target_arch = "x86_64")] 2998 fn test_dmi_serial_number() { 2999 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3000 let guest = Guest::new(Box::new(focal)); 3001 3002 let mut child = GuestCommand::new(&guest) 3003 .args(["--cpus", "boot=1"]) 3004 .args(["--memory", "size=512M"]) 3005 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3006 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3007 .args(["--platform", "serial_number=a=b;c=d"]) 3008 .default_disks() 3009 .default_net() 3010 .capture_output() 3011 .spawn() 3012 .unwrap(); 3013 3014 let r = std::panic::catch_unwind(|| { 3015 guest.wait_vm_boot(None).unwrap(); 3016 3017 assert_eq!( 3018 guest 3019 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3020 .unwrap() 3021 .trim(), 3022 "a=b;c=d" 3023 ); 3024 }); 3025 3026 let _ = child.kill(); 3027 let output = child.wait_with_output().unwrap(); 3028 3029 handle_child_output(r, &output); 3030 } 3031 3032 #[test] 3033 #[cfg(target_arch = "x86_64")] 3034 fn test_dmi_uuid() { 3035 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3036 let guest = Guest::new(Box::new(focal)); 3037 3038 let mut child = GuestCommand::new(&guest) 3039 .args(["--cpus", "boot=1"]) 3040 .args(["--memory", "size=512M"]) 3041 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3042 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3043 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3044 .default_disks() 3045 .default_net() 3046 .capture_output() 3047 .spawn() 3048 .unwrap(); 3049 3050 let r = std::panic::catch_unwind(|| { 3051 guest.wait_vm_boot(None).unwrap(); 3052 3053 assert_eq!( 3054 guest 3055 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3056 .unwrap() 3057 .trim(), 3058 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3059 ); 3060 }); 3061 3062 let _ = child.kill(); 3063 let output = child.wait_with_output().unwrap(); 3064 3065 handle_child_output(r, &output); 3066 } 3067 3068 #[test] 3069 #[cfg(target_arch = "x86_64")] 3070 fn test_dmi_oem_strings() { 3071 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3072 let guest = Guest::new(Box::new(focal)); 3073 3074 let s1 = "io.systemd.credential:xx=yy"; 3075 let s2 = "This is a test string"; 3076 3077 let oem_strings = format!("oem_strings=[{s1},{s2}]"); 3078 3079 let mut child = GuestCommand::new(&guest) 3080 .args(["--cpus", "boot=1"]) 3081 .args(["--memory", "size=512M"]) 3082 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3083 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3084 .args(["--platform", &oem_strings]) 3085 .default_disks() 3086 .default_net() 3087 .capture_output() 3088 .spawn() 3089 .unwrap(); 3090 3091 let r = std::panic::catch_unwind(|| { 3092 guest.wait_vm_boot(None).unwrap(); 3093 3094 assert_eq!( 3095 guest 3096 .ssh_command("sudo dmidecode --oem-string count") 3097 .unwrap() 3098 .trim(), 3099 "2" 3100 ); 3101 3102 assert_eq!( 3103 guest 3104 .ssh_command("sudo dmidecode --oem-string 1") 3105 .unwrap() 3106 .trim(), 3107 s1 3108 ); 3109 3110 assert_eq!( 3111 guest 3112 .ssh_command("sudo dmidecode --oem-string 2") 3113 .unwrap() 3114 .trim(), 3115 s2 3116 ); 3117 }); 3118 3119 let _ = child.kill(); 3120 let output = child.wait_with_output().unwrap(); 3121 3122 handle_child_output(r, &output); 3123 } 3124 3125 #[test] 3126 fn test_virtio_fs() { 3127 _test_virtio_fs(&prepare_virtiofsd, false, None) 3128 } 3129 3130 #[test] 3131 fn test_virtio_fs_hotplug() { 3132 _test_virtio_fs(&prepare_virtiofsd, true, None) 3133 } 3134 3135 #[test] 3136 #[cfg(not(feature = "mshv"))] 3137 fn test_virtio_fs_multi_segment_hotplug() { 3138 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3139 } 3140 3141 #[test] 3142 #[cfg(not(feature = "mshv"))] 3143 fn test_virtio_fs_multi_segment() { 3144 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3145 } 3146 3147 #[test] 3148 fn test_virtio_pmem_persist_writes() { 3149 test_virtio_pmem(false, false) 3150 } 3151 3152 #[test] 3153 fn test_virtio_pmem_discard_writes() { 3154 test_virtio_pmem(true, false) 3155 } 3156 3157 #[test] 3158 fn test_virtio_pmem_with_size() { 3159 test_virtio_pmem(true, true) 3160 } 3161 3162 #[test] 3163 fn test_boot_from_virtio_pmem() { 3164 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3165 let guest = Guest::new(Box::new(focal)); 3166 3167 let kernel_path = direct_kernel_boot_path(); 3168 3169 let mut child = GuestCommand::new(&guest) 3170 .args(["--cpus", "boot=1"]) 3171 .args(["--memory", "size=512M"]) 3172 .args(["--kernel", kernel_path.to_str().unwrap()]) 3173 .args([ 3174 "--disk", 3175 format!( 3176 "path={}", 3177 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3178 ) 3179 .as_str(), 3180 ]) 3181 .default_net() 3182 .args([ 3183 "--pmem", 3184 format!( 3185 "file={},size={}", 3186 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3187 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3188 .unwrap() 3189 .len() 3190 ) 3191 .as_str(), 3192 ]) 3193 .args([ 3194 "--cmdline", 3195 DIRECT_KERNEL_BOOT_CMDLINE 3196 .replace("vda1", "pmem0p1") 3197 .as_str(), 3198 ]) 3199 .capture_output() 3200 .spawn() 3201 .unwrap(); 3202 3203 let r = std::panic::catch_unwind(|| { 3204 guest.wait_vm_boot(None).unwrap(); 3205 3206 // Simple checks to validate the VM booted properly 3207 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3208 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3209 }); 3210 3211 let _ = child.kill(); 3212 let output = child.wait_with_output().unwrap(); 3213 3214 handle_child_output(r, &output); 3215 } 3216 3217 #[test] 3218 fn test_multiple_network_interfaces() { 3219 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3220 let guest = Guest::new(Box::new(focal)); 3221 3222 let kernel_path = direct_kernel_boot_path(); 3223 3224 let mut child = GuestCommand::new(&guest) 3225 .args(["--cpus", "boot=1"]) 3226 .args(["--memory", "size=512M"]) 3227 .args(["--kernel", kernel_path.to_str().unwrap()]) 3228 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3229 .default_disks() 3230 .args([ 3231 "--net", 3232 guest.default_net_string().as_str(), 3233 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3234 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3235 ]) 3236 .capture_output() 3237 .spawn() 3238 .unwrap(); 3239 3240 let r = std::panic::catch_unwind(|| { 3241 guest.wait_vm_boot(None).unwrap(); 3242 3243 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3244 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3245 3246 // 3 network interfaces + default localhost ==> 4 interfaces 3247 assert_eq!( 3248 guest 3249 .ssh_command("ip -o link | wc -l") 3250 .unwrap() 3251 .trim() 3252 .parse::<u32>() 3253 .unwrap_or_default(), 3254 4 3255 ); 3256 }); 3257 3258 let _ = child.kill(); 3259 let output = child.wait_with_output().unwrap(); 3260 3261 handle_child_output(r, &output); 3262 } 3263 3264 #[test] 3265 #[cfg(target_arch = "aarch64")] 3266 fn test_pmu_on() { 3267 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3268 let guest = Guest::new(Box::new(focal)); 3269 let mut child = GuestCommand::new(&guest) 3270 .args(["--cpus", "boot=1"]) 3271 .args(["--memory", "size=512M"]) 3272 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3273 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3274 .default_disks() 3275 .default_net() 3276 .capture_output() 3277 .spawn() 3278 .unwrap(); 3279 3280 let r = std::panic::catch_unwind(|| { 3281 guest.wait_vm_boot(None).unwrap(); 3282 3283 // Test that PMU exists. 3284 assert_eq!( 3285 guest 3286 .ssh_command(GREP_PMU_IRQ_CMD) 3287 .unwrap() 3288 .trim() 3289 .parse::<u32>() 3290 .unwrap_or_default(), 3291 1 3292 ); 3293 }); 3294 3295 let _ = child.kill(); 3296 let output = child.wait_with_output().unwrap(); 3297 3298 handle_child_output(r, &output); 3299 } 3300 3301 #[test] 3302 fn test_serial_off() { 3303 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3304 let guest = Guest::new(Box::new(focal)); 3305 let mut child = GuestCommand::new(&guest) 3306 .args(["--cpus", "boot=1"]) 3307 .args(["--memory", "size=512M"]) 3308 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3309 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3310 .default_disks() 3311 .default_net() 3312 .args(["--serial", "off"]) 3313 .capture_output() 3314 .spawn() 3315 .unwrap(); 3316 3317 let r = std::panic::catch_unwind(|| { 3318 guest.wait_vm_boot(None).unwrap(); 3319 3320 // Test that there is no ttyS0 3321 assert_eq!( 3322 guest 3323 .ssh_command(GREP_SERIAL_IRQ_CMD) 3324 .unwrap() 3325 .trim() 3326 .parse::<u32>() 3327 .unwrap_or(1), 3328 0 3329 ); 3330 }); 3331 3332 let _ = child.kill(); 3333 let output = child.wait_with_output().unwrap(); 3334 3335 handle_child_output(r, &output); 3336 } 3337 3338 #[test] 3339 fn test_serial_null() { 3340 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3341 let guest = Guest::new(Box::new(focal)); 3342 let mut cmd = GuestCommand::new(&guest); 3343 #[cfg(target_arch = "x86_64")] 3344 let console_str: &str = "console=ttyS0"; 3345 #[cfg(target_arch = "aarch64")] 3346 let console_str: &str = "console=ttyAMA0"; 3347 3348 cmd.args(["--cpus", "boot=1"]) 3349 .args(["--memory", "size=512M"]) 3350 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3351 .args([ 3352 "--cmdline", 3353 DIRECT_KERNEL_BOOT_CMDLINE 3354 .replace("console=hvc0 ", console_str) 3355 .as_str(), 3356 ]) 3357 .default_disks() 3358 .default_net() 3359 .args(["--serial", "null"]) 3360 .args(["--console", "off"]) 3361 .capture_output(); 3362 3363 let mut child = cmd.spawn().unwrap(); 3364 3365 let r = std::panic::catch_unwind(|| { 3366 guest.wait_vm_boot(None).unwrap(); 3367 3368 // Test that there is a ttyS0 3369 assert_eq!( 3370 guest 3371 .ssh_command(GREP_SERIAL_IRQ_CMD) 3372 .unwrap() 3373 .trim() 3374 .parse::<u32>() 3375 .unwrap_or_default(), 3376 1 3377 ); 3378 }); 3379 3380 let _ = child.kill(); 3381 let output = child.wait_with_output().unwrap(); 3382 handle_child_output(r, &output); 3383 3384 let r = std::panic::catch_unwind(|| { 3385 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3386 }); 3387 3388 handle_child_output(r, &output); 3389 } 3390 3391 #[test] 3392 fn test_serial_tty() { 3393 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3394 let guest = Guest::new(Box::new(focal)); 3395 3396 let kernel_path = direct_kernel_boot_path(); 3397 3398 #[cfg(target_arch = "x86_64")] 3399 let console_str: &str = "console=ttyS0"; 3400 #[cfg(target_arch = "aarch64")] 3401 let console_str: &str = "console=ttyAMA0"; 3402 3403 let mut child = GuestCommand::new(&guest) 3404 .args(["--cpus", "boot=1"]) 3405 .args(["--memory", "size=512M"]) 3406 .args(["--kernel", kernel_path.to_str().unwrap()]) 3407 .args([ 3408 "--cmdline", 3409 DIRECT_KERNEL_BOOT_CMDLINE 3410 .replace("console=hvc0 ", console_str) 3411 .as_str(), 3412 ]) 3413 .default_disks() 3414 .default_net() 3415 .args(["--serial", "tty"]) 3416 .args(["--console", "off"]) 3417 .capture_output() 3418 .spawn() 3419 .unwrap(); 3420 3421 let r = std::panic::catch_unwind(|| { 3422 guest.wait_vm_boot(None).unwrap(); 3423 3424 // Test that there is a ttyS0 3425 assert_eq!( 3426 guest 3427 .ssh_command(GREP_SERIAL_IRQ_CMD) 3428 .unwrap() 3429 .trim() 3430 .parse::<u32>() 3431 .unwrap_or_default(), 3432 1 3433 ); 3434 }); 3435 3436 // This sleep is needed to wait for the login prompt 3437 thread::sleep(std::time::Duration::new(2, 0)); 3438 3439 let _ = child.kill(); 3440 let output = child.wait_with_output().unwrap(); 3441 handle_child_output(r, &output); 3442 3443 let r = std::panic::catch_unwind(|| { 3444 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3445 }); 3446 3447 handle_child_output(r, &output); 3448 } 3449 3450 #[test] 3451 fn test_serial_file() { 3452 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3453 let guest = Guest::new(Box::new(focal)); 3454 3455 let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output"); 3456 #[cfg(target_arch = "x86_64")] 3457 let console_str: &str = "console=ttyS0"; 3458 #[cfg(target_arch = "aarch64")] 3459 let console_str: &str = "console=ttyAMA0"; 3460 3461 let mut child = GuestCommand::new(&guest) 3462 .args(["--cpus", "boot=1"]) 3463 .args(["--memory", "size=512M"]) 3464 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3465 .args([ 3466 "--cmdline", 3467 DIRECT_KERNEL_BOOT_CMDLINE 3468 .replace("console=hvc0 ", console_str) 3469 .as_str(), 3470 ]) 3471 .default_disks() 3472 .default_net() 3473 .args([ 3474 "--serial", 3475 format!("file={}", serial_path.to_str().unwrap()).as_str(), 3476 ]) 3477 .capture_output() 3478 .spawn() 3479 .unwrap(); 3480 3481 let r = std::panic::catch_unwind(|| { 3482 guest.wait_vm_boot(None).unwrap(); 3483 3484 // Test that there is a ttyS0 3485 assert_eq!( 3486 guest 3487 .ssh_command(GREP_SERIAL_IRQ_CMD) 3488 .unwrap() 3489 .trim() 3490 .parse::<u32>() 3491 .unwrap_or_default(), 3492 1 3493 ); 3494 3495 guest.ssh_command("sudo shutdown -h now").unwrap(); 3496 }); 3497 3498 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3499 let _ = child.kill(); 3500 let output = child.wait_with_output().unwrap(); 3501 handle_child_output(r, &output); 3502 3503 let r = std::panic::catch_unwind(|| { 3504 // Check that the cloud-hypervisor binary actually terminated 3505 assert!(output.status.success()); 3506 3507 // Do this check after shutdown of the VM as an easy way to ensure 3508 // all writes are flushed to disk 3509 let mut f = std::fs::File::open(serial_path).unwrap(); 3510 let mut buf = String::new(); 3511 f.read_to_string(&mut buf).unwrap(); 3512 assert!(buf.contains(CONSOLE_TEST_STRING)); 3513 }); 3514 3515 handle_child_output(r, &output); 3516 } 3517 3518 #[test] 3519 fn test_pty_interaction() { 3520 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3521 let guest = Guest::new(Box::new(focal)); 3522 let api_socket = temp_api_path(&guest.tmp_dir); 3523 let serial_option = if cfg!(target_arch = "x86_64") { 3524 " console=ttyS0" 3525 } else { 3526 " console=ttyAMA0" 3527 }; 3528 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 3529 3530 let mut child = GuestCommand::new(&guest) 3531 .args(["--cpus", "boot=1"]) 3532 .args(["--memory", "size=512M"]) 3533 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3534 .args(["--cmdline", &cmdline]) 3535 .default_disks() 3536 .default_net() 3537 .args(["--serial", "null"]) 3538 .args(["--console", "pty"]) 3539 .args(["--api-socket", &api_socket]) 3540 .spawn() 3541 .unwrap(); 3542 3543 let r = std::panic::catch_unwind(|| { 3544 guest.wait_vm_boot(None).unwrap(); 3545 // Get pty fd for console 3546 let console_path = get_pty_path(&api_socket, "console"); 3547 // TODO: Get serial pty test working 3548 let mut cf = std::fs::OpenOptions::new() 3549 .write(true) 3550 .read(true) 3551 .open(console_path) 3552 .unwrap(); 3553 3554 // Some dumb sleeps but we don't want to write 3555 // before the console is up and we don't want 3556 // to try and write the next line before the 3557 // login process is ready. 3558 thread::sleep(std::time::Duration::new(5, 0)); 3559 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 3560 thread::sleep(std::time::Duration::new(2, 0)); 3561 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 3562 thread::sleep(std::time::Duration::new(2, 0)); 3563 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 3564 thread::sleep(std::time::Duration::new(2, 0)); 3565 3566 // read pty and ensure they have a login shell 3567 // some fairly hacky workarounds to avoid looping 3568 // forever in case the channel is blocked getting output 3569 let ptyc = pty_read(cf); 3570 let mut empty = 0; 3571 let mut prev = String::new(); 3572 loop { 3573 thread::sleep(std::time::Duration::new(2, 0)); 3574 match ptyc.try_recv() { 3575 Ok(line) => { 3576 empty = 0; 3577 prev = prev + &line; 3578 if prev.contains("test_pty_console") { 3579 break; 3580 } 3581 } 3582 Err(mpsc::TryRecvError::Empty) => { 3583 empty += 1; 3584 assert!(empty <= 5, "No login on pty"); 3585 } 3586 _ => panic!("No login on pty"), 3587 } 3588 } 3589 3590 guest.ssh_command("sudo shutdown -h now").unwrap(); 3591 }); 3592 3593 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3594 let _ = child.kill(); 3595 let output = child.wait_with_output().unwrap(); 3596 handle_child_output(r, &output); 3597 3598 let r = std::panic::catch_unwind(|| { 3599 // Check that the cloud-hypervisor binary actually terminated 3600 assert!(output.status.success()) 3601 }); 3602 handle_child_output(r, &output); 3603 } 3604 3605 #[test] 3606 fn test_virtio_console() { 3607 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3608 let guest = Guest::new(Box::new(focal)); 3609 3610 let kernel_path = direct_kernel_boot_path(); 3611 3612 let mut child = GuestCommand::new(&guest) 3613 .args(["--cpus", "boot=1"]) 3614 .args(["--memory", "size=512M"]) 3615 .args(["--kernel", kernel_path.to_str().unwrap()]) 3616 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3617 .default_disks() 3618 .default_net() 3619 .args(["--console", "tty"]) 3620 .args(["--serial", "null"]) 3621 .capture_output() 3622 .spawn() 3623 .unwrap(); 3624 3625 let text = String::from("On a branch floating down river a cricket, singing."); 3626 let cmd = format!("echo {text} | sudo tee /dev/hvc0"); 3627 3628 let r = std::panic::catch_unwind(|| { 3629 guest.wait_vm_boot(None).unwrap(); 3630 3631 assert!(guest 3632 .does_device_vendor_pair_match("0x1043", "0x1af4") 3633 .unwrap_or_default()); 3634 3635 guest.ssh_command(&cmd).unwrap(); 3636 }); 3637 3638 let _ = child.kill(); 3639 let output = child.wait_with_output().unwrap(); 3640 handle_child_output(r, &output); 3641 3642 let r = std::panic::catch_unwind(|| { 3643 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 3644 }); 3645 3646 handle_child_output(r, &output); 3647 } 3648 3649 #[test] 3650 fn test_console_file() { 3651 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3652 let guest = Guest::new(Box::new(focal)); 3653 3654 let console_path = guest.tmp_dir.as_path().join("/tmp/console-output"); 3655 let mut child = GuestCommand::new(&guest) 3656 .args(["--cpus", "boot=1"]) 3657 .args(["--memory", "size=512M"]) 3658 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3659 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3660 .default_disks() 3661 .default_net() 3662 .args([ 3663 "--console", 3664 format!("file={}", console_path.to_str().unwrap()).as_str(), 3665 ]) 3666 .capture_output() 3667 .spawn() 3668 .unwrap(); 3669 3670 guest.wait_vm_boot(None).unwrap(); 3671 3672 guest.ssh_command("sudo shutdown -h now").unwrap(); 3673 3674 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3675 let _ = child.kill(); 3676 let output = child.wait_with_output().unwrap(); 3677 3678 let r = std::panic::catch_unwind(|| { 3679 // Check that the cloud-hypervisor binary actually terminated 3680 assert!(output.status.success()); 3681 3682 // Do this check after shutdown of the VM as an easy way to ensure 3683 // all writes are flushed to disk 3684 let mut f = std::fs::File::open(console_path).unwrap(); 3685 let mut buf = String::new(); 3686 f.read_to_string(&mut buf).unwrap(); 3687 3688 if !buf.contains(CONSOLE_TEST_STRING) { 3689 eprintln!( 3690 "\n\n==== Console file output ====\n\n{buf}\n\n==== End console file output ====" 3691 ); 3692 } 3693 assert!(buf.contains(CONSOLE_TEST_STRING)); 3694 }); 3695 3696 handle_child_output(r, &output); 3697 } 3698 3699 #[test] 3700 #[cfg(target_arch = "x86_64")] 3701 #[cfg(not(feature = "mshv"))] 3702 #[ignore = "See #4324"] 3703 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 3704 // backed networking interfaces, bound through a simple bridge on the host. 3705 // So if the nested cloud-hypervisor succeeds in getting a directly 3706 // assigned interface from its cloud-hypervisor host, we should be able to 3707 // ssh into it, and verify that it's running with the right kernel command 3708 // line (We tag the command line from cloud-hypervisor for that purpose). 3709 // The third device is added to validate that hotplug works correctly since 3710 // it is being added to the L2 VM through hotplugging mechanism. 3711 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 3712 // vfio device support 3713 fn test_vfio() { 3714 setup_vfio_network_interfaces(); 3715 3716 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3717 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 3718 3719 let mut workload_path = dirs::home_dir().unwrap(); 3720 workload_path.push("workloads"); 3721 3722 let kernel_path = direct_kernel_boot_path(); 3723 3724 let mut vfio_path = workload_path.clone(); 3725 vfio_path.push("vfio"); 3726 3727 let mut cloud_init_vfio_base_path = vfio_path.clone(); 3728 cloud_init_vfio_base_path.push("cloudinit.img"); 3729 3730 // We copy our cloudinit into the vfio mount point, for the nested 3731 // cloud-hypervisor guest to use. 3732 rate_limited_copy( 3733 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 3734 &cloud_init_vfio_base_path, 3735 ) 3736 .expect("copying of cloud-init disk failed"); 3737 3738 let mut vfio_disk_path = workload_path.clone(); 3739 vfio_disk_path.push("vfio.img"); 3740 3741 // Create the vfio disk image 3742 let output = Command::new("mkfs.ext4") 3743 .arg("-d") 3744 .arg(vfio_path.to_str().unwrap()) 3745 .arg(vfio_disk_path.to_str().unwrap()) 3746 .arg("2g") 3747 .output() 3748 .unwrap(); 3749 if !output.status.success() { 3750 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 3751 panic!("mkfs.ext4 command generated an error"); 3752 } 3753 3754 let mut blk_file_path = workload_path; 3755 blk_file_path.push("blk.img"); 3756 3757 let vfio_tap0 = "vfio-tap0"; 3758 let vfio_tap1 = "vfio-tap1"; 3759 let vfio_tap2 = "vfio-tap2"; 3760 let vfio_tap3 = "vfio-tap3"; 3761 3762 let mut child = GuestCommand::new(&guest) 3763 .args(["--cpus", "boot=4"]) 3764 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 3765 .args(["--kernel", kernel_path.to_str().unwrap()]) 3766 .args([ 3767 "--disk", 3768 format!( 3769 "path={}", 3770 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3771 ) 3772 .as_str(), 3773 format!( 3774 "path={}", 3775 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3776 ) 3777 .as_str(), 3778 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 3779 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 3780 ]) 3781 .args([ 3782 "--cmdline", 3783 format!( 3784 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 3785 ) 3786 .as_str(), 3787 ]) 3788 .args([ 3789 "--net", 3790 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 3791 format!( 3792 "tap={},mac={},iommu=on", 3793 vfio_tap1, guest.network.l2_guest_mac1 3794 ) 3795 .as_str(), 3796 format!( 3797 "tap={},mac={},iommu=on", 3798 vfio_tap2, guest.network.l2_guest_mac2 3799 ) 3800 .as_str(), 3801 format!( 3802 "tap={},mac={},iommu=on", 3803 vfio_tap3, guest.network.l2_guest_mac3 3804 ) 3805 .as_str(), 3806 ]) 3807 .capture_output() 3808 .spawn() 3809 .unwrap(); 3810 3811 thread::sleep(std::time::Duration::new(30, 0)); 3812 3813 let r = std::panic::catch_unwind(|| { 3814 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 3815 thread::sleep(std::time::Duration::new(120, 0)); 3816 3817 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 3818 // added to its kernel command line. 3819 // Let's ssh into it and verify that it's there. If it is it means 3820 // we're in the right guest (The L2 one) because the QEMU L1 guest 3821 // does not have this command line tag. 3822 assert_eq!( 3823 guest 3824 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 3825 .unwrap() 3826 .trim() 3827 .parse::<u32>() 3828 .unwrap_or_default(), 3829 1 3830 ); 3831 3832 // Let's also verify from the second virtio-net device passed to 3833 // the L2 VM. 3834 assert_eq!( 3835 guest 3836 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 3837 .unwrap() 3838 .trim() 3839 .parse::<u32>() 3840 .unwrap_or_default(), 3841 1 3842 ); 3843 3844 // Check the amount of PCI devices appearing in L2 VM. 3845 assert_eq!( 3846 guest 3847 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3848 .unwrap() 3849 .trim() 3850 .parse::<u32>() 3851 .unwrap_or_default(), 3852 8, 3853 ); 3854 3855 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 3856 assert_eq!( 3857 guest 3858 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 3859 .unwrap() 3860 .trim() 3861 .parse::<u32>() 3862 .unwrap_or_default(), 3863 1 3864 ); 3865 3866 // Hotplug an extra virtio-net device through L2 VM. 3867 guest 3868 .ssh_command_l1( 3869 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 3870 ) 3871 .unwrap(); 3872 guest 3873 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 3874 .unwrap(); 3875 let vfio_hotplug_output = guest 3876 .ssh_command_l1( 3877 "sudo /mnt/ch-remote \ 3878 --api-socket=/tmp/ch_api.sock \ 3879 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 3880 ) 3881 .unwrap(); 3882 assert!(vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}")); 3883 3884 thread::sleep(std::time::Duration::new(10, 0)); 3885 3886 // Let's also verify from the third virtio-net device passed to 3887 // the L2 VM. This third device has been hotplugged through the L2 3888 // VM, so this is our way to validate hotplug works for VFIO PCI. 3889 assert_eq!( 3890 guest 3891 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 3892 .unwrap() 3893 .trim() 3894 .parse::<u32>() 3895 .unwrap_or_default(), 3896 1 3897 ); 3898 3899 // Check the amount of PCI devices appearing in L2 VM. 3900 // There should be one more device than before, raising the count 3901 // up to 9 PCI devices. 3902 assert_eq!( 3903 guest 3904 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3905 .unwrap() 3906 .trim() 3907 .parse::<u32>() 3908 .unwrap_or_default(), 3909 9, 3910 ); 3911 3912 // Let's now verify that we can correctly remove the virtio-net 3913 // device through the "remove-device" command responsible for 3914 // unplugging VFIO devices. 3915 guest 3916 .ssh_command_l1( 3917 "sudo /mnt/ch-remote \ 3918 --api-socket=/tmp/ch_api.sock \ 3919 remove-device vfio123", 3920 ) 3921 .unwrap(); 3922 thread::sleep(std::time::Duration::new(10, 0)); 3923 3924 // Check the amount of PCI devices appearing in L2 VM is back down 3925 // to 8 devices. 3926 assert_eq!( 3927 guest 3928 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3929 .unwrap() 3930 .trim() 3931 .parse::<u32>() 3932 .unwrap_or_default(), 3933 8, 3934 ); 3935 3936 // Perform memory hotplug in L2 and validate the memory is showing 3937 // up as expected. In order to check, we will use the virtio-net 3938 // device already passed through L2 as a VFIO device, this will 3939 // verify that VFIO devices are functional with memory hotplug. 3940 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 3941 guest 3942 .ssh_command_l2_1( 3943 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 3944 ) 3945 .unwrap(); 3946 guest 3947 .ssh_command_l1( 3948 "sudo /mnt/ch-remote \ 3949 --api-socket=/tmp/ch_api.sock \ 3950 resize --memory=1073741824", 3951 ) 3952 .unwrap(); 3953 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 3954 }); 3955 3956 let _ = child.kill(); 3957 let output = child.wait_with_output().unwrap(); 3958 3959 cleanup_vfio_network_interfaces(); 3960 3961 handle_child_output(r, &output); 3962 } 3963 3964 #[test] 3965 fn test_direct_kernel_boot_noacpi() { 3966 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3967 let guest = Guest::new(Box::new(focal)); 3968 3969 let kernel_path = direct_kernel_boot_path(); 3970 3971 let mut child = GuestCommand::new(&guest) 3972 .args(["--cpus", "boot=1"]) 3973 .args(["--memory", "size=512M"]) 3974 .args(["--kernel", kernel_path.to_str().unwrap()]) 3975 .args([ 3976 "--cmdline", 3977 format!("{DIRECT_KERNEL_BOOT_CMDLINE} acpi=off").as_str(), 3978 ]) 3979 .default_disks() 3980 .default_net() 3981 .capture_output() 3982 .spawn() 3983 .unwrap(); 3984 3985 let r = std::panic::catch_unwind(|| { 3986 guest.wait_vm_boot(None).unwrap(); 3987 3988 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3989 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3990 }); 3991 3992 let _ = child.kill(); 3993 let output = child.wait_with_output().unwrap(); 3994 3995 handle_child_output(r, &output); 3996 } 3997 3998 #[test] 3999 fn test_virtio_vsock() { 4000 _test_virtio_vsock(false) 4001 } 4002 4003 #[test] 4004 fn test_virtio_vsock_hotplug() { 4005 _test_virtio_vsock(true); 4006 } 4007 4008 #[test] 4009 // Start cloud-hypervisor with no VM parameters, only the API server running. 4010 // From the API: Create a VM, boot it and check that it looks as expected. 4011 fn test_api_create_boot() { 4012 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4013 let guest = Guest::new(Box::new(focal)); 4014 4015 let api_socket = temp_api_path(&guest.tmp_dir); 4016 4017 let mut child = GuestCommand::new(&guest) 4018 .args(["--api-socket", &api_socket]) 4019 .capture_output() 4020 .spawn() 4021 .unwrap(); 4022 4023 thread::sleep(std::time::Duration::new(1, 0)); 4024 4025 // Verify API server is running 4026 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4027 4028 // Create the VM first 4029 let cpu_count: u8 = 4; 4030 let http_body = guest.api_create_body( 4031 cpu_count, 4032 direct_kernel_boot_path().to_str().unwrap(), 4033 DIRECT_KERNEL_BOOT_CMDLINE, 4034 ); 4035 4036 let temp_config_path = guest.tmp_dir.as_path().join("config"); 4037 std::fs::write(&temp_config_path, http_body).unwrap(); 4038 4039 remote_command( 4040 &api_socket, 4041 "create", 4042 Some(temp_config_path.as_os_str().to_str().unwrap()), 4043 ); 4044 4045 // Then boot it 4046 remote_command(&api_socket, "boot", None); 4047 thread::sleep(std::time::Duration::new(20, 0)); 4048 4049 let r = std::panic::catch_unwind(|| { 4050 // Check that the VM booted as expected 4051 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4052 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4053 }); 4054 4055 let _ = child.kill(); 4056 let output = child.wait_with_output().unwrap(); 4057 4058 handle_child_output(r, &output); 4059 } 4060 4061 #[test] 4062 // Start cloud-hypervisor with no VM parameters, only the API server running. 4063 // From the API: Create a VM, boot it and check it can be shutdown and then 4064 // booted again 4065 fn test_api_shutdown() { 4066 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4067 let guest = Guest::new(Box::new(focal)); 4068 4069 let api_socket = temp_api_path(&guest.tmp_dir); 4070 4071 let mut child = GuestCommand::new(&guest) 4072 .args(["--api-socket", &api_socket]) 4073 .capture_output() 4074 .spawn() 4075 .unwrap(); 4076 4077 thread::sleep(std::time::Duration::new(1, 0)); 4078 4079 // Verify API server is running 4080 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4081 4082 // Create the VM first 4083 let cpu_count: u8 = 4; 4084 let http_body = guest.api_create_body( 4085 cpu_count, 4086 direct_kernel_boot_path().to_str().unwrap(), 4087 DIRECT_KERNEL_BOOT_CMDLINE, 4088 ); 4089 4090 let r = std::panic::catch_unwind(|| { 4091 curl_command( 4092 &api_socket, 4093 "PUT", 4094 "http://localhost/api/v1/vm.create", 4095 Some(&http_body), 4096 ); 4097 4098 // Then boot it 4099 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4100 4101 guest.wait_vm_boot(None).unwrap(); 4102 4103 // Check that the VM booted as expected 4104 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4105 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4106 4107 // Sync and shutdown without powering off to prevent filesystem 4108 // corruption. 4109 guest.ssh_command("sync").unwrap(); 4110 guest.ssh_command("sudo shutdown -H now").unwrap(); 4111 4112 // Wait for the guest to be fully shutdown 4113 thread::sleep(std::time::Duration::new(20, 0)); 4114 4115 // Then shut it down 4116 curl_command( 4117 &api_socket, 4118 "PUT", 4119 "http://localhost/api/v1/vm.shutdown", 4120 None, 4121 ); 4122 4123 // Then boot it again 4124 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4125 4126 guest.wait_vm_boot(None).unwrap(); 4127 4128 // Check that the VM booted as expected 4129 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4130 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4131 }); 4132 4133 let _ = child.kill(); 4134 let output = child.wait_with_output().unwrap(); 4135 4136 handle_child_output(r, &output); 4137 } 4138 4139 #[test] 4140 // Start cloud-hypervisor with no VM parameters, only the API server running. 4141 // From the API: Create a VM, boot it and check it can be deleted and then recreated 4142 // booted again. 4143 fn test_api_delete() { 4144 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4145 let guest = Guest::new(Box::new(focal)); 4146 4147 let api_socket = temp_api_path(&guest.tmp_dir); 4148 4149 let mut child = GuestCommand::new(&guest) 4150 .args(["--api-socket", &api_socket]) 4151 .capture_output() 4152 .spawn() 4153 .unwrap(); 4154 4155 thread::sleep(std::time::Duration::new(1, 0)); 4156 4157 // Verify API server is running 4158 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4159 4160 // Create the VM first 4161 let cpu_count: u8 = 4; 4162 let http_body = guest.api_create_body( 4163 cpu_count, 4164 direct_kernel_boot_path().to_str().unwrap(), 4165 DIRECT_KERNEL_BOOT_CMDLINE, 4166 ); 4167 4168 let r = std::panic::catch_unwind(|| { 4169 curl_command( 4170 &api_socket, 4171 "PUT", 4172 "http://localhost/api/v1/vm.create", 4173 Some(&http_body), 4174 ); 4175 4176 // Then boot it 4177 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4178 4179 guest.wait_vm_boot(None).unwrap(); 4180 4181 // Check that the VM booted as expected 4182 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4183 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4184 4185 // Sync and shutdown without powering off to prevent filesystem 4186 // corruption. 4187 guest.ssh_command("sync").unwrap(); 4188 guest.ssh_command("sudo shutdown -H now").unwrap(); 4189 4190 // Wait for the guest to be fully shutdown 4191 thread::sleep(std::time::Duration::new(20, 0)); 4192 4193 // Then delete it 4194 curl_command( 4195 &api_socket, 4196 "PUT", 4197 "http://localhost/api/v1/vm.delete", 4198 None, 4199 ); 4200 4201 curl_command( 4202 &api_socket, 4203 "PUT", 4204 "http://localhost/api/v1/vm.create", 4205 Some(&http_body), 4206 ); 4207 4208 // Then boot it again 4209 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4210 4211 guest.wait_vm_boot(None).unwrap(); 4212 4213 // Check that the VM booted as expected 4214 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4215 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4216 }); 4217 4218 let _ = child.kill(); 4219 let output = child.wait_with_output().unwrap(); 4220 4221 handle_child_output(r, &output); 4222 } 4223 4224 #[test] 4225 // Start cloud-hypervisor with no VM parameters, only the API server running. 4226 // From the API: Create a VM, boot it and check that it looks as expected. 4227 // Then we pause the VM, check that it's no longer available. 4228 // Finally we resume the VM and check that it's available. 4229 fn test_api_pause_resume() { 4230 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4231 let guest = Guest::new(Box::new(focal)); 4232 4233 let api_socket = temp_api_path(&guest.tmp_dir); 4234 4235 let mut child = GuestCommand::new(&guest) 4236 .args(["--api-socket", &api_socket]) 4237 .capture_output() 4238 .spawn() 4239 .unwrap(); 4240 4241 thread::sleep(std::time::Duration::new(1, 0)); 4242 4243 // Verify API server is running 4244 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4245 4246 // Create the VM first 4247 let cpu_count: u8 = 4; 4248 let http_body = guest.api_create_body( 4249 cpu_count, 4250 direct_kernel_boot_path().to_str().unwrap(), 4251 DIRECT_KERNEL_BOOT_CMDLINE, 4252 ); 4253 curl_command( 4254 &api_socket, 4255 "PUT", 4256 "http://localhost/api/v1/vm.create", 4257 Some(&http_body), 4258 ); 4259 4260 // Then boot it 4261 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4262 thread::sleep(std::time::Duration::new(20, 0)); 4263 4264 let r = std::panic::catch_unwind(|| { 4265 // Check that the VM booted as expected 4266 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4267 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4268 4269 // We now pause the VM 4270 assert!(remote_command(&api_socket, "pause", None)); 4271 4272 // Check pausing again fails 4273 assert!(!remote_command(&api_socket, "pause", None)); 4274 4275 thread::sleep(std::time::Duration::new(2, 0)); 4276 4277 // SSH into the VM should fail 4278 assert!(ssh_command_ip( 4279 "grep -c processor /proc/cpuinfo", 4280 &guest.network.guest_ip, 4281 2, 4282 5 4283 ) 4284 .is_err()); 4285 4286 // Resume the VM 4287 assert!(remote_command(&api_socket, "resume", None)); 4288 4289 // Check resuming again fails 4290 assert!(!remote_command(&api_socket, "resume", None)); 4291 4292 thread::sleep(std::time::Duration::new(2, 0)); 4293 4294 // Now we should be able to SSH back in and get the right number of CPUs 4295 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4296 }); 4297 4298 let _ = child.kill(); 4299 let output = child.wait_with_output().unwrap(); 4300 4301 handle_child_output(r, &output); 4302 } 4303 4304 #[test] 4305 fn test_virtio_iommu() { 4306 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4307 } 4308 4309 #[test] 4310 // We cannot force the software running in the guest to reprogram the BAR 4311 // with some different addresses, but we have a reliable way of testing it 4312 // with a standard Linux kernel. 4313 // By removing a device from the PCI tree, and then rescanning the tree, 4314 // Linux consistently chooses to reorganize the PCI device BARs to other 4315 // locations in the guest address space. 4316 // This test creates a dedicated PCI network device to be checked as being 4317 // properly probed first, then removing it, and adding it again by doing a 4318 // rescan. 4319 fn test_pci_bar_reprogramming() { 4320 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4321 let guest = Guest::new(Box::new(focal)); 4322 4323 #[cfg(target_arch = "x86_64")] 4324 let kernel_path = direct_kernel_boot_path(); 4325 #[cfg(target_arch = "aarch64")] 4326 let kernel_path = edk2_path(); 4327 4328 let mut child = GuestCommand::new(&guest) 4329 .args(["--cpus", "boot=1"]) 4330 .args(["--memory", "size=512M"]) 4331 .args(["--kernel", kernel_path.to_str().unwrap()]) 4332 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4333 .default_disks() 4334 .args([ 4335 "--net", 4336 guest.default_net_string().as_str(), 4337 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4338 ]) 4339 .capture_output() 4340 .spawn() 4341 .unwrap(); 4342 4343 let r = std::panic::catch_unwind(|| { 4344 guest.wait_vm_boot(None).unwrap(); 4345 4346 // 2 network interfaces + default localhost ==> 3 interfaces 4347 assert_eq!( 4348 guest 4349 .ssh_command("ip -o link | wc -l") 4350 .unwrap() 4351 .trim() 4352 .parse::<u32>() 4353 .unwrap_or_default(), 4354 3 4355 ); 4356 4357 let init_bar_addr = guest 4358 .ssh_command( 4359 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4360 ) 4361 .unwrap(); 4362 4363 // Remove the PCI device 4364 guest 4365 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4366 .unwrap(); 4367 4368 // Only 1 network interface left + default localhost ==> 2 interfaces 4369 assert_eq!( 4370 guest 4371 .ssh_command("ip -o link | wc -l") 4372 .unwrap() 4373 .trim() 4374 .parse::<u32>() 4375 .unwrap_or_default(), 4376 2 4377 ); 4378 4379 // Remove the PCI device 4380 guest 4381 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4382 .unwrap(); 4383 4384 // Back to 2 network interface + default localhost ==> 3 interfaces 4385 assert_eq!( 4386 guest 4387 .ssh_command("ip -o link | wc -l") 4388 .unwrap() 4389 .trim() 4390 .parse::<u32>() 4391 .unwrap_or_default(), 4392 3 4393 ); 4394 4395 let new_bar_addr = guest 4396 .ssh_command( 4397 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4398 ) 4399 .unwrap(); 4400 4401 // Let's compare the BAR addresses for our virtio-net device. 4402 // They should be different as we expect the BAR reprogramming 4403 // to have happened. 4404 assert_ne!(init_bar_addr, new_bar_addr); 4405 }); 4406 4407 let _ = child.kill(); 4408 let output = child.wait_with_output().unwrap(); 4409 4410 handle_child_output(r, &output); 4411 } 4412 4413 #[test] 4414 fn test_memory_mergeable_off() { 4415 test_memory_mergeable(false) 4416 } 4417 4418 #[test] 4419 #[cfg(target_arch = "x86_64")] 4420 fn test_cpu_hotplug() { 4421 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4422 let guest = Guest::new(Box::new(focal)); 4423 let api_socket = temp_api_path(&guest.tmp_dir); 4424 4425 let kernel_path = direct_kernel_boot_path(); 4426 4427 let mut child = GuestCommand::new(&guest) 4428 .args(["--cpus", "boot=2,max=4"]) 4429 .args(["--memory", "size=512M"]) 4430 .args(["--kernel", kernel_path.to_str().unwrap()]) 4431 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4432 .default_disks() 4433 .default_net() 4434 .args(["--api-socket", &api_socket]) 4435 .capture_output() 4436 .spawn() 4437 .unwrap(); 4438 4439 let r = std::panic::catch_unwind(|| { 4440 guest.wait_vm_boot(None).unwrap(); 4441 4442 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4443 4444 // Resize the VM 4445 let desired_vcpus = 4; 4446 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4447 4448 guest 4449 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4450 .unwrap(); 4451 guest 4452 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4453 .unwrap(); 4454 thread::sleep(std::time::Duration::new(10, 0)); 4455 assert_eq!( 4456 guest.get_cpu_count().unwrap_or_default(), 4457 u32::from(desired_vcpus) 4458 ); 4459 4460 guest.reboot_linux(0, None); 4461 4462 assert_eq!( 4463 guest.get_cpu_count().unwrap_or_default(), 4464 u32::from(desired_vcpus) 4465 ); 4466 4467 // Resize the VM 4468 let desired_vcpus = 2; 4469 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4470 4471 thread::sleep(std::time::Duration::new(10, 0)); 4472 assert_eq!( 4473 guest.get_cpu_count().unwrap_or_default(), 4474 u32::from(desired_vcpus) 4475 ); 4476 4477 // Resize the VM back up to 4 4478 let desired_vcpus = 4; 4479 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4480 4481 guest 4482 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4483 .unwrap(); 4484 guest 4485 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4486 .unwrap(); 4487 thread::sleep(std::time::Duration::new(10, 0)); 4488 assert_eq!( 4489 guest.get_cpu_count().unwrap_or_default(), 4490 u32::from(desired_vcpus) 4491 ); 4492 }); 4493 4494 let _ = child.kill(); 4495 let output = child.wait_with_output().unwrap(); 4496 4497 handle_child_output(r, &output); 4498 } 4499 4500 #[test] 4501 fn test_memory_hotplug() { 4502 #[cfg(target_arch = "aarch64")] 4503 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4504 #[cfg(target_arch = "x86_64")] 4505 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4506 let focal = UbuntuDiskConfig::new(focal_image); 4507 let guest = Guest::new(Box::new(focal)); 4508 let api_socket = temp_api_path(&guest.tmp_dir); 4509 4510 #[cfg(target_arch = "aarch64")] 4511 let kernel_path = edk2_path(); 4512 #[cfg(target_arch = "x86_64")] 4513 let kernel_path = direct_kernel_boot_path(); 4514 4515 let mut child = GuestCommand::new(&guest) 4516 .args(["--cpus", "boot=2,max=4"]) 4517 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4518 .args(["--kernel", kernel_path.to_str().unwrap()]) 4519 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4520 .default_disks() 4521 .default_net() 4522 .args(["--balloon", "size=0"]) 4523 .args(["--api-socket", &api_socket]) 4524 .capture_output() 4525 .spawn() 4526 .unwrap(); 4527 4528 let r = std::panic::catch_unwind(|| { 4529 guest.wait_vm_boot(None).unwrap(); 4530 4531 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4532 4533 guest.enable_memory_hotplug(); 4534 4535 // Add RAM to the VM 4536 let desired_ram = 1024 << 20; 4537 resize_command(&api_socket, None, Some(desired_ram), None, None); 4538 4539 thread::sleep(std::time::Duration::new(10, 0)); 4540 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4541 4542 // Use balloon to remove RAM from the VM 4543 let desired_balloon = 512 << 20; 4544 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4545 4546 thread::sleep(std::time::Duration::new(10, 0)); 4547 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4548 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4549 4550 guest.reboot_linux(0, None); 4551 4552 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4553 4554 // Use balloon add RAM to the VM 4555 let desired_balloon = 0; 4556 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4557 4558 thread::sleep(std::time::Duration::new(10, 0)); 4559 4560 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4561 4562 guest.enable_memory_hotplug(); 4563 4564 // Add RAM to the VM 4565 let desired_ram = 2048 << 20; 4566 resize_command(&api_socket, None, Some(desired_ram), None, None); 4567 4568 thread::sleep(std::time::Duration::new(10, 0)); 4569 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4570 4571 // Remove RAM to the VM (only applies after reboot) 4572 let desired_ram = 1024 << 20; 4573 resize_command(&api_socket, None, Some(desired_ram), None, None); 4574 4575 guest.reboot_linux(1, None); 4576 4577 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4578 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4579 }); 4580 4581 let _ = child.kill(); 4582 let output = child.wait_with_output().unwrap(); 4583 4584 handle_child_output(r, &output); 4585 } 4586 4587 #[test] 4588 #[cfg(not(feature = "mshv"))] 4589 fn test_virtio_mem() { 4590 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4591 let guest = Guest::new(Box::new(focal)); 4592 let api_socket = temp_api_path(&guest.tmp_dir); 4593 4594 let kernel_path = direct_kernel_boot_path(); 4595 4596 let mut child = GuestCommand::new(&guest) 4597 .args(["--cpus", "boot=2,max=4"]) 4598 .args([ 4599 "--memory", 4600 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4601 ]) 4602 .args(["--kernel", kernel_path.to_str().unwrap()]) 4603 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4604 .default_disks() 4605 .default_net() 4606 .args(["--api-socket", &api_socket]) 4607 .capture_output() 4608 .spawn() 4609 .unwrap(); 4610 4611 let r = std::panic::catch_unwind(|| { 4612 guest.wait_vm_boot(None).unwrap(); 4613 4614 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4615 4616 guest.enable_memory_hotplug(); 4617 4618 // Add RAM to the VM 4619 let desired_ram = 1024 << 20; 4620 resize_command(&api_socket, None, Some(desired_ram), None, None); 4621 4622 thread::sleep(std::time::Duration::new(10, 0)); 4623 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4624 4625 // Add RAM to the VM 4626 let desired_ram = 2048 << 20; 4627 resize_command(&api_socket, None, Some(desired_ram), None, None); 4628 4629 thread::sleep(std::time::Duration::new(10, 0)); 4630 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4631 4632 // Remove RAM from the VM 4633 let desired_ram = 1024 << 20; 4634 resize_command(&api_socket, None, Some(desired_ram), None, None); 4635 4636 thread::sleep(std::time::Duration::new(10, 0)); 4637 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4638 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4639 4640 guest.reboot_linux(0, None); 4641 4642 // Check the amount of memory after reboot is 1GiB 4643 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4644 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4645 4646 // Check we can still resize to 512MiB 4647 let desired_ram = 512 << 20; 4648 resize_command(&api_socket, None, Some(desired_ram), None, None); 4649 thread::sleep(std::time::Duration::new(10, 0)); 4650 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4651 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4652 }); 4653 4654 let _ = child.kill(); 4655 let output = child.wait_with_output().unwrap(); 4656 4657 handle_child_output(r, &output); 4658 } 4659 4660 #[test] 4661 #[cfg(target_arch = "x86_64")] 4662 #[cfg(not(feature = "mshv"))] 4663 // Test both vCPU and memory resizing together 4664 fn test_resize() { 4665 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4666 let guest = Guest::new(Box::new(focal)); 4667 let api_socket = temp_api_path(&guest.tmp_dir); 4668 4669 let kernel_path = direct_kernel_boot_path(); 4670 4671 let mut child = GuestCommand::new(&guest) 4672 .args(["--cpus", "boot=2,max=4"]) 4673 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4674 .args(["--kernel", kernel_path.to_str().unwrap()]) 4675 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4676 .default_disks() 4677 .default_net() 4678 .args(["--api-socket", &api_socket]) 4679 .capture_output() 4680 .spawn() 4681 .unwrap(); 4682 4683 let r = std::panic::catch_unwind(|| { 4684 guest.wait_vm_boot(None).unwrap(); 4685 4686 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4687 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4688 4689 guest.enable_memory_hotplug(); 4690 4691 // Resize the VM 4692 let desired_vcpus = 4; 4693 let desired_ram = 1024 << 20; 4694 resize_command( 4695 &api_socket, 4696 Some(desired_vcpus), 4697 Some(desired_ram), 4698 None, 4699 None, 4700 ); 4701 4702 guest 4703 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4704 .unwrap(); 4705 guest 4706 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4707 .unwrap(); 4708 thread::sleep(std::time::Duration::new(10, 0)); 4709 assert_eq!( 4710 guest.get_cpu_count().unwrap_or_default(), 4711 u32::from(desired_vcpus) 4712 ); 4713 4714 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4715 }); 4716 4717 let _ = child.kill(); 4718 let output = child.wait_with_output().unwrap(); 4719 4720 handle_child_output(r, &output); 4721 } 4722 4723 #[test] 4724 fn test_memory_overhead() { 4725 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4726 let guest = Guest::new(Box::new(focal)); 4727 4728 let kernel_path = direct_kernel_boot_path(); 4729 4730 let guest_memory_size_kb = 512 * 1024; 4731 4732 let mut child = GuestCommand::new(&guest) 4733 .args(["--cpus", "boot=1"]) 4734 .args(["--memory", format!("size={guest_memory_size_kb}K").as_str()]) 4735 .args(["--kernel", kernel_path.to_str().unwrap()]) 4736 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4737 .default_disks() 4738 .capture_output() 4739 .spawn() 4740 .unwrap(); 4741 4742 thread::sleep(std::time::Duration::new(20, 0)); 4743 4744 let r = std::panic::catch_unwind(|| { 4745 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 4746 eprintln!("Guest memory overhead: {overhead} vs {MAXIMUM_VMM_OVERHEAD_KB}"); 4747 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 4748 }); 4749 4750 let _ = child.kill(); 4751 let output = child.wait_with_output().unwrap(); 4752 4753 handle_child_output(r, &output); 4754 } 4755 4756 #[test] 4757 fn test_disk_hotplug() { 4758 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4759 let guest = Guest::new(Box::new(focal)); 4760 4761 #[cfg(target_arch = "x86_64")] 4762 let kernel_path = direct_kernel_boot_path(); 4763 #[cfg(target_arch = "aarch64")] 4764 let kernel_path = edk2_path(); 4765 4766 let api_socket = temp_api_path(&guest.tmp_dir); 4767 4768 let mut child = GuestCommand::new(&guest) 4769 .args(["--api-socket", &api_socket]) 4770 .args(["--cpus", "boot=1"]) 4771 .args(["--memory", "size=512M"]) 4772 .args(["--kernel", kernel_path.to_str().unwrap()]) 4773 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4774 .default_disks() 4775 .default_net() 4776 .capture_output() 4777 .spawn() 4778 .unwrap(); 4779 4780 let r = std::panic::catch_unwind(|| { 4781 guest.wait_vm_boot(None).unwrap(); 4782 4783 // Check /dev/vdc is not there 4784 assert_eq!( 4785 guest 4786 .ssh_command("lsblk | grep -c vdc.*16M || true") 4787 .unwrap() 4788 .trim() 4789 .parse::<u32>() 4790 .unwrap_or(1), 4791 0 4792 ); 4793 4794 // Now let's add the extra disk. 4795 let mut blk_file_path = dirs::home_dir().unwrap(); 4796 blk_file_path.push("workloads"); 4797 blk_file_path.push("blk.img"); 4798 let (cmd_success, cmd_output) = remote_command_w_output( 4799 &api_socket, 4800 "add-disk", 4801 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4802 ); 4803 assert!(cmd_success); 4804 assert!(String::from_utf8_lossy(&cmd_output) 4805 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4806 4807 thread::sleep(std::time::Duration::new(10, 0)); 4808 4809 // Check that /dev/vdc exists and the block size is 16M. 4810 assert_eq!( 4811 guest 4812 .ssh_command("lsblk | grep vdc | grep -c 16M") 4813 .unwrap() 4814 .trim() 4815 .parse::<u32>() 4816 .unwrap_or_default(), 4817 1 4818 ); 4819 // And check the block device can be read. 4820 guest 4821 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4822 .unwrap(); 4823 4824 // Let's remove it the extra disk. 4825 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4826 thread::sleep(std::time::Duration::new(5, 0)); 4827 // And check /dev/vdc is not there 4828 assert_eq!( 4829 guest 4830 .ssh_command("lsblk | grep -c vdc.*16M || true") 4831 .unwrap() 4832 .trim() 4833 .parse::<u32>() 4834 .unwrap_or(1), 4835 0 4836 ); 4837 4838 // And add it back to validate unplug did work correctly. 4839 let (cmd_success, cmd_output) = remote_command_w_output( 4840 &api_socket, 4841 "add-disk", 4842 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4843 ); 4844 assert!(cmd_success); 4845 assert!(String::from_utf8_lossy(&cmd_output) 4846 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4847 4848 thread::sleep(std::time::Duration::new(10, 0)); 4849 4850 // Check that /dev/vdc exists and the block size is 16M. 4851 assert_eq!( 4852 guest 4853 .ssh_command("lsblk | grep vdc | grep -c 16M") 4854 .unwrap() 4855 .trim() 4856 .parse::<u32>() 4857 .unwrap_or_default(), 4858 1 4859 ); 4860 // And check the block device can be read. 4861 guest 4862 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4863 .unwrap(); 4864 4865 // Reboot the VM. 4866 guest.reboot_linux(0, None); 4867 4868 // Check still there after reboot 4869 assert_eq!( 4870 guest 4871 .ssh_command("lsblk | grep vdc | grep -c 16M") 4872 .unwrap() 4873 .trim() 4874 .parse::<u32>() 4875 .unwrap_or_default(), 4876 1 4877 ); 4878 4879 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4880 4881 thread::sleep(std::time::Duration::new(20, 0)); 4882 4883 // Check device has gone away 4884 assert_eq!( 4885 guest 4886 .ssh_command("lsblk | grep -c vdc.*16M || true") 4887 .unwrap() 4888 .trim() 4889 .parse::<u32>() 4890 .unwrap_or(1), 4891 0 4892 ); 4893 4894 guest.reboot_linux(1, None); 4895 4896 // Check device still absent 4897 assert_eq!( 4898 guest 4899 .ssh_command("lsblk | grep -c vdc.*16M || true") 4900 .unwrap() 4901 .trim() 4902 .parse::<u32>() 4903 .unwrap_or(1), 4904 0 4905 ); 4906 }); 4907 4908 let _ = child.kill(); 4909 let output = child.wait_with_output().unwrap(); 4910 4911 handle_child_output(r, &output); 4912 } 4913 4914 #[allow(clippy::useless_conversion)] 4915 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 4916 const LOOP_CONFIGURE: u64 = 0x4c0a; 4917 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 4918 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 4919 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 4920 4921 #[repr(C)] 4922 struct LoopInfo64 { 4923 lo_device: u64, 4924 lo_inode: u64, 4925 lo_rdevice: u64, 4926 lo_offset: u64, 4927 lo_sizelimit: u64, 4928 lo_number: u32, 4929 lo_encrypt_type: u32, 4930 lo_encrypt_key_size: u32, 4931 lo_flags: u32, 4932 lo_file_name: [u8; 64], 4933 lo_crypt_name: [u8; 64], 4934 lo_encrypt_key: [u8; 32], 4935 lo_init: [u64; 2], 4936 } 4937 4938 impl Default for LoopInfo64 { 4939 fn default() -> Self { 4940 LoopInfo64 { 4941 lo_device: 0, 4942 lo_inode: 0, 4943 lo_rdevice: 0, 4944 lo_offset: 0, 4945 lo_sizelimit: 0, 4946 lo_number: 0, 4947 lo_encrypt_type: 0, 4948 lo_encrypt_key_size: 0, 4949 lo_flags: 0, 4950 lo_file_name: [0; 64], 4951 lo_crypt_name: [0; 64], 4952 lo_encrypt_key: [0; 32], 4953 lo_init: [0; 2], 4954 } 4955 } 4956 } 4957 4958 #[derive(Default)] 4959 #[repr(C)] 4960 struct LoopConfig { 4961 fd: u32, 4962 block_size: u32, 4963 info: LoopInfo64, 4964 _reserved: [u64; 8], 4965 } 4966 4967 // Open loop-control device 4968 let loop_ctl_file = OpenOptions::new() 4969 .read(true) 4970 .write(true) 4971 .open(LOOP_CTL_PATH) 4972 .unwrap(); 4973 4974 // Request a free loop device 4975 let loop_device_number = unsafe { 4976 libc::ioctl( 4977 loop_ctl_file.as_raw_fd(), 4978 LOOP_CTL_GET_FREE.try_into().unwrap(), 4979 ) 4980 }; 4981 if loop_device_number < 0 { 4982 panic!("Couldn't find a free loop device"); 4983 } 4984 4985 // Create loop device path 4986 let loop_device_path = format!("{LOOP_DEVICE_PREFIX}{loop_device_number}"); 4987 4988 // Open loop device 4989 let loop_device_file = OpenOptions::new() 4990 .read(true) 4991 .write(true) 4992 .open(&loop_device_path) 4993 .unwrap(); 4994 4995 // Open backing file 4996 let backing_file = OpenOptions::new() 4997 .read(true) 4998 .write(true) 4999 .open(backing_file_path) 5000 .unwrap(); 5001 5002 let loop_config = LoopConfig { 5003 fd: backing_file.as_raw_fd() as u32, 5004 block_size, 5005 ..Default::default() 5006 }; 5007 5008 for i in 0..num_retries { 5009 let ret = unsafe { 5010 libc::ioctl( 5011 loop_device_file.as_raw_fd(), 5012 LOOP_CONFIGURE.try_into().unwrap(), 5013 &loop_config, 5014 ) 5015 }; 5016 if ret != 0 { 5017 if i < num_retries - 1 { 5018 println!( 5019 "Iteration {}: Failed to configure the loop device {}: {}", 5020 i, 5021 loop_device_path, 5022 std::io::Error::last_os_error() 5023 ); 5024 } else { 5025 panic!( 5026 "Failed {} times trying to configure the loop device {}: {}", 5027 num_retries, 5028 loop_device_path, 5029 std::io::Error::last_os_error() 5030 ); 5031 } 5032 } else { 5033 break; 5034 } 5035 5036 // Wait for a bit before retrying 5037 thread::sleep(std::time::Duration::new(5, 0)); 5038 } 5039 5040 loop_device_path 5041 } 5042 5043 #[test] 5044 fn test_virtio_block_topology() { 5045 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5046 let guest = Guest::new(Box::new(focal)); 5047 5048 let kernel_path = direct_kernel_boot_path(); 5049 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5050 5051 let output = exec_host_command_output( 5052 format!( 5053 "qemu-img create -f raw {} 16M", 5054 test_disk_path.to_str().unwrap() 5055 ) 5056 .as_str(), 5057 ); 5058 if !output.status.success() { 5059 let stdout = String::from_utf8_lossy(&output.stdout); 5060 let stderr = String::from_utf8_lossy(&output.stderr); 5061 panic!("qemu-img command failed\nstdout\n{stdout}\nstderr\n{stderr}"); 5062 } 5063 5064 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5065 5066 let mut child = GuestCommand::new(&guest) 5067 .args(["--cpus", "boot=1"]) 5068 .args(["--memory", "size=512M"]) 5069 .args(["--kernel", kernel_path.to_str().unwrap()]) 5070 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5071 .args([ 5072 "--disk", 5073 format!( 5074 "path={}", 5075 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5076 ) 5077 .as_str(), 5078 format!( 5079 "path={}", 5080 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5081 ) 5082 .as_str(), 5083 format!("path={}", &loop_dev).as_str(), 5084 ]) 5085 .default_net() 5086 .capture_output() 5087 .spawn() 5088 .unwrap(); 5089 5090 let r = std::panic::catch_unwind(|| { 5091 guest.wait_vm_boot(None).unwrap(); 5092 5093 // MIN-IO column 5094 assert_eq!( 5095 guest 5096 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5097 .unwrap() 5098 .trim() 5099 .parse::<u32>() 5100 .unwrap_or_default(), 5101 4096 5102 ); 5103 // PHY-SEC column 5104 assert_eq!( 5105 guest 5106 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5107 .unwrap() 5108 .trim() 5109 .parse::<u32>() 5110 .unwrap_or_default(), 5111 4096 5112 ); 5113 // LOG-SEC column 5114 assert_eq!( 5115 guest 5116 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5117 .unwrap() 5118 .trim() 5119 .parse::<u32>() 5120 .unwrap_or_default(), 5121 4096 5122 ); 5123 }); 5124 5125 let _ = child.kill(); 5126 let output = child.wait_with_output().unwrap(); 5127 5128 handle_child_output(r, &output); 5129 5130 Command::new("losetup") 5131 .args(["-d", &loop_dev]) 5132 .output() 5133 .expect("loop device not found"); 5134 } 5135 5136 #[test] 5137 fn test_virtio_balloon_deflate_on_oom() { 5138 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5139 let guest = Guest::new(Box::new(focal)); 5140 5141 let kernel_path = direct_kernel_boot_path(); 5142 5143 let api_socket = temp_api_path(&guest.tmp_dir); 5144 5145 //Let's start a 4G guest with balloon occupied 2G memory 5146 let mut child = GuestCommand::new(&guest) 5147 .args(["--api-socket", &api_socket]) 5148 .args(["--cpus", "boot=1"]) 5149 .args(["--memory", "size=4G"]) 5150 .args(["--kernel", kernel_path.to_str().unwrap()]) 5151 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5152 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5153 .default_disks() 5154 .default_net() 5155 .capture_output() 5156 .spawn() 5157 .unwrap(); 5158 5159 let r = std::panic::catch_unwind(|| { 5160 guest.wait_vm_boot(None).unwrap(); 5161 5162 // Wait for balloon memory's initialization and check its size. 5163 // The virtio-balloon driver might take a few seconds to report the 5164 // balloon effective size back to the VMM. 5165 thread::sleep(std::time::Duration::new(20, 0)); 5166 5167 let orig_balloon = balloon_size(&api_socket); 5168 println!("The original balloon memory size is {orig_balloon} bytes"); 5169 assert!(orig_balloon == 2147483648); 5170 5171 // Two steps to verify if the 'deflate_on_oom' parameter works. 5172 // 1st: run a command to trigger an OOM in the guest. 5173 guest 5174 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5175 .unwrap(); 5176 5177 // Give some time for the OOM to happen in the guest and be reported 5178 // back to the host. 5179 thread::sleep(std::time::Duration::new(20, 0)); 5180 5181 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5182 let deflated_balloon = balloon_size(&api_socket); 5183 println!("After deflating, balloon memory size is {deflated_balloon} bytes"); 5184 // Verify the balloon size deflated 5185 assert!(deflated_balloon < 2147483648); 5186 }); 5187 5188 let _ = child.kill(); 5189 let output = child.wait_with_output().unwrap(); 5190 5191 handle_child_output(r, &output); 5192 } 5193 5194 #[test] 5195 fn test_virtio_balloon_free_page_reporting() { 5196 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5197 let guest = Guest::new(Box::new(focal)); 5198 5199 //Let's start a 4G guest with balloon occupied 2G memory 5200 let mut child = GuestCommand::new(&guest) 5201 .args(["--cpus", "boot=1"]) 5202 .args(["--memory", "size=4G"]) 5203 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5204 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5205 .args(["--balloon", "size=0,free_page_reporting=on"]) 5206 .default_disks() 5207 .default_net() 5208 .capture_output() 5209 .spawn() 5210 .unwrap(); 5211 5212 let pid = child.id(); 5213 let r = std::panic::catch_unwind(|| { 5214 guest.wait_vm_boot(None).unwrap(); 5215 5216 // Check the initial RSS is less than 1GiB 5217 let rss = process_rss_kib(pid); 5218 println!("RSS {rss} < 1048576"); 5219 assert!(rss < 1048576); 5220 5221 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5222 // seconds 5223 let guest_ip = guest.network.guest_ip.clone(); 5224 thread::spawn(move || { 5225 ssh_command_ip( 5226 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5227 &guest_ip, 5228 DEFAULT_SSH_RETRIES, 5229 DEFAULT_SSH_TIMEOUT, 5230 ) 5231 .unwrap(); 5232 }); 5233 5234 // Wait for 50 seconds to make sure the stress command is consuming 5235 // the expected amount of memory. 5236 thread::sleep(std::time::Duration::new(50, 0)); 5237 let rss = process_rss_kib(pid); 5238 println!("RSS {rss} >= 2097152"); 5239 assert!(rss >= 2097152); 5240 5241 // Wait for an extra minute to make sure the stress command has 5242 // completed and that the guest reported the free pages to the VMM 5243 // through the virtio-balloon device. We expect the RSS to be under 5244 // 2GiB. 5245 thread::sleep(std::time::Duration::new(60, 0)); 5246 let rss = process_rss_kib(pid); 5247 println!("RSS {rss} < 2097152"); 5248 assert!(rss < 2097152); 5249 }); 5250 5251 let _ = child.kill(); 5252 let output = child.wait_with_output().unwrap(); 5253 5254 handle_child_output(r, &output); 5255 } 5256 5257 #[test] 5258 fn test_pmem_hotplug() { 5259 _test_pmem_hotplug(None) 5260 } 5261 5262 #[test] 5263 fn test_pmem_multi_segment_hotplug() { 5264 _test_pmem_hotplug(Some(15)) 5265 } 5266 5267 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5268 #[cfg(target_arch = "aarch64")] 5269 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5270 #[cfg(target_arch = "x86_64")] 5271 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5272 let focal = UbuntuDiskConfig::new(focal_image); 5273 let guest = Guest::new(Box::new(focal)); 5274 5275 #[cfg(target_arch = "x86_64")] 5276 let kernel_path = direct_kernel_boot_path(); 5277 #[cfg(target_arch = "aarch64")] 5278 let kernel_path = edk2_path(); 5279 5280 let api_socket = temp_api_path(&guest.tmp_dir); 5281 5282 let mut cmd = GuestCommand::new(&guest); 5283 5284 cmd.args(["--api-socket", &api_socket]) 5285 .args(["--cpus", "boot=1"]) 5286 .args(["--memory", "size=512M"]) 5287 .args(["--kernel", kernel_path.to_str().unwrap()]) 5288 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5289 .default_disks() 5290 .default_net() 5291 .capture_output(); 5292 5293 if pci_segment.is_some() { 5294 cmd.args(["--platform", "num_pci_segments=16"]); 5295 } 5296 5297 let mut child = cmd.spawn().unwrap(); 5298 5299 let r = std::panic::catch_unwind(|| { 5300 guest.wait_vm_boot(None).unwrap(); 5301 5302 // Check /dev/pmem0 is not there 5303 assert_eq!( 5304 guest 5305 .ssh_command("lsblk | grep -c pmem0 || true") 5306 .unwrap() 5307 .trim() 5308 .parse::<u32>() 5309 .unwrap_or(1), 5310 0 5311 ); 5312 5313 let pmem_temp_file = TempFile::new().unwrap(); 5314 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5315 let (cmd_success, cmd_output) = remote_command_w_output( 5316 &api_socket, 5317 "add-pmem", 5318 Some(&format!( 5319 "file={},id=test0{}", 5320 pmem_temp_file.as_path().to_str().unwrap(), 5321 if let Some(pci_segment) = pci_segment { 5322 format!(",pci_segment={pci_segment}") 5323 } else { 5324 "".to_owned() 5325 } 5326 )), 5327 ); 5328 assert!(cmd_success); 5329 if let Some(pci_segment) = pci_segment { 5330 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5331 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5332 ))); 5333 } else { 5334 assert!(String::from_utf8_lossy(&cmd_output) 5335 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5336 } 5337 5338 // Check that /dev/pmem0 exists and the block size is 128M 5339 assert_eq!( 5340 guest 5341 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5342 .unwrap() 5343 .trim() 5344 .parse::<u32>() 5345 .unwrap_or_default(), 5346 1 5347 ); 5348 5349 guest.reboot_linux(0, None); 5350 5351 // Check still there after reboot 5352 assert_eq!( 5353 guest 5354 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5355 .unwrap() 5356 .trim() 5357 .parse::<u32>() 5358 .unwrap_or_default(), 5359 1 5360 ); 5361 5362 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5363 5364 thread::sleep(std::time::Duration::new(20, 0)); 5365 5366 // Check device has gone away 5367 assert_eq!( 5368 guest 5369 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5370 .unwrap() 5371 .trim() 5372 .parse::<u32>() 5373 .unwrap_or(1), 5374 0 5375 ); 5376 5377 guest.reboot_linux(1, None); 5378 5379 // Check still absent after reboot 5380 assert_eq!( 5381 guest 5382 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5383 .unwrap() 5384 .trim() 5385 .parse::<u32>() 5386 .unwrap_or(1), 5387 0 5388 ); 5389 }); 5390 5391 let _ = child.kill(); 5392 let output = child.wait_with_output().unwrap(); 5393 5394 handle_child_output(r, &output); 5395 } 5396 5397 #[test] 5398 fn test_net_hotplug() { 5399 _test_net_hotplug(None) 5400 } 5401 5402 #[test] 5403 fn test_net_multi_segment_hotplug() { 5404 _test_net_hotplug(Some(15)) 5405 } 5406 5407 fn _test_net_hotplug(pci_segment: Option<u16>) { 5408 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5409 let guest = Guest::new(Box::new(focal)); 5410 5411 #[cfg(target_arch = "x86_64")] 5412 let kernel_path = direct_kernel_boot_path(); 5413 #[cfg(target_arch = "aarch64")] 5414 let kernel_path = edk2_path(); 5415 5416 let api_socket = temp_api_path(&guest.tmp_dir); 5417 5418 // Boot without network 5419 let mut cmd = GuestCommand::new(&guest); 5420 5421 cmd.args(["--api-socket", &api_socket]) 5422 .args(["--cpus", "boot=1"]) 5423 .args(["--memory", "size=512M"]) 5424 .args(["--kernel", kernel_path.to_str().unwrap()]) 5425 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5426 .default_disks() 5427 .capture_output(); 5428 5429 if pci_segment.is_some() { 5430 cmd.args(["--platform", "num_pci_segments=16"]); 5431 } 5432 5433 let mut child = cmd.spawn().unwrap(); 5434 5435 thread::sleep(std::time::Duration::new(20, 0)); 5436 5437 let r = std::panic::catch_unwind(|| { 5438 // Add network 5439 let (cmd_success, cmd_output) = remote_command_w_output( 5440 &api_socket, 5441 "add-net", 5442 Some( 5443 format!( 5444 "{}{},id=test0", 5445 guest.default_net_string(), 5446 if let Some(pci_segment) = pci_segment { 5447 format!(",pci_segment={pci_segment}") 5448 } else { 5449 "".to_owned() 5450 } 5451 ) 5452 .as_str(), 5453 ), 5454 ); 5455 assert!(cmd_success); 5456 5457 if let Some(pci_segment) = pci_segment { 5458 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5459 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5460 ))); 5461 } else { 5462 assert!(String::from_utf8_lossy(&cmd_output) 5463 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5464 } 5465 5466 thread::sleep(std::time::Duration::new(5, 0)); 5467 5468 // 1 network interfaces + default localhost ==> 2 interfaces 5469 assert_eq!( 5470 guest 5471 .ssh_command("ip -o link | wc -l") 5472 .unwrap() 5473 .trim() 5474 .parse::<u32>() 5475 .unwrap_or_default(), 5476 2 5477 ); 5478 5479 // Remove network 5480 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5481 thread::sleep(std::time::Duration::new(5, 0)); 5482 5483 let (cmd_success, cmd_output) = remote_command_w_output( 5484 &api_socket, 5485 "add-net", 5486 Some( 5487 format!( 5488 "{}{},id=test1", 5489 guest.default_net_string(), 5490 if let Some(pci_segment) = pci_segment { 5491 format!(",pci_segment={pci_segment}") 5492 } else { 5493 "".to_owned() 5494 } 5495 ) 5496 .as_str(), 5497 ), 5498 ); 5499 assert!(cmd_success); 5500 5501 if let Some(pci_segment) = pci_segment { 5502 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5503 "{{\"id\":\"test1\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5504 ))); 5505 } else { 5506 assert!(String::from_utf8_lossy(&cmd_output) 5507 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5508 } 5509 5510 thread::sleep(std::time::Duration::new(5, 0)); 5511 5512 // 1 network interfaces + default localhost ==> 2 interfaces 5513 assert_eq!( 5514 guest 5515 .ssh_command("ip -o link | wc -l") 5516 .unwrap() 5517 .trim() 5518 .parse::<u32>() 5519 .unwrap_or_default(), 5520 2 5521 ); 5522 5523 guest.reboot_linux(0, None); 5524 5525 // Check still there after reboot 5526 // 1 network interfaces + default localhost ==> 2 interfaces 5527 assert_eq!( 5528 guest 5529 .ssh_command("ip -o link | wc -l") 5530 .unwrap() 5531 .trim() 5532 .parse::<u32>() 5533 .unwrap_or_default(), 5534 2 5535 ); 5536 }); 5537 5538 let _ = child.kill(); 5539 let output = child.wait_with_output().unwrap(); 5540 5541 handle_child_output(r, &output); 5542 } 5543 5544 #[test] 5545 fn test_initramfs() { 5546 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5547 let guest = Guest::new(Box::new(focal)); 5548 let mut workload_path = dirs::home_dir().unwrap(); 5549 workload_path.push("workloads"); 5550 5551 #[cfg(target_arch = "x86_64")] 5552 let mut kernels = vec![direct_kernel_boot_path()]; 5553 #[cfg(target_arch = "aarch64")] 5554 let kernels = vec![direct_kernel_boot_path()]; 5555 5556 #[cfg(target_arch = "x86_64")] 5557 { 5558 let mut pvh_kernel_path = workload_path.clone(); 5559 pvh_kernel_path.push("vmlinux"); 5560 kernels.push(pvh_kernel_path); 5561 } 5562 5563 let mut initramfs_path = workload_path; 5564 initramfs_path.push("alpine_initramfs.img"); 5565 5566 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5567 let cmdline = format!("console=hvc0 quiet TEST_STRING={test_string}"); 5568 5569 kernels.iter().for_each(|k_path| { 5570 let mut child = GuestCommand::new(&guest) 5571 .args(["--kernel", k_path.to_str().unwrap()]) 5572 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 5573 .args(["--cmdline", &cmdline]) 5574 .capture_output() 5575 .spawn() 5576 .unwrap(); 5577 5578 thread::sleep(std::time::Duration::new(20, 0)); 5579 5580 let _ = child.kill(); 5581 let output = child.wait_with_output().unwrap(); 5582 5583 let r = std::panic::catch_unwind(|| { 5584 let s = String::from_utf8_lossy(&output.stdout); 5585 5586 assert_ne!(s.lines().position(|line| line == test_string), None); 5587 }); 5588 5589 handle_child_output(r, &output); 5590 }); 5591 } 5592 5593 // One thing to note about this test. The virtio-net device is heavily used 5594 // through each ssh command. There's no need to perform a dedicated test to 5595 // verify the migration went well for virtio-net. 5596 #[test] 5597 #[cfg(not(feature = "mshv"))] 5598 fn test_snapshot_restore_hotplug_virtiomem() { 5599 _test_snapshot_restore(true); 5600 } 5601 5602 #[test] 5603 fn test_snapshot_restore_basic() { 5604 _test_snapshot_restore(false); 5605 } 5606 5607 fn _test_snapshot_restore(use_hotplug: bool) { 5608 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5609 let guest = Guest::new(Box::new(focal)); 5610 let kernel_path = direct_kernel_boot_path(); 5611 5612 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 5613 5614 let net_id = "net123"; 5615 let net_params = format!( 5616 "id={},tap=,mac={},ip={},mask=255.255.255.0", 5617 net_id, guest.network.guest_mac, guest.network.host_ip 5618 ); 5619 let mut mem_params = "size=4G"; 5620 5621 if use_hotplug { 5622 mem_params = "size=4G,hotplug_method=virtio-mem,hotplug_size=32G" 5623 } 5624 5625 let cloudinit_params = format!( 5626 "path={},iommu=on", 5627 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5628 ); 5629 5630 let socket = temp_vsock_path(&guest.tmp_dir); 5631 let event_path = temp_event_monitor_path(&guest.tmp_dir); 5632 5633 let mut child = GuestCommand::new(&guest) 5634 .args(["--api-socket", &api_socket_source]) 5635 .args(["--event-monitor", format!("path={event_path}").as_str()]) 5636 .args(["--cpus", "boot=4"]) 5637 .args(["--memory", mem_params]) 5638 .args(["--balloon", "size=0"]) 5639 .args(["--kernel", kernel_path.to_str().unwrap()]) 5640 .args([ 5641 "--disk", 5642 format!( 5643 "path={}", 5644 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5645 ) 5646 .as_str(), 5647 cloudinit_params.as_str(), 5648 ]) 5649 .args(["--net", net_params.as_str()]) 5650 .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) 5651 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5652 .capture_output() 5653 .spawn() 5654 .unwrap(); 5655 5656 let console_text = String::from("On a branch floating down river a cricket, singing."); 5657 // Create the snapshot directory 5658 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 5659 5660 let r = std::panic::catch_unwind(|| { 5661 guest.wait_vm_boot(None).unwrap(); 5662 5663 // Check the number of vCPUs 5664 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5665 // Check the guest RAM 5666 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5667 if use_hotplug { 5668 // Increase guest RAM with virtio-mem 5669 resize_command( 5670 &api_socket_source, 5671 None, 5672 Some(6 << 30), 5673 None, 5674 Some(&event_path), 5675 ); 5676 thread::sleep(std::time::Duration::new(5, 0)); 5677 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5678 // Use balloon to remove RAM from the VM 5679 resize_command( 5680 &api_socket_source, 5681 None, 5682 None, 5683 Some(1 << 30), 5684 Some(&event_path), 5685 ); 5686 thread::sleep(std::time::Duration::new(5, 0)); 5687 let total_memory = guest.get_total_memory().unwrap_or_default(); 5688 assert!(total_memory > 4_800_000); 5689 assert!(total_memory < 5_760_000); 5690 } 5691 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 5692 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5693 5694 // x86_64: We check that removing and adding back the virtio-net device 5695 // does not break the snapshot/restore support for virtio-pci. 5696 // This is an important thing to test as the hotplug will 5697 // trigger a PCI BAR reprogramming, which is a good way of 5698 // checking if the stored resources are correctly restored. 5699 // Unplug the virtio-net device 5700 // AArch64: Device hotplug is currently not supported, skipping here. 5701 #[cfg(target_arch = "x86_64")] 5702 { 5703 assert!(remote_command( 5704 &api_socket_source, 5705 "remove-device", 5706 Some(net_id), 5707 )); 5708 thread::sleep(std::time::Duration::new(10, 0)); 5709 let latest_events = [&MetaEvent { 5710 event: "device-removed".to_string(), 5711 device_id: Some(net_id.to_string()), 5712 }]; 5713 assert!(check_latest_events_exact(&latest_events, &event_path)); 5714 5715 // Plug the virtio-net device again 5716 assert!(remote_command( 5717 &api_socket_source, 5718 "add-net", 5719 Some(net_params.as_str()), 5720 )); 5721 thread::sleep(std::time::Duration::new(10, 0)); 5722 } 5723 5724 // Pause the VM 5725 assert!(remote_command(&api_socket_source, "pause", None)); 5726 let latest_events = [ 5727 &MetaEvent { 5728 event: "pausing".to_string(), 5729 device_id: None, 5730 }, 5731 &MetaEvent { 5732 event: "paused".to_string(), 5733 device_id: None, 5734 }, 5735 ]; 5736 assert!(check_latest_events_exact(&latest_events, &event_path)); 5737 5738 // Take a snapshot from the VM 5739 assert!(remote_command( 5740 &api_socket_source, 5741 "snapshot", 5742 Some(format!("file://{snapshot_dir}").as_str()), 5743 )); 5744 5745 // Wait to make sure the snapshot is completed 5746 thread::sleep(std::time::Duration::new(10, 0)); 5747 5748 let latest_events = [ 5749 &MetaEvent { 5750 event: "snapshotting".to_string(), 5751 device_id: None, 5752 }, 5753 &MetaEvent { 5754 event: "snapshotted".to_string(), 5755 device_id: None, 5756 }, 5757 ]; 5758 assert!(check_latest_events_exact(&latest_events, &event_path)); 5759 }); 5760 5761 // Shutdown the source VM and check console output 5762 let _ = child.kill(); 5763 let output = child.wait_with_output().unwrap(); 5764 handle_child_output(r, &output); 5765 5766 let r = std::panic::catch_unwind(|| { 5767 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5768 }); 5769 5770 handle_child_output(r, &output); 5771 5772 // Remove the vsock socket file. 5773 Command::new("rm") 5774 .arg("-f") 5775 .arg(socket.as_str()) 5776 .output() 5777 .unwrap(); 5778 5779 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 5780 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 5781 5782 // Restore the VM from the snapshot 5783 let mut child = GuestCommand::new(&guest) 5784 .args(["--api-socket", &api_socket_restored]) 5785 .args([ 5786 "--event-monitor", 5787 format!("path={event_path_restored}").as_str(), 5788 ]) 5789 .args([ 5790 "--restore", 5791 format!("source_url=file://{snapshot_dir}").as_str(), 5792 ]) 5793 .capture_output() 5794 .spawn() 5795 .unwrap(); 5796 5797 // Wait for the VM to be restored 5798 thread::sleep(std::time::Duration::new(10, 0)); 5799 let expected_events = [ 5800 &MetaEvent { 5801 event: "starting".to_string(), 5802 device_id: None, 5803 }, 5804 &MetaEvent { 5805 event: "activated".to_string(), 5806 device_id: Some("__console".to_string()), 5807 }, 5808 &MetaEvent { 5809 event: "activated".to_string(), 5810 device_id: Some("__rng".to_string()), 5811 }, 5812 &MetaEvent { 5813 event: "restoring".to_string(), 5814 device_id: None, 5815 }, 5816 ]; 5817 assert!(check_sequential_events( 5818 &expected_events, 5819 &event_path_restored 5820 )); 5821 let latest_events = [&MetaEvent { 5822 event: "restored".to_string(), 5823 device_id: None, 5824 }]; 5825 assert!(check_latest_events_exact( 5826 &latest_events, 5827 &event_path_restored 5828 )); 5829 5830 let r = std::panic::catch_unwind(|| { 5831 // Resume the VM 5832 assert!(remote_command(&api_socket_restored, "resume", None)); 5833 let latest_events = [ 5834 &MetaEvent { 5835 event: "resuming".to_string(), 5836 device_id: None, 5837 }, 5838 &MetaEvent { 5839 event: "resumed".to_string(), 5840 device_id: None, 5841 }, 5842 ]; 5843 assert!(check_latest_events_exact( 5844 &latest_events, 5845 &event_path_restored 5846 )); 5847 5848 // Perform same checks to validate VM has been properly restored 5849 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5850 let total_memory = guest.get_total_memory().unwrap_or_default(); 5851 if !use_hotplug { 5852 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5853 } else { 5854 assert!(total_memory > 4_800_000); 5855 assert!(total_memory < 5_760_000); 5856 // Deflate balloon to restore entire RAM to the VM 5857 resize_command(&api_socket_restored, None, None, Some(0), None); 5858 thread::sleep(std::time::Duration::new(5, 0)); 5859 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5860 // Decrease guest RAM with virtio-mem 5861 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 5862 thread::sleep(std::time::Duration::new(5, 0)); 5863 let total_memory = guest.get_total_memory().unwrap_or_default(); 5864 assert!(total_memory > 4_800_000); 5865 assert!(total_memory < 5_760_000); 5866 } 5867 5868 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5869 }); 5870 // Shutdown the target VM and check console output 5871 let _ = child.kill(); 5872 let output = child.wait_with_output().unwrap(); 5873 handle_child_output(r, &output); 5874 5875 let r = std::panic::catch_unwind(|| { 5876 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5877 }); 5878 5879 handle_child_output(r, &output); 5880 } 5881 5882 #[test] 5883 fn test_counters() { 5884 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5885 let guest = Guest::new(Box::new(focal)); 5886 let api_socket = temp_api_path(&guest.tmp_dir); 5887 5888 let mut cmd = GuestCommand::new(&guest); 5889 cmd.args(["--cpus", "boot=1"]) 5890 .args(["--memory", "size=512M"]) 5891 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5892 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5893 .default_disks() 5894 .args(["--net", guest.default_net_string().as_str()]) 5895 .args(["--api-socket", &api_socket]) 5896 .capture_output(); 5897 5898 let mut child = cmd.spawn().unwrap(); 5899 5900 let r = std::panic::catch_unwind(|| { 5901 guest.wait_vm_boot(None).unwrap(); 5902 5903 let orig_counters = get_counters(&api_socket); 5904 guest 5905 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 5906 .unwrap(); 5907 5908 let new_counters = get_counters(&api_socket); 5909 5910 // Check that all the counters have increased 5911 assert!(new_counters > orig_counters); 5912 }); 5913 5914 let _ = child.kill(); 5915 let output = child.wait_with_output().unwrap(); 5916 5917 handle_child_output(r, &output); 5918 } 5919 5920 #[test] 5921 #[cfg(feature = "guest_debug")] 5922 fn test_coredump() { 5923 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5924 let guest = Guest::new(Box::new(focal)); 5925 let api_socket = temp_api_path(&guest.tmp_dir); 5926 5927 let mut cmd = GuestCommand::new(&guest); 5928 cmd.args(["--cpus", "boot=4"]) 5929 .args(["--memory", "size=4G"]) 5930 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 5931 .default_disks() 5932 .args(["--net", guest.default_net_string().as_str()]) 5933 .args(["--api-socket", &api_socket]) 5934 .capture_output(); 5935 5936 let mut child = cmd.spawn().unwrap(); 5937 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 5938 5939 let r = std::panic::catch_unwind(|| { 5940 guest.wait_vm_boot(None).unwrap(); 5941 5942 assert!(remote_command(&api_socket, "pause", None)); 5943 5944 assert!(remote_command( 5945 &api_socket, 5946 "coredump", 5947 Some(format!("file://{vmcore_file}").as_str()), 5948 )); 5949 5950 // the num of CORE notes should equals to vcpu 5951 let readelf_core_num_cmd = 5952 format!("readelf --all {vmcore_file} |grep CORE |grep -v Type |wc -l"); 5953 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 5954 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 5955 5956 // the num of QEMU notes should equals to vcpu 5957 let readelf_vmm_num_cmd = format!("readelf --all {vmcore_file} |grep QEMU |wc -l"); 5958 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 5959 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 5960 }); 5961 5962 let _ = child.kill(); 5963 let output = child.wait_with_output().unwrap(); 5964 5965 handle_child_output(r, &output); 5966 } 5967 5968 #[test] 5969 fn test_watchdog() { 5970 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5971 let guest = Guest::new(Box::new(focal)); 5972 let api_socket = temp_api_path(&guest.tmp_dir); 5973 5974 let kernel_path = direct_kernel_boot_path(); 5975 5976 let mut cmd = GuestCommand::new(&guest); 5977 cmd.args(["--cpus", "boot=1"]) 5978 .args(["--memory", "size=512M"]) 5979 .args(["--kernel", kernel_path.to_str().unwrap()]) 5980 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5981 .default_disks() 5982 .args(["--net", guest.default_net_string().as_str()]) 5983 .args(["--watchdog"]) 5984 .args(["--api-socket", &api_socket]) 5985 .capture_output(); 5986 5987 let mut child = cmd.spawn().unwrap(); 5988 5989 let r = std::panic::catch_unwind(|| { 5990 guest.wait_vm_boot(None).unwrap(); 5991 5992 let mut expected_reboot_count = 1; 5993 5994 // Enable the watchdog with a 15s timeout 5995 enable_guest_watchdog(&guest, 15); 5996 5997 // Reboot and check that systemd has activated the watchdog 5998 guest.ssh_command("sudo reboot").unwrap(); 5999 guest.wait_vm_boot(None).unwrap(); 6000 expected_reboot_count += 1; 6001 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6002 assert_eq!( 6003 guest 6004 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6005 .unwrap() 6006 .trim() 6007 .parse::<u32>() 6008 .unwrap_or_default(), 6009 2 6010 ); 6011 6012 // Allow some normal time to elapse to check we don't get spurious reboots 6013 thread::sleep(std::time::Duration::new(40, 0)); 6014 // Check no reboot 6015 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6016 6017 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6018 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6019 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6020 guest.wait_vm_boot(Some(50)).unwrap(); 6021 // Check a reboot is triggerred by the watchdog 6022 expected_reboot_count += 1; 6023 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6024 6025 #[cfg(target_arch = "x86_64")] 6026 { 6027 // Now pause the VM and remain offline for 30s 6028 assert!(remote_command(&api_socket, "pause", None)); 6029 thread::sleep(std::time::Duration::new(30, 0)); 6030 assert!(remote_command(&api_socket, "resume", None)); 6031 6032 // Check no reboot 6033 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6034 } 6035 }); 6036 6037 let _ = child.kill(); 6038 let output = child.wait_with_output().unwrap(); 6039 6040 handle_child_output(r, &output); 6041 } 6042 6043 #[test] 6044 fn test_tap_from_fd() { 6045 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6046 let guest = Guest::new(Box::new(focal)); 6047 let kernel_path = direct_kernel_boot_path(); 6048 6049 // Create a TAP interface with multi-queue enabled 6050 let num_queue_pairs: usize = 2; 6051 6052 use std::str::FromStr; 6053 let taps = net_util::open_tap( 6054 Some("chtap0"), 6055 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6056 None, 6057 &mut None, 6058 None, 6059 num_queue_pairs, 6060 Some(libc::O_RDWR | libc::O_NONBLOCK), 6061 ) 6062 .unwrap(); 6063 6064 let mut child = GuestCommand::new(&guest) 6065 .args(["--cpus", &format!("boot={num_queue_pairs}")]) 6066 .args(["--memory", "size=512M"]) 6067 .args(["--kernel", kernel_path.to_str().unwrap()]) 6068 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6069 .default_disks() 6070 .args([ 6071 "--net", 6072 &format!( 6073 "fd=[{},{}],mac={},num_queues={}", 6074 taps[0].as_raw_fd(), 6075 taps[1].as_raw_fd(), 6076 guest.network.guest_mac, 6077 num_queue_pairs * 2 6078 ), 6079 ]) 6080 .capture_output() 6081 .spawn() 6082 .unwrap(); 6083 6084 let r = std::panic::catch_unwind(|| { 6085 guest.wait_vm_boot(None).unwrap(); 6086 6087 assert_eq!( 6088 guest 6089 .ssh_command("ip -o link | wc -l") 6090 .unwrap() 6091 .trim() 6092 .parse::<u32>() 6093 .unwrap_or_default(), 6094 2 6095 ); 6096 6097 guest.reboot_linux(0, None); 6098 6099 assert_eq!( 6100 guest 6101 .ssh_command("ip -o link | wc -l") 6102 .unwrap() 6103 .trim() 6104 .parse::<u32>() 6105 .unwrap_or_default(), 6106 2 6107 ); 6108 }); 6109 6110 let _ = child.kill(); 6111 let output = child.wait_with_output().unwrap(); 6112 6113 handle_child_output(r, &output); 6114 } 6115 6116 // By design, a guest VM won't be able to connect to the host 6117 // machine when using a macvtap network interface (while it can 6118 // communicate externally). As a workaround, this integration 6119 // test creates two macvtap interfaces in 'bridge' mode on the 6120 // same physical net interface, one for the guest and one for 6121 // the host. With additional setup on the IP address and the 6122 // routing table, it enables the communications between the 6123 // guest VM and the host machine. 6124 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6125 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6126 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6127 let guest = Guest::new(Box::new(focal)); 6128 let api_socket = temp_api_path(&guest.tmp_dir); 6129 6130 #[cfg(target_arch = "x86_64")] 6131 let kernel_path = direct_kernel_boot_path(); 6132 #[cfg(target_arch = "aarch64")] 6133 let kernel_path = edk2_path(); 6134 6135 let phy_net = "eth0"; 6136 6137 // Create a macvtap interface for the guest VM to use 6138 assert!(exec_host_command_status(&format!( 6139 "sudo ip link add link {phy_net} name {guest_macvtap_name} type macvtap mod bridge" 6140 )) 6141 .success()); 6142 assert!(exec_host_command_status(&format!( 6143 "sudo ip link set {} address {} up", 6144 guest_macvtap_name, guest.network.guest_mac 6145 )) 6146 .success()); 6147 assert!( 6148 exec_host_command_status(&format!("sudo ip link show {guest_macvtap_name}")).success() 6149 ); 6150 6151 let tap_index = 6152 fs::read_to_string(format!("/sys/class/net/{guest_macvtap_name}/ifindex")).unwrap(); 6153 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6154 6155 assert!(exec_host_command_status(&format!("sudo chown $UID.$UID {tap_device}")).success()); 6156 6157 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6158 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6159 assert!(tap_fd1 > 0); 6160 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6161 assert!(tap_fd2 > 0); 6162 6163 // Create a macvtap on the same physical net interface for 6164 // the host machine to use 6165 assert!(exec_host_command_status(&format!( 6166 "sudo ip link add link {phy_net} name {host_macvtap_name} type macvtap mod bridge" 6167 )) 6168 .success()); 6169 // Use default mask "255.255.255.0" 6170 assert!(exec_host_command_status(&format!( 6171 "sudo ip address add {}/24 dev {}", 6172 guest.network.host_ip, host_macvtap_name 6173 )) 6174 .success()); 6175 assert!( 6176 exec_host_command_status(&format!("sudo ip link set dev {host_macvtap_name} up")) 6177 .success() 6178 ); 6179 6180 let mut guest_command = GuestCommand::new(&guest); 6181 guest_command 6182 .args(["--cpus", "boot=2"]) 6183 .args(["--memory", "size=512M"]) 6184 .args(["--kernel", kernel_path.to_str().unwrap()]) 6185 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6186 .default_disks() 6187 .args(["--api-socket", &api_socket]); 6188 6189 let net_params = format!( 6190 "fd=[{},{}],mac={},num_queues=4", 6191 tap_fd1, tap_fd2, guest.network.guest_mac 6192 ); 6193 6194 if !hotplug { 6195 guest_command.args(["--net", &net_params]); 6196 } 6197 6198 let mut child = guest_command.capture_output().spawn().unwrap(); 6199 6200 if hotplug { 6201 // Give some time to the VMM process to listen to the API 6202 // socket. This is the only requirement to avoid the following 6203 // call to ch-remote from failing. 6204 thread::sleep(std::time::Duration::new(10, 0)); 6205 // Hotplug the virtio-net device 6206 let (cmd_success, cmd_output) = 6207 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6208 assert!(cmd_success); 6209 #[cfg(target_arch = "x86_64")] 6210 assert!(String::from_utf8_lossy(&cmd_output) 6211 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6212 #[cfg(target_arch = "aarch64")] 6213 assert!(String::from_utf8_lossy(&cmd_output) 6214 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6215 } 6216 6217 // The functional connectivity provided by the virtio-net device 6218 // gets tested through wait_vm_boot() as it expects to receive a 6219 // HTTP request, and through the SSH command as well. 6220 let r = std::panic::catch_unwind(|| { 6221 guest.wait_vm_boot(None).unwrap(); 6222 6223 assert_eq!( 6224 guest 6225 .ssh_command("ip -o link | wc -l") 6226 .unwrap() 6227 .trim() 6228 .parse::<u32>() 6229 .unwrap_or_default(), 6230 2 6231 ); 6232 }); 6233 6234 let _ = child.kill(); 6235 6236 exec_host_command_status(&format!("sudo ip link del {guest_macvtap_name}")); 6237 exec_host_command_status(&format!("sudo ip link del {host_macvtap_name}")); 6238 6239 let output = child.wait_with_output().unwrap(); 6240 6241 handle_child_output(r, &output); 6242 } 6243 6244 #[test] 6245 fn test_macvtap() { 6246 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6247 } 6248 6249 #[test] 6250 fn test_macvtap_hotplug() { 6251 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6252 } 6253 6254 #[test] 6255 #[cfg(not(feature = "mshv"))] 6256 fn test_ovs_dpdk() { 6257 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6258 let guest1 = Guest::new(Box::new(focal1)); 6259 6260 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6261 let guest2 = Guest::new(Box::new(focal2)); 6262 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6263 6264 let (mut child1, mut child2) = 6265 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6266 6267 // Create the snapshot directory 6268 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6269 6270 let r = std::panic::catch_unwind(|| { 6271 // Remove one of the two ports from the OVS bridge 6272 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6273 6274 // Spawn a new netcat listener in the first VM 6275 let guest_ip = guest1.network.guest_ip.clone(); 6276 thread::spawn(move || { 6277 ssh_command_ip( 6278 "nc -l 12345", 6279 &guest_ip, 6280 DEFAULT_SSH_RETRIES, 6281 DEFAULT_SSH_TIMEOUT, 6282 ) 6283 .unwrap(); 6284 }); 6285 6286 // Wait for the server to be listening 6287 thread::sleep(std::time::Duration::new(5, 0)); 6288 6289 // Check the connection fails this time 6290 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6291 6292 // Add the OVS port back 6293 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6294 6295 // And finally check the connection is functional again 6296 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6297 6298 // Pause the VM 6299 assert!(remote_command(&api_socket_source, "pause", None)); 6300 6301 // Take a snapshot from the VM 6302 assert!(remote_command( 6303 &api_socket_source, 6304 "snapshot", 6305 Some(format!("file://{snapshot_dir}").as_str()), 6306 )); 6307 6308 // Wait to make sure the snapshot is completed 6309 thread::sleep(std::time::Duration::new(10, 0)); 6310 }); 6311 6312 // Shutdown the source VM 6313 let _ = child2.kill(); 6314 let output = child2.wait_with_output().unwrap(); 6315 handle_child_output(r, &output); 6316 6317 // Remove the vhost-user socket file. 6318 Command::new("rm") 6319 .arg("-f") 6320 .arg("/tmp/dpdkvhostclient2") 6321 .output() 6322 .unwrap(); 6323 6324 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6325 // Restore the VM from the snapshot 6326 let mut child2 = GuestCommand::new(&guest2) 6327 .args(["--api-socket", &api_socket_restored]) 6328 .args([ 6329 "--restore", 6330 format!("source_url=file://{snapshot_dir}").as_str(), 6331 ]) 6332 .capture_output() 6333 .spawn() 6334 .unwrap(); 6335 6336 // Wait for the VM to be restored 6337 thread::sleep(std::time::Duration::new(10, 0)); 6338 6339 let r = std::panic::catch_unwind(|| { 6340 // Resume the VM 6341 assert!(remote_command(&api_socket_restored, "resume", None)); 6342 6343 // Spawn a new netcat listener in the first VM 6344 let guest_ip = guest1.network.guest_ip.clone(); 6345 thread::spawn(move || { 6346 ssh_command_ip( 6347 "nc -l 12345", 6348 &guest_ip, 6349 DEFAULT_SSH_RETRIES, 6350 DEFAULT_SSH_TIMEOUT, 6351 ) 6352 .unwrap(); 6353 }); 6354 6355 // Wait for the server to be listening 6356 thread::sleep(std::time::Duration::new(5, 0)); 6357 6358 // And check the connection is still functional after restore 6359 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6360 }); 6361 6362 let _ = child1.kill(); 6363 let _ = child2.kill(); 6364 6365 let output = child1.wait_with_output().unwrap(); 6366 child2.wait().unwrap(); 6367 6368 cleanup_ovs_dpdk(); 6369 6370 handle_child_output(r, &output); 6371 } 6372 6373 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6374 cleanup_spdk_nvme(); 6375 6376 assert!(exec_host_command_status(&format!( 6377 "mkdir -p {}", 6378 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6379 )) 6380 .success()); 6381 assert!(exec_host_command_status(&format!( 6382 "truncate {} -s 128M", 6383 nvme_dir.join("test-disk.raw").to_str().unwrap() 6384 )) 6385 .success()); 6386 assert!(exec_host_command_status(&format!( 6387 "mkfs.ext4 {}", 6388 nvme_dir.join("test-disk.raw").to_str().unwrap() 6389 )) 6390 .success()); 6391 6392 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6393 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6394 .args(["-i", "0", "-m", "0x1"]) 6395 .spawn() 6396 .unwrap(); 6397 thread::sleep(std::time::Duration::new(2, 0)); 6398 6399 assert!(exec_host_command_status( 6400 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER" 6401 ) 6402 .success()); 6403 assert!(exec_host_command_status(&format!( 6404 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6405 nvme_dir.join("test-disk.raw").to_str().unwrap() 6406 )) 6407 .success()); 6408 assert!(exec_host_command_status( 6409 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6410 ) 6411 .success()); 6412 assert!(exec_host_command_status( 6413 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6414 ) 6415 .success()); 6416 assert!(exec_host_command_status(&format!( 6417 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6418 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6419 )) 6420 .success()); 6421 } 6422 6423 fn cleanup_spdk_nvme() { 6424 exec_host_command_status("pkill -f nvmf_tgt"); 6425 } 6426 6427 #[test] 6428 fn test_vfio_user() { 6429 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6430 let jammy = UbuntuDiskConfig::new(jammy_image); 6431 let guest = Guest::new(Box::new(jammy)); 6432 6433 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6434 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6435 6436 let api_socket = temp_api_path(&guest.tmp_dir); 6437 let mut child = GuestCommand::new(&guest) 6438 .args(["--api-socket", &api_socket]) 6439 .args(["--cpus", "boot=1"]) 6440 .args(["--memory", "size=512M,shared=on,hugepages=on"]) 6441 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6442 .args(["--serial", "tty", "--console", "off"]) 6443 .default_disks() 6444 .default_net() 6445 .capture_output() 6446 .spawn() 6447 .unwrap(); 6448 6449 let r = std::panic::catch_unwind(|| { 6450 guest.wait_vm_boot(None).unwrap(); 6451 6452 // Hotplug the SPDK-NVMe device to the VM 6453 let (cmd_success, cmd_output) = remote_command_w_output( 6454 &api_socket, 6455 "add-user-device", 6456 Some(&format!( 6457 "socket={},id=vfio_user0", 6458 spdk_nvme_dir 6459 .as_path() 6460 .join("nvme-vfio-user/cntrl") 6461 .to_str() 6462 .unwrap(), 6463 )), 6464 ); 6465 assert!(cmd_success); 6466 assert!(String::from_utf8_lossy(&cmd_output) 6467 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6468 6469 thread::sleep(std::time::Duration::new(10, 0)); 6470 6471 // Check both if /dev/nvme exists and if the block size is 128M. 6472 assert_eq!( 6473 guest 6474 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6475 .unwrap() 6476 .trim() 6477 .parse::<u32>() 6478 .unwrap_or_default(), 6479 1 6480 ); 6481 6482 // Check changes persist after reboot 6483 assert_eq!( 6484 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6485 "" 6486 ); 6487 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6488 guest 6489 .ssh_command("echo test123 | sudo tee /mnt/test") 6490 .unwrap(); 6491 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6492 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6493 6494 guest.reboot_linux(0, None); 6495 assert_eq!( 6496 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6497 "" 6498 ); 6499 assert_eq!( 6500 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6501 "test123" 6502 ); 6503 }); 6504 6505 cleanup_spdk_nvme(); 6506 6507 let _ = child.kill(); 6508 let output = child.wait_with_output().unwrap(); 6509 6510 handle_child_output(r, &output); 6511 } 6512 6513 #[test] 6514 #[cfg(target_arch = "x86_64")] 6515 fn test_vdpa_block() { 6516 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6517 if !exec_host_command_status("lsmod | grep vdpa_sim_blk").success() { 6518 return; 6519 } 6520 6521 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6522 let guest = Guest::new(Box::new(focal)); 6523 let api_socket = temp_api_path(&guest.tmp_dir); 6524 6525 let kernel_path = direct_kernel_boot_path(); 6526 6527 let mut child = GuestCommand::new(&guest) 6528 .args(["--cpus", "boot=2"]) 6529 .args(["--memory", "size=512M,hugepages=on"]) 6530 .args(["--kernel", kernel_path.to_str().unwrap()]) 6531 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6532 .default_disks() 6533 .default_net() 6534 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6535 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 6536 .args(["--api-socket", &api_socket]) 6537 .capture_output() 6538 .spawn() 6539 .unwrap(); 6540 6541 let r = std::panic::catch_unwind(|| { 6542 guest.wait_vm_boot(None).unwrap(); 6543 6544 // Check both if /dev/vdc exists and if the block size is 128M. 6545 assert_eq!( 6546 guest 6547 .ssh_command("lsblk | grep vdc | grep -c 128M") 6548 .unwrap() 6549 .trim() 6550 .parse::<u32>() 6551 .unwrap_or_default(), 6552 1 6553 ); 6554 6555 // Check the content of the block device after we wrote to it. 6556 // The vpda-sim-blk should let us read what we previously wrote. 6557 guest 6558 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6559 .unwrap(); 6560 assert_eq!( 6561 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6562 "foobar" 6563 ); 6564 6565 // Hotplug an extra vDPA block device behind the vIOMMU 6566 // Add a new vDPA device to the VM 6567 let (cmd_success, cmd_output) = remote_command_w_output( 6568 &api_socket, 6569 "add-vdpa", 6570 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6571 ); 6572 assert!(cmd_success); 6573 assert!(String::from_utf8_lossy(&cmd_output) 6574 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6575 6576 thread::sleep(std::time::Duration::new(10, 0)); 6577 6578 // Check IOMMU setup 6579 assert!(guest 6580 .does_device_vendor_pair_match("0x1057", "0x1af4") 6581 .unwrap_or_default()); 6582 assert_eq!( 6583 guest 6584 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6585 .unwrap() 6586 .trim(), 6587 "0001:00:01.0" 6588 ); 6589 6590 // Check both if /dev/vdd exists and if the block size is 128M. 6591 assert_eq!( 6592 guest 6593 .ssh_command("lsblk | grep vdd | grep -c 128M") 6594 .unwrap() 6595 .trim() 6596 .parse::<u32>() 6597 .unwrap_or_default(), 6598 1 6599 ); 6600 6601 // Write some content to the block device we've just plugged. 6602 guest 6603 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6604 .unwrap(); 6605 6606 // Check we can read the content back. 6607 assert_eq!( 6608 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6609 "foobar" 6610 ); 6611 6612 // Unplug the device 6613 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6614 assert!(cmd_success); 6615 thread::sleep(std::time::Duration::new(10, 0)); 6616 6617 // Check /dev/vdd doesn't exist anymore 6618 assert_eq!( 6619 guest 6620 .ssh_command("lsblk | grep -c vdd || true") 6621 .unwrap() 6622 .trim() 6623 .parse::<u32>() 6624 .unwrap_or(1), 6625 0 6626 ); 6627 }); 6628 6629 let _ = child.kill(); 6630 let output = child.wait_with_output().unwrap(); 6631 6632 handle_child_output(r, &output); 6633 } 6634 6635 #[test] 6636 #[cfg(target_arch = "x86_64")] 6637 fn test_vdpa_net() { 6638 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6639 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6640 return; 6641 } 6642 6643 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6644 let guest = Guest::new(Box::new(focal)); 6645 6646 let kernel_path = direct_kernel_boot_path(); 6647 6648 let mut child = GuestCommand::new(&guest) 6649 .args(["--cpus", "boot=2"]) 6650 .args(["--memory", "size=512M,hugepages=on"]) 6651 .args(["--kernel", kernel_path.to_str().unwrap()]) 6652 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6653 .default_disks() 6654 .default_net() 6655 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 6656 .capture_output() 6657 .spawn() 6658 .unwrap(); 6659 6660 let r = std::panic::catch_unwind(|| { 6661 guest.wait_vm_boot(None).unwrap(); 6662 6663 // Check we can find network interface related to vDPA device 6664 assert_eq!( 6665 guest 6666 .ssh_command("ip -o link | grep -c ens6") 6667 .unwrap() 6668 .trim() 6669 .parse::<u32>() 6670 .unwrap_or(0), 6671 1 6672 ); 6673 6674 guest 6675 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 6676 .unwrap(); 6677 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 6678 6679 // Check there is no packet yet on both TX/RX of the network interface 6680 assert_eq!( 6681 guest 6682 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 6683 .unwrap() 6684 .trim() 6685 .parse::<u32>() 6686 .unwrap_or(0), 6687 2 6688 ); 6689 6690 // Send 6 packets with ping command 6691 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 6692 6693 // Check we can find 6 packets on both TX/RX of the network interface 6694 assert_eq!( 6695 guest 6696 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 6697 .unwrap() 6698 .trim() 6699 .parse::<u32>() 6700 .unwrap_or(0), 6701 2 6702 ); 6703 6704 // No need to check for hotplug as we already tested it through 6705 // test_vdpa_block() 6706 }); 6707 6708 let _ = child.kill(); 6709 let output = child.wait_with_output().unwrap(); 6710 6711 handle_child_output(r, &output); 6712 } 6713 } 6714 6715 mod common_sequential { 6716 use crate::*; 6717 6718 #[test] 6719 fn test_memory_mergeable_on() { 6720 test_memory_mergeable(true) 6721 } 6722 } 6723 6724 mod windows { 6725 use crate::*; 6726 use once_cell::sync::Lazy; 6727 6728 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 6729 6730 struct WindowsGuest { 6731 guest: Guest, 6732 auth: PasswordAuth, 6733 } 6734 6735 trait FsType { 6736 const FS_FAT: u8; 6737 const FS_NTFS: u8; 6738 } 6739 impl FsType for WindowsGuest { 6740 const FS_FAT: u8 = 0; 6741 const FS_NTFS: u8 = 1; 6742 } 6743 6744 impl WindowsGuest { 6745 fn new() -> Self { 6746 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 6747 let guest = Guest::new(Box::new(disk)); 6748 let auth = PasswordAuth { 6749 username: String::from("administrator"), 6750 password: String::from("Admin123"), 6751 }; 6752 6753 WindowsGuest { guest, auth } 6754 } 6755 6756 fn guest(&self) -> &Guest { 6757 &self.guest 6758 } 6759 6760 fn ssh_cmd(&self, cmd: &str) -> String { 6761 ssh_command_ip_with_auth( 6762 cmd, 6763 &self.auth, 6764 &self.guest.network.guest_ip, 6765 DEFAULT_SSH_RETRIES, 6766 DEFAULT_SSH_TIMEOUT, 6767 ) 6768 .unwrap() 6769 } 6770 6771 fn cpu_count(&self) -> u8 { 6772 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 6773 .trim() 6774 .parse::<u8>() 6775 .unwrap_or(0) 6776 } 6777 6778 fn ram_size(&self) -> usize { 6779 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 6780 .trim() 6781 .parse::<usize>() 6782 .unwrap_or(0) 6783 } 6784 6785 fn netdev_count(&self) -> u8 { 6786 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6787 .trim() 6788 .parse::<u8>() 6789 .unwrap_or(0) 6790 } 6791 6792 fn disk_count(&self) -> u8 { 6793 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6794 .trim() 6795 .parse::<u8>() 6796 .unwrap_or(0) 6797 } 6798 6799 fn reboot(&self) { 6800 let _ = self.ssh_cmd("shutdown /r /t 0"); 6801 } 6802 6803 fn shutdown(&self) { 6804 let _ = self.ssh_cmd("shutdown /s /t 0"); 6805 } 6806 6807 fn run_dnsmasq(&self) -> std::process::Child { 6808 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 6809 let dhcp_host = format!( 6810 "--dhcp-host={},{}", 6811 self.guest.network.guest_mac, self.guest.network.guest_ip 6812 ); 6813 let dhcp_range = format!( 6814 "--dhcp-range=eth,{},{}", 6815 self.guest.network.guest_ip, self.guest.network.guest_ip 6816 ); 6817 6818 Command::new("dnsmasq") 6819 .arg("--no-daemon") 6820 .arg("--log-queries") 6821 .arg(listen_address.as_str()) 6822 .arg("--except-interface=lo") 6823 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 6824 .arg("--conf-file=/dev/null") 6825 .arg(dhcp_host.as_str()) 6826 .arg(dhcp_range.as_str()) 6827 .spawn() 6828 .unwrap() 6829 } 6830 6831 // TODO Cleanup image file explicitly after test, if there's some space issues. 6832 fn disk_new(&self, fs: u8, sz: usize) -> String { 6833 let mut guard = NEXT_DISK_ID.lock().unwrap(); 6834 let id = *guard; 6835 *guard = id + 1; 6836 6837 let img = PathBuf::from(format!("/tmp/test-hotplug-{id}.raw")); 6838 let _ = fs::remove_file(&img); 6839 6840 // Create an image file 6841 let out = Command::new("qemu-img") 6842 .args([ 6843 "create", 6844 "-f", 6845 "raw", 6846 img.to_str().unwrap(), 6847 format!("{sz}m").as_str(), 6848 ]) 6849 .output() 6850 .expect("qemu-img command failed") 6851 .stdout; 6852 println!("{out:?}"); 6853 6854 // Associate image to a loop device 6855 let out = Command::new("losetup") 6856 .args(["--show", "-f", img.to_str().unwrap()]) 6857 .output() 6858 .expect("failed to create loop device") 6859 .stdout; 6860 let _tmp = String::from_utf8_lossy(&out); 6861 let loop_dev = _tmp.trim(); 6862 println!("{out:?}"); 6863 6864 // Create a partition table 6865 // echo 'type=7' | sudo sfdisk "${LOOP}" 6866 let mut child = Command::new("sfdisk") 6867 .args([loop_dev]) 6868 .stdin(Stdio::piped()) 6869 .spawn() 6870 .unwrap(); 6871 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 6872 stdin 6873 .write_all("type=7".as_bytes()) 6874 .expect("failed to write stdin"); 6875 let out = child.wait_with_output().expect("sfdisk failed").stdout; 6876 println!("{out:?}"); 6877 6878 // Disengage the loop device 6879 let out = Command::new("losetup") 6880 .args(["-d", loop_dev]) 6881 .output() 6882 .expect("loop device not found") 6883 .stdout; 6884 println!("{out:?}"); 6885 6886 // Re-associate loop device pointing to the partition only 6887 let out = Command::new("losetup") 6888 .args([ 6889 "--show", 6890 "--offset", 6891 (512 * 2048).to_string().as_str(), 6892 "-f", 6893 img.to_str().unwrap(), 6894 ]) 6895 .output() 6896 .expect("failed to create loop device") 6897 .stdout; 6898 let _tmp = String::from_utf8_lossy(&out); 6899 let loop_dev = _tmp.trim(); 6900 println!("{out:?}"); 6901 6902 // Create filesystem. 6903 let fs_cmd = match fs { 6904 WindowsGuest::FS_FAT => "mkfs.msdos", 6905 WindowsGuest::FS_NTFS => "mkfs.ntfs", 6906 _ => panic!("Unknown filesystem type '{fs}'"), 6907 }; 6908 let out = Command::new(fs_cmd) 6909 .args([&loop_dev]) 6910 .output() 6911 .unwrap_or_else(|_| panic!("{fs_cmd} failed")) 6912 .stdout; 6913 println!("{out:?}"); 6914 6915 // Disengage the loop device 6916 let out = Command::new("losetup") 6917 .args(["-d", loop_dev]) 6918 .output() 6919 .unwrap_or_else(|_| panic!("loop device '{loop_dev}' not found")) 6920 .stdout; 6921 println!("{out:?}"); 6922 6923 img.to_str().unwrap().to_string() 6924 } 6925 6926 fn disks_set_rw(&self) { 6927 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 6928 } 6929 6930 fn disks_online(&self) { 6931 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 6932 } 6933 6934 fn disk_file_put(&self, fname: &str, data: &str) { 6935 let _ = self.ssh_cmd(&format!( 6936 "powershell -Command \"'{data}' | Set-Content -Path {fname}\"" 6937 )); 6938 } 6939 6940 fn disk_file_read(&self, fname: &str) -> String { 6941 self.ssh_cmd(&format!( 6942 "powershell -Command \"Get-Content -Path {fname}\"" 6943 )) 6944 } 6945 6946 fn wait_for_boot(&self) -> bool { 6947 let cmd = "dir /b c:\\ | find \"Windows\""; 6948 let tmo_max = 180; 6949 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 6950 // interval must be small. 6951 let tmo_int = 2; 6952 let out = ssh_command_ip_with_auth( 6953 cmd, 6954 &self.auth, 6955 &self.guest.network.guest_ip, 6956 { 6957 let mut ret = 1; 6958 let mut tmo_acc = 0; 6959 loop { 6960 tmo_acc += tmo_int * ret; 6961 if tmo_acc >= tmo_max { 6962 break; 6963 } 6964 ret += 1; 6965 } 6966 ret 6967 }, 6968 tmo_int, 6969 ) 6970 .unwrap(); 6971 6972 if "Windows" == out.trim() { 6973 return true; 6974 } 6975 6976 false 6977 } 6978 } 6979 6980 fn vcpu_threads_count(pid: u32) -> u8 { 6981 // ps -T -p 12345 | grep vcpu | wc -l 6982 let out = Command::new("ps") 6983 .args(["-T", "-p", format!("{pid}").as_str()]) 6984 .output() 6985 .expect("ps command failed") 6986 .stdout; 6987 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 6988 } 6989 6990 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 6991 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 6992 let out = Command::new("ps") 6993 .args(["-T", "-p", format!("{pid}").as_str()]) 6994 .output() 6995 .expect("ps command failed") 6996 .stdout; 6997 let mut n = 0; 6998 String::from_utf8_lossy(&out) 6999 .split_whitespace() 7000 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 7001 n 7002 } 7003 7004 fn disk_ctrl_threads_count(pid: u32) -> u8 { 7005 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 7006 let out = Command::new("ps") 7007 .args(["-T", "-p", format!("{pid}").as_str()]) 7008 .output() 7009 .expect("ps command failed") 7010 .stdout; 7011 let mut n = 0; 7012 String::from_utf8_lossy(&out) 7013 .split_whitespace() 7014 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 7015 n 7016 } 7017 7018 #[test] 7019 fn test_windows_guest() { 7020 let windows_guest = WindowsGuest::new(); 7021 7022 let mut child = GuestCommand::new(windows_guest.guest()) 7023 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7024 .args(["--memory", "size=4G"]) 7025 .args(["--kernel", edk2_path().to_str().unwrap()]) 7026 .args(["--serial", "tty"]) 7027 .args(["--console", "off"]) 7028 .default_disks() 7029 .default_net() 7030 .capture_output() 7031 .spawn() 7032 .unwrap(); 7033 7034 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7035 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7036 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7037 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7038 7039 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7040 7041 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7042 7043 let r = std::panic::catch_unwind(|| { 7044 // Wait to make sure Windows boots up 7045 assert!(windows_guest.wait_for_boot()); 7046 7047 windows_guest.shutdown(); 7048 }); 7049 7050 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7051 let _ = child.kill(); 7052 let output = child.wait_with_output().unwrap(); 7053 7054 let _ = child_dnsmasq.kill(); 7055 let _ = child_dnsmasq.wait(); 7056 7057 handle_child_output(r, &output); 7058 } 7059 7060 #[test] 7061 fn test_windows_guest_multiple_queues() { 7062 let windows_guest = WindowsGuest::new(); 7063 7064 let mut ovmf_path = dirs::home_dir().unwrap(); 7065 ovmf_path.push("workloads"); 7066 ovmf_path.push(OVMF_NAME); 7067 7068 let mut child = GuestCommand::new(windows_guest.guest()) 7069 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 7070 .args(["--memory", "size=4G"]) 7071 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7072 .args(["--serial", "tty"]) 7073 .args(["--console", "off"]) 7074 .args([ 7075 "--disk", 7076 format!( 7077 "path={},num_queues=4", 7078 windows_guest 7079 .guest() 7080 .disk_config 7081 .disk(DiskType::OperatingSystem) 7082 .unwrap() 7083 ) 7084 .as_str(), 7085 ]) 7086 .args([ 7087 "--net", 7088 format!( 7089 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 7090 windows_guest.guest().network.guest_mac, 7091 windows_guest.guest().network.host_ip 7092 ) 7093 .as_str(), 7094 ]) 7095 .capture_output() 7096 .spawn() 7097 .unwrap(); 7098 7099 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7100 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7101 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7102 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7103 7104 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7105 7106 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7107 7108 let r = std::panic::catch_unwind(|| { 7109 // Wait to make sure Windows boots up 7110 assert!(windows_guest.wait_for_boot()); 7111 7112 windows_guest.shutdown(); 7113 }); 7114 7115 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7116 let _ = child.kill(); 7117 let output = child.wait_with_output().unwrap(); 7118 7119 let _ = child_dnsmasq.kill(); 7120 let _ = child_dnsmasq.wait(); 7121 7122 handle_child_output(r, &output); 7123 } 7124 7125 #[test] 7126 #[cfg(not(feature = "mshv"))] 7127 #[ignore = "See #4327"] 7128 fn test_windows_guest_snapshot_restore() { 7129 let windows_guest = WindowsGuest::new(); 7130 7131 let mut ovmf_path = dirs::home_dir().unwrap(); 7132 ovmf_path.push("workloads"); 7133 ovmf_path.push(OVMF_NAME); 7134 7135 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7136 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 7137 7138 let mut child = GuestCommand::new(windows_guest.guest()) 7139 .args(["--api-socket", &api_socket_source]) 7140 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7141 .args(["--memory", "size=4G"]) 7142 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7143 .args(["--serial", "tty"]) 7144 .args(["--console", "off"]) 7145 .default_disks() 7146 .default_net() 7147 .capture_output() 7148 .spawn() 7149 .unwrap(); 7150 7151 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7152 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7153 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7154 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7155 7156 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7157 7158 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7159 7160 // Wait to make sure Windows boots up 7161 assert!(windows_guest.wait_for_boot()); 7162 7163 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 7164 7165 // Pause the VM 7166 assert!(remote_command(&api_socket_source, "pause", None)); 7167 7168 // Take a snapshot from the VM 7169 assert!(remote_command( 7170 &api_socket_source, 7171 "snapshot", 7172 Some(format!("file://{snapshot_dir}").as_str()), 7173 )); 7174 7175 // Wait to make sure the snapshot is completed 7176 thread::sleep(std::time::Duration::new(30, 0)); 7177 7178 let _ = child.kill(); 7179 child.wait().unwrap(); 7180 7181 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 7182 7183 // Restore the VM from the snapshot 7184 let mut child = GuestCommand::new(windows_guest.guest()) 7185 .args(["--api-socket", &api_socket_restored]) 7186 .args([ 7187 "--restore", 7188 format!("source_url=file://{snapshot_dir}").as_str(), 7189 ]) 7190 .capture_output() 7191 .spawn() 7192 .unwrap(); 7193 7194 // Wait for the VM to be restored 7195 thread::sleep(std::time::Duration::new(20, 0)); 7196 7197 let r = std::panic::catch_unwind(|| { 7198 // Resume the VM 7199 assert!(remote_command(&api_socket_restored, "resume", None)); 7200 7201 windows_guest.shutdown(); 7202 }); 7203 7204 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7205 let _ = child.kill(); 7206 let output = child.wait_with_output().unwrap(); 7207 7208 let _ = child_dnsmasq.kill(); 7209 let _ = child_dnsmasq.wait(); 7210 7211 handle_child_output(r, &output); 7212 } 7213 7214 #[test] 7215 #[cfg(not(feature = "mshv"))] 7216 #[cfg(not(target_arch = "aarch64"))] 7217 fn test_windows_guest_cpu_hotplug() { 7218 let windows_guest = WindowsGuest::new(); 7219 7220 let mut ovmf_path = dirs::home_dir().unwrap(); 7221 ovmf_path.push("workloads"); 7222 ovmf_path.push(OVMF_NAME); 7223 7224 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7225 let api_socket = temp_api_path(&tmp_dir); 7226 7227 let mut child = GuestCommand::new(windows_guest.guest()) 7228 .args(["--api-socket", &api_socket]) 7229 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 7230 .args(["--memory", "size=4G"]) 7231 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7232 .args(["--serial", "tty"]) 7233 .args(["--console", "off"]) 7234 .default_disks() 7235 .default_net() 7236 .capture_output() 7237 .spawn() 7238 .unwrap(); 7239 7240 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7241 7242 let r = std::panic::catch_unwind(|| { 7243 // Wait to make sure Windows boots up 7244 assert!(windows_guest.wait_for_boot()); 7245 7246 let vcpu_num = 2; 7247 // Check the initial number of CPUs the guest sees 7248 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7249 // Check the initial number of vcpu threads in the CH process 7250 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7251 7252 let vcpu_num = 6; 7253 // Hotplug some CPUs 7254 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7255 // Wait to make sure CPUs are added 7256 thread::sleep(std::time::Duration::new(10, 0)); 7257 // Check the guest sees the correct number 7258 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7259 // Check the CH process has the correct number of vcpu threads 7260 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7261 7262 let vcpu_num = 4; 7263 // Remove some CPUs. Note that Windows doesn't support hot-remove. 7264 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7265 // Wait to make sure CPUs are removed 7266 thread::sleep(std::time::Duration::new(10, 0)); 7267 // Reboot to let Windows catch up 7268 windows_guest.reboot(); 7269 // Wait to make sure Windows completely rebooted 7270 thread::sleep(std::time::Duration::new(60, 0)); 7271 // Check the guest sees the correct number 7272 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7273 // Check the CH process has the correct number of vcpu threads 7274 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7275 7276 windows_guest.shutdown(); 7277 }); 7278 7279 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7280 let _ = child.kill(); 7281 let output = child.wait_with_output().unwrap(); 7282 7283 let _ = child_dnsmasq.kill(); 7284 let _ = child_dnsmasq.wait(); 7285 7286 handle_child_output(r, &output); 7287 } 7288 7289 #[test] 7290 #[cfg(not(feature = "mshv"))] 7291 #[cfg(not(target_arch = "aarch64"))] 7292 fn test_windows_guest_ram_hotplug() { 7293 let windows_guest = WindowsGuest::new(); 7294 7295 let mut ovmf_path = dirs::home_dir().unwrap(); 7296 ovmf_path.push("workloads"); 7297 ovmf_path.push(OVMF_NAME); 7298 7299 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7300 let api_socket = temp_api_path(&tmp_dir); 7301 7302 let mut child = GuestCommand::new(windows_guest.guest()) 7303 .args(["--api-socket", &api_socket]) 7304 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7305 .args(["--memory", "size=2G,hotplug_size=5G"]) 7306 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7307 .args(["--serial", "tty"]) 7308 .args(["--console", "off"]) 7309 .default_disks() 7310 .default_net() 7311 .capture_output() 7312 .spawn() 7313 .unwrap(); 7314 7315 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7316 7317 let r = std::panic::catch_unwind(|| { 7318 // Wait to make sure Windows boots up 7319 assert!(windows_guest.wait_for_boot()); 7320 7321 let ram_size = 2 * 1024 * 1024 * 1024; 7322 // Check the initial number of RAM the guest sees 7323 let current_ram_size = windows_guest.ram_size(); 7324 // This size seems to be reserved by the system and thus the 7325 // reported amount differs by this constant value. 7326 let reserved_ram_size = ram_size - current_ram_size; 7327 // Verify that there's not more than 4mb constant diff wasted 7328 // by the reserved ram. 7329 assert!(reserved_ram_size < 4 * 1024 * 1024); 7330 7331 let ram_size = 4 * 1024 * 1024 * 1024; 7332 // Hotplug some RAM 7333 resize_command(&api_socket, None, Some(ram_size), None, None); 7334 // Wait to make sure RAM has been added 7335 thread::sleep(std::time::Duration::new(10, 0)); 7336 // Check the guest sees the correct number 7337 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7338 7339 let ram_size = 3 * 1024 * 1024 * 1024; 7340 // Unplug some RAM. Note that hot-remove most likely won't work. 7341 resize_command(&api_socket, None, Some(ram_size), None, None); 7342 // Wait to make sure RAM has been added 7343 thread::sleep(std::time::Duration::new(10, 0)); 7344 // Reboot to let Windows catch up 7345 windows_guest.reboot(); 7346 // Wait to make sure guest completely rebooted 7347 thread::sleep(std::time::Duration::new(60, 0)); 7348 // Check the guest sees the correct number 7349 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7350 7351 windows_guest.shutdown(); 7352 }); 7353 7354 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7355 let _ = child.kill(); 7356 let output = child.wait_with_output().unwrap(); 7357 7358 let _ = child_dnsmasq.kill(); 7359 let _ = child_dnsmasq.wait(); 7360 7361 handle_child_output(r, &output); 7362 } 7363 7364 #[test] 7365 #[cfg(not(feature = "mshv"))] 7366 fn test_windows_guest_netdev_hotplug() { 7367 let windows_guest = WindowsGuest::new(); 7368 7369 let mut ovmf_path = dirs::home_dir().unwrap(); 7370 ovmf_path.push("workloads"); 7371 ovmf_path.push(OVMF_NAME); 7372 7373 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7374 let api_socket = temp_api_path(&tmp_dir); 7375 7376 let mut child = GuestCommand::new(windows_guest.guest()) 7377 .args(["--api-socket", &api_socket]) 7378 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7379 .args(["--memory", "size=4G"]) 7380 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7381 .args(["--serial", "tty"]) 7382 .args(["--console", "off"]) 7383 .default_disks() 7384 .default_net() 7385 .capture_output() 7386 .spawn() 7387 .unwrap(); 7388 7389 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7390 7391 let r = std::panic::catch_unwind(|| { 7392 // Wait to make sure Windows boots up 7393 assert!(windows_guest.wait_for_boot()); 7394 7395 // Initially present network device 7396 let netdev_num = 1; 7397 assert_eq!(windows_guest.netdev_count(), netdev_num); 7398 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7399 7400 // Hotplug network device 7401 let (cmd_success, cmd_output) = remote_command_w_output( 7402 &api_socket, 7403 "add-net", 7404 Some(windows_guest.guest().default_net_string().as_str()), 7405 ); 7406 assert!(cmd_success); 7407 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 7408 thread::sleep(std::time::Duration::new(5, 0)); 7409 // Verify the device is on the system 7410 let netdev_num = 2; 7411 assert_eq!(windows_guest.netdev_count(), netdev_num); 7412 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7413 7414 // Remove network device 7415 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 7416 assert!(cmd_success); 7417 thread::sleep(std::time::Duration::new(5, 0)); 7418 // Verify the device has been removed 7419 let netdev_num = 1; 7420 assert_eq!(windows_guest.netdev_count(), netdev_num); 7421 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7422 7423 windows_guest.shutdown(); 7424 }); 7425 7426 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7427 let _ = child.kill(); 7428 let output = child.wait_with_output().unwrap(); 7429 7430 let _ = child_dnsmasq.kill(); 7431 let _ = child_dnsmasq.wait(); 7432 7433 handle_child_output(r, &output); 7434 } 7435 7436 #[test] 7437 #[cfg(not(feature = "mshv"))] 7438 #[cfg(not(target_arch = "aarch64"))] 7439 fn test_windows_guest_disk_hotplug() { 7440 let windows_guest = WindowsGuest::new(); 7441 7442 let mut ovmf_path = dirs::home_dir().unwrap(); 7443 ovmf_path.push("workloads"); 7444 ovmf_path.push(OVMF_NAME); 7445 7446 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7447 let api_socket = temp_api_path(&tmp_dir); 7448 7449 let mut child = GuestCommand::new(windows_guest.guest()) 7450 .args(["--api-socket", &api_socket]) 7451 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7452 .args(["--memory", "size=4G"]) 7453 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7454 .args(["--serial", "tty"]) 7455 .args(["--console", "off"]) 7456 .default_disks() 7457 .default_net() 7458 .capture_output() 7459 .spawn() 7460 .unwrap(); 7461 7462 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7463 7464 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 7465 7466 let r = std::panic::catch_unwind(|| { 7467 // Wait to make sure Windows boots up 7468 assert!(windows_guest.wait_for_boot()); 7469 7470 // Initially present disk device 7471 let disk_num = 1; 7472 assert_eq!(windows_guest.disk_count(), disk_num); 7473 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7474 7475 // Hotplug disk device 7476 let (cmd_success, cmd_output) = remote_command_w_output( 7477 &api_socket, 7478 "add-disk", 7479 Some(format!("path={disk},readonly=off").as_str()), 7480 ); 7481 assert!(cmd_success); 7482 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 7483 thread::sleep(std::time::Duration::new(5, 0)); 7484 // Online disk device 7485 windows_guest.disks_set_rw(); 7486 windows_guest.disks_online(); 7487 // Verify the device is on the system 7488 let disk_num = 2; 7489 assert_eq!(windows_guest.disk_count(), disk_num); 7490 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7491 7492 let data = "hello"; 7493 let fname = "d:\\world"; 7494 windows_guest.disk_file_put(fname, data); 7495 7496 // Unmount disk device 7497 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 7498 assert!(cmd_success); 7499 thread::sleep(std::time::Duration::new(5, 0)); 7500 // Verify the device has been removed 7501 let disk_num = 1; 7502 assert_eq!(windows_guest.disk_count(), disk_num); 7503 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7504 7505 // Remount and check the file exists with the expected contents 7506 let (cmd_success, _cmd_output) = remote_command_w_output( 7507 &api_socket, 7508 "add-disk", 7509 Some(format!("path={disk},readonly=off").as_str()), 7510 ); 7511 assert!(cmd_success); 7512 thread::sleep(std::time::Duration::new(5, 0)); 7513 let out = windows_guest.disk_file_read(fname); 7514 assert_eq!(data, out.trim()); 7515 7516 // Intentionally no unmount, it'll happen at shutdown. 7517 7518 windows_guest.shutdown(); 7519 }); 7520 7521 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7522 let _ = child.kill(); 7523 let output = child.wait_with_output().unwrap(); 7524 7525 let _ = child_dnsmasq.kill(); 7526 let _ = child_dnsmasq.wait(); 7527 7528 handle_child_output(r, &output); 7529 } 7530 7531 #[test] 7532 #[cfg(not(feature = "mshv"))] 7533 #[cfg(not(target_arch = "aarch64"))] 7534 fn test_windows_guest_disk_hotplug_multi() { 7535 let windows_guest = WindowsGuest::new(); 7536 7537 let mut ovmf_path = dirs::home_dir().unwrap(); 7538 ovmf_path.push("workloads"); 7539 ovmf_path.push(OVMF_NAME); 7540 7541 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7542 let api_socket = temp_api_path(&tmp_dir); 7543 7544 let mut child = GuestCommand::new(windows_guest.guest()) 7545 .args(["--api-socket", &api_socket]) 7546 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7547 .args(["--memory", "size=2G"]) 7548 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7549 .args(["--serial", "tty"]) 7550 .args(["--console", "off"]) 7551 .default_disks() 7552 .default_net() 7553 .capture_output() 7554 .spawn() 7555 .unwrap(); 7556 7557 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7558 7559 // Predefined data to used at various test stages 7560 let disk_test_data: [[String; 4]; 2] = [ 7561 [ 7562 "_disk2".to_string(), 7563 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 7564 "d:\\world".to_string(), 7565 "hello".to_string(), 7566 ], 7567 [ 7568 "_disk3".to_string(), 7569 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 7570 "e:\\hello".to_string(), 7571 "world".to_string(), 7572 ], 7573 ]; 7574 7575 let r = std::panic::catch_unwind(|| { 7576 // Wait to make sure Windows boots up 7577 assert!(windows_guest.wait_for_boot()); 7578 7579 // Initially present disk device 7580 let disk_num = 1; 7581 assert_eq!(windows_guest.disk_count(), disk_num); 7582 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7583 7584 for it in &disk_test_data { 7585 let disk_id = it[0].as_str(); 7586 let disk = it[1].as_str(); 7587 // Hotplug disk device 7588 let (cmd_success, cmd_output) = remote_command_w_output( 7589 &api_socket, 7590 "add-disk", 7591 Some(format!("path={disk},readonly=off").as_str()), 7592 ); 7593 assert!(cmd_success); 7594 assert!(String::from_utf8_lossy(&cmd_output) 7595 .contains(format!("\"id\":\"{disk_id}\"").as_str())); 7596 thread::sleep(std::time::Duration::new(5, 0)); 7597 // Online disk devices 7598 windows_guest.disks_set_rw(); 7599 windows_guest.disks_online(); 7600 } 7601 // Verify the devices are on the system 7602 let disk_num = (disk_test_data.len() + 1) as u8; 7603 assert_eq!(windows_guest.disk_count(), disk_num); 7604 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7605 7606 // Put test data 7607 for it in &disk_test_data { 7608 let fname = it[2].as_str(); 7609 let data = it[3].as_str(); 7610 windows_guest.disk_file_put(fname, data); 7611 } 7612 7613 // Unmount disk devices 7614 for it in &disk_test_data { 7615 let disk_id = it[0].as_str(); 7616 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 7617 assert!(cmd_success); 7618 thread::sleep(std::time::Duration::new(5, 0)); 7619 } 7620 7621 // Verify the devices have been removed 7622 let disk_num = 1; 7623 assert_eq!(windows_guest.disk_count(), disk_num); 7624 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7625 7626 // Remount 7627 for it in &disk_test_data { 7628 let disk = it[1].as_str(); 7629 let (cmd_success, _cmd_output) = remote_command_w_output( 7630 &api_socket, 7631 "add-disk", 7632 Some(format!("path={disk},readonly=off").as_str()), 7633 ); 7634 assert!(cmd_success); 7635 thread::sleep(std::time::Duration::new(5, 0)); 7636 } 7637 7638 // Check the files exists with the expected contents 7639 for it in &disk_test_data { 7640 let fname = it[2].as_str(); 7641 let data = it[3].as_str(); 7642 let out = windows_guest.disk_file_read(fname); 7643 assert_eq!(data, out.trim()); 7644 } 7645 7646 // Intentionally no unmount, it'll happen at shutdown. 7647 7648 windows_guest.shutdown(); 7649 }); 7650 7651 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7652 let _ = child.kill(); 7653 let output = child.wait_with_output().unwrap(); 7654 7655 let _ = child_dnsmasq.kill(); 7656 let _ = child_dnsmasq.wait(); 7657 7658 handle_child_output(r, &output); 7659 } 7660 7661 #[test] 7662 #[cfg(not(feature = "mshv"))] 7663 #[cfg(not(target_arch = "aarch64"))] 7664 fn test_windows_guest_netdev_multi() { 7665 let windows_guest = WindowsGuest::new(); 7666 7667 let mut ovmf_path = dirs::home_dir().unwrap(); 7668 ovmf_path.push("workloads"); 7669 ovmf_path.push(OVMF_NAME); 7670 7671 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7672 let api_socket = temp_api_path(&tmp_dir); 7673 7674 let mut child = GuestCommand::new(windows_guest.guest()) 7675 .args(["--api-socket", &api_socket]) 7676 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7677 .args(["--memory", "size=4G"]) 7678 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7679 .args(["--serial", "tty"]) 7680 .args(["--console", "off"]) 7681 .default_disks() 7682 // The multi net dev config is borrowed from test_multiple_network_interfaces 7683 .args([ 7684 "--net", 7685 windows_guest.guest().default_net_string().as_str(), 7686 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 7687 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 7688 ]) 7689 .capture_output() 7690 .spawn() 7691 .unwrap(); 7692 7693 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7694 7695 let r = std::panic::catch_unwind(|| { 7696 // Wait to make sure Windows boots up 7697 assert!(windows_guest.wait_for_boot()); 7698 7699 let netdev_num = 3; 7700 assert_eq!(windows_guest.netdev_count(), netdev_num); 7701 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7702 7703 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 7704 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 7705 7706 windows_guest.shutdown(); 7707 }); 7708 7709 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7710 let _ = child.kill(); 7711 let output = child.wait_with_output().unwrap(); 7712 7713 let _ = child_dnsmasq.kill(); 7714 let _ = child_dnsmasq.wait(); 7715 7716 handle_child_output(r, &output); 7717 } 7718 } 7719 7720 #[cfg(target_arch = "x86_64")] 7721 mod sgx { 7722 use crate::*; 7723 7724 #[test] 7725 fn test_sgx() { 7726 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 7727 let jammy = UbuntuDiskConfig::new(jammy_image); 7728 let guest = Guest::new(Box::new(jammy)); 7729 7730 let mut child = GuestCommand::new(&guest) 7731 .args(["--cpus", "boot=1"]) 7732 .args(["--memory", "size=512M"]) 7733 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7734 .default_disks() 7735 .default_net() 7736 .args(["--sgx-epc", "id=epc0,size=64M"]) 7737 .capture_output() 7738 .spawn() 7739 .unwrap(); 7740 7741 let r = std::panic::catch_unwind(|| { 7742 guest.wait_vm_boot(None).unwrap(); 7743 7744 // Check if SGX is correctly detected in the guest. 7745 guest.check_sgx_support().unwrap(); 7746 7747 // Validate the SGX EPC section is 64MiB. 7748 assert_eq!( 7749 guest 7750 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 7751 .unwrap() 7752 .trim(), 7753 "0x0000000004000000" 7754 ); 7755 }); 7756 7757 let _ = child.kill(); 7758 let output = child.wait_with_output().unwrap(); 7759 7760 handle_child_output(r, &output); 7761 } 7762 } 7763 7764 #[cfg(target_arch = "x86_64")] 7765 mod vfio { 7766 use crate::*; 7767 7768 #[test] 7769 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 7770 // backed networking interfaces, bound through a simple bridge on the host. 7771 // So if the nested cloud-hypervisor succeeds in getting a directly 7772 // assigned interface from its cloud-hypervisor host, we should be able to 7773 // ssh into it, and verify that it's running with the right kernel command 7774 // line (We tag the command line from cloud-hypervisor for that purpose). 7775 // The third device is added to validate that hotplug works correctly since 7776 // it is being added to the L2 VM through hotplugging mechanism. 7777 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 7778 // vfio device support 7779 fn test_vfio() { 7780 setup_vfio_network_interfaces(); 7781 7782 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7783 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 7784 7785 let mut workload_path = dirs::home_dir().unwrap(); 7786 workload_path.push("workloads"); 7787 7788 let kernel_path = direct_kernel_boot_path(); 7789 7790 let mut vfio_path = workload_path.clone(); 7791 vfio_path.push("vfio"); 7792 7793 let mut cloud_init_vfio_base_path = vfio_path.clone(); 7794 cloud_init_vfio_base_path.push("cloudinit.img"); 7795 7796 // We copy our cloudinit into the vfio mount point, for the nested 7797 // cloud-hypervisor guest to use. 7798 rate_limited_copy( 7799 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 7800 &cloud_init_vfio_base_path, 7801 ) 7802 .expect("copying of cloud-init disk failed"); 7803 7804 let mut vfio_disk_path = workload_path.clone(); 7805 vfio_disk_path.push("vfio.img"); 7806 7807 // Create the vfio disk image 7808 let output = Command::new("mkfs.ext4") 7809 .arg("-d") 7810 .arg(vfio_path.to_str().unwrap()) 7811 .arg(vfio_disk_path.to_str().unwrap()) 7812 .arg("2g") 7813 .output() 7814 .unwrap(); 7815 if !output.status.success() { 7816 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 7817 panic!("mkfs.ext4 command generated an error"); 7818 } 7819 7820 let mut blk_file_path = workload_path; 7821 blk_file_path.push("blk.img"); 7822 7823 let vfio_tap0 = "vfio-tap0"; 7824 let vfio_tap1 = "vfio-tap1"; 7825 let vfio_tap2 = "vfio-tap2"; 7826 let vfio_tap3 = "vfio-tap3"; 7827 7828 let mut child = GuestCommand::new(&guest) 7829 .args(["--cpus", "boot=4"]) 7830 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 7831 .args(["--kernel", kernel_path.to_str().unwrap()]) 7832 .args([ 7833 "--disk", 7834 format!( 7835 "path={}", 7836 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7837 ) 7838 .as_str(), 7839 format!( 7840 "path={}", 7841 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7842 ) 7843 .as_str(), 7844 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 7845 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 7846 ]) 7847 .args([ 7848 "--cmdline", 7849 format!( 7850 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 7851 ) 7852 .as_str(), 7853 ]) 7854 .args([ 7855 "--net", 7856 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 7857 format!( 7858 "tap={},mac={},iommu=on", 7859 vfio_tap1, guest.network.l2_guest_mac1 7860 ) 7861 .as_str(), 7862 format!( 7863 "tap={},mac={},iommu=on", 7864 vfio_tap2, guest.network.l2_guest_mac2 7865 ) 7866 .as_str(), 7867 format!( 7868 "tap={},mac={},iommu=on", 7869 vfio_tap3, guest.network.l2_guest_mac3 7870 ) 7871 .as_str(), 7872 ]) 7873 .capture_output() 7874 .spawn() 7875 .unwrap(); 7876 7877 thread::sleep(std::time::Duration::new(30, 0)); 7878 7879 let r = std::panic::catch_unwind(|| { 7880 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 7881 thread::sleep(std::time::Duration::new(120, 0)); 7882 7883 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 7884 // added to its kernel command line. 7885 // Let's ssh into it and verify that it's there. If it is it means 7886 // we're in the right guest (The L2 one) because the QEMU L1 guest 7887 // does not have this command line tag. 7888 assert_eq!( 7889 guest 7890 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 7891 .unwrap() 7892 .trim() 7893 .parse::<u32>() 7894 .unwrap_or_default(), 7895 1 7896 ); 7897 7898 // Let's also verify from the second virtio-net device passed to 7899 // the L2 VM. 7900 assert_eq!( 7901 guest 7902 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 7903 .unwrap() 7904 .trim() 7905 .parse::<u32>() 7906 .unwrap_or_default(), 7907 1 7908 ); 7909 7910 // Check the amount of PCI devices appearing in L2 VM. 7911 assert_eq!( 7912 guest 7913 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 7914 .unwrap() 7915 .trim() 7916 .parse::<u32>() 7917 .unwrap_or_default(), 7918 8, 7919 ); 7920 7921 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 7922 assert_eq!( 7923 guest 7924 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 7925 .unwrap() 7926 .trim() 7927 .parse::<u32>() 7928 .unwrap_or_default(), 7929 1 7930 ); 7931 7932 // Hotplug an extra virtio-net device through L2 VM. 7933 guest 7934 .ssh_command_l1( 7935 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 7936 ) 7937 .unwrap(); 7938 guest 7939 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 7940 .unwrap(); 7941 let vfio_hotplug_output = guest 7942 .ssh_command_l1( 7943 "sudo /mnt/ch-remote \ 7944 --api-socket=/tmp/ch_api.sock \ 7945 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 7946 ) 7947 .unwrap(); 7948 assert!(vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}")); 7949 7950 thread::sleep(std::time::Duration::new(10, 0)); 7951 7952 // Let's also verify from the third virtio-net device passed to 7953 // the L2 VM. This third device has been hotplugged through the L2 7954 // VM, so this is our way to validate hotplug works for VFIO PCI. 7955 assert_eq!( 7956 guest 7957 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 7958 .unwrap() 7959 .trim() 7960 .parse::<u32>() 7961 .unwrap_or_default(), 7962 1 7963 ); 7964 7965 // Check the amount of PCI devices appearing in L2 VM. 7966 // There should be one more device than before, raising the count 7967 // up to 9 PCI devices. 7968 assert_eq!( 7969 guest 7970 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 7971 .unwrap() 7972 .trim() 7973 .parse::<u32>() 7974 .unwrap_or_default(), 7975 9, 7976 ); 7977 7978 // Let's now verify that we can correctly remove the virtio-net 7979 // device through the "remove-device" command responsible for 7980 // unplugging VFIO devices. 7981 guest 7982 .ssh_command_l1( 7983 "sudo /mnt/ch-remote \ 7984 --api-socket=/tmp/ch_api.sock \ 7985 remove-device vfio123", 7986 ) 7987 .unwrap(); 7988 thread::sleep(std::time::Duration::new(10, 0)); 7989 7990 // Check the amount of PCI devices appearing in L2 VM is back down 7991 // to 8 devices. 7992 assert_eq!( 7993 guest 7994 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 7995 .unwrap() 7996 .trim() 7997 .parse::<u32>() 7998 .unwrap_or_default(), 7999 8, 8000 ); 8001 8002 // Perform memory hotplug in L2 and validate the memory is showing 8003 // up as expected. In order to check, we will use the virtio-net 8004 // device already passed through L2 as a VFIO device, this will 8005 // verify that VFIO devices are functional with memory hotplug. 8006 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 8007 guest 8008 .ssh_command_l2_1( 8009 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 8010 ) 8011 .unwrap(); 8012 guest 8013 .ssh_command_l1( 8014 "sudo /mnt/ch-remote \ 8015 --api-socket=/tmp/ch_api.sock \ 8016 resize --memory=1073741824", 8017 ) 8018 .unwrap(); 8019 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 8020 }); 8021 8022 let _ = child.kill(); 8023 let output = child.wait_with_output().unwrap(); 8024 8025 cleanup_vfio_network_interfaces(); 8026 8027 handle_child_output(r, &output); 8028 } 8029 8030 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 8031 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8032 let guest = Guest::new(Box::new(jammy)); 8033 let api_socket = temp_api_path(&guest.tmp_dir); 8034 8035 let mut child = GuestCommand::new(&guest) 8036 .args(["--cpus", "boot=4"]) 8037 .args([ 8038 "--memory", 8039 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(), 8040 ]) 8041 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8042 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8043 .args(["--api-socket", &api_socket]) 8044 .default_disks() 8045 .default_net() 8046 .capture_output() 8047 .spawn() 8048 .unwrap(); 8049 8050 let r = std::panic::catch_unwind(|| { 8051 guest.wait_vm_boot(None).unwrap(); 8052 8053 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8054 8055 guest.enable_memory_hotplug(); 8056 8057 // Add RAM to the VM 8058 let desired_ram = 6 << 30; 8059 resize_command(&api_socket, None, Some(desired_ram), None, None); 8060 thread::sleep(std::time::Duration::new(30, 0)); 8061 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8062 8063 // Check the VFIO device works when RAM is increased to 6GiB 8064 guest.check_nvidia_gpu(); 8065 }); 8066 8067 let _ = child.kill(); 8068 let output = child.wait_with_output().unwrap(); 8069 8070 handle_child_output(r, &output); 8071 } 8072 8073 #[test] 8074 fn test_nvidia_card_memory_hotplug_acpi() { 8075 test_nvidia_card_memory_hotplug("acpi") 8076 } 8077 8078 #[test] 8079 fn test_nvidia_card_memory_hotplug_virtio_mem() { 8080 test_nvidia_card_memory_hotplug("virtio-mem") 8081 } 8082 8083 #[test] 8084 fn test_nvidia_card_pci_hotplug() { 8085 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8086 let guest = Guest::new(Box::new(jammy)); 8087 let api_socket = temp_api_path(&guest.tmp_dir); 8088 8089 let mut child = GuestCommand::new(&guest) 8090 .args(["--cpus", "boot=4"]) 8091 .args(["--memory", "size=4G"]) 8092 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8093 .args(["--api-socket", &api_socket]) 8094 .default_disks() 8095 .default_net() 8096 .capture_output() 8097 .spawn() 8098 .unwrap(); 8099 8100 let r = std::panic::catch_unwind(|| { 8101 guest.wait_vm_boot(None).unwrap(); 8102 8103 // Hotplug the card to the VM 8104 let (cmd_success, cmd_output) = remote_command_w_output( 8105 &api_socket, 8106 "add-device", 8107 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 8108 ); 8109 assert!(cmd_success); 8110 assert!(String::from_utf8_lossy(&cmd_output) 8111 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 8112 8113 thread::sleep(std::time::Duration::new(10, 0)); 8114 8115 // Check the VFIO device works after hotplug 8116 guest.check_nvidia_gpu(); 8117 }); 8118 8119 let _ = child.kill(); 8120 let output = child.wait_with_output().unwrap(); 8121 8122 handle_child_output(r, &output); 8123 } 8124 8125 #[test] 8126 fn test_nvidia_card_reboot() { 8127 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8128 let guest = Guest::new(Box::new(jammy)); 8129 let api_socket = temp_api_path(&guest.tmp_dir); 8130 8131 let mut child = GuestCommand::new(&guest) 8132 .args(["--cpus", "boot=4"]) 8133 .args(["--memory", "size=4G"]) 8134 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8135 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8136 .args(["--api-socket", &api_socket]) 8137 .default_disks() 8138 .default_net() 8139 .capture_output() 8140 .spawn() 8141 .unwrap(); 8142 8143 let r = std::panic::catch_unwind(|| { 8144 guest.wait_vm_boot(None).unwrap(); 8145 8146 // Check the VFIO device works after boot 8147 guest.check_nvidia_gpu(); 8148 8149 guest.reboot_linux(0, None); 8150 8151 // Check the VFIO device works after reboot 8152 guest.check_nvidia_gpu(); 8153 }); 8154 8155 let _ = child.kill(); 8156 let output = child.wait_with_output().unwrap(); 8157 8158 handle_child_output(r, &output); 8159 } 8160 } 8161 8162 mod live_migration { 8163 use crate::*; 8164 8165 fn start_live_migration( 8166 migration_socket: &str, 8167 src_api_socket: &str, 8168 dest_api_socket: &str, 8169 local: bool, 8170 ) -> bool { 8171 // Start to receive migration from the destintion VM 8172 let mut receive_migration = Command::new(clh_command("ch-remote")) 8173 .args([ 8174 &format!("--api-socket={dest_api_socket}"), 8175 "receive-migration", 8176 &format! {"unix:{migration_socket}"}, 8177 ]) 8178 .stderr(Stdio::piped()) 8179 .stdout(Stdio::piped()) 8180 .spawn() 8181 .unwrap(); 8182 // Give it '1s' to make sure the 'migration_socket' file is properly created 8183 thread::sleep(std::time::Duration::new(1, 0)); 8184 // Start to send migration from the source VM 8185 8186 let mut args = [ 8187 format!("--api-socket={}", &src_api_socket), 8188 "send-migration".to_string(), 8189 format! {"unix:{migration_socket}"}, 8190 ] 8191 .to_vec(); 8192 8193 if local { 8194 args.insert(2, "--local".to_string()); 8195 } 8196 8197 let mut send_migration = Command::new(clh_command("ch-remote")) 8198 .args(&args) 8199 .stderr(Stdio::piped()) 8200 .stdout(Stdio::piped()) 8201 .spawn() 8202 .unwrap(); 8203 8204 // The 'send-migration' command should be executed successfully within the given timeout 8205 let send_success = if let Some(status) = send_migration 8206 .wait_timeout(std::time::Duration::from_secs(30)) 8207 .unwrap() 8208 { 8209 status.success() 8210 } else { 8211 false 8212 }; 8213 8214 if !send_success { 8215 let _ = send_migration.kill(); 8216 let output = send_migration.wait_with_output().unwrap(); 8217 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 8218 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8219 } 8220 8221 // The 'receive-migration' command should be executed successfully within the given timeout 8222 let receive_success = if let Some(status) = receive_migration 8223 .wait_timeout(std::time::Duration::from_secs(30)) 8224 .unwrap() 8225 { 8226 status.success() 8227 } else { 8228 false 8229 }; 8230 8231 if !receive_success { 8232 let _ = receive_migration.kill(); 8233 let output = receive_migration.wait_with_output().unwrap(); 8234 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 8235 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8236 } 8237 8238 send_success && receive_success 8239 } 8240 8241 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 8242 let mut src_vm = src_vm; 8243 let mut dest_vm = dest_vm; 8244 8245 let _ = src_vm.kill(); 8246 let src_output = src_vm.wait_with_output().unwrap(); 8247 eprintln!( 8248 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8249 String::from_utf8_lossy(&src_output.stdout) 8250 ); 8251 eprintln!( 8252 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8253 String::from_utf8_lossy(&src_output.stderr) 8254 ); 8255 let _ = dest_vm.kill(); 8256 let dest_output = dest_vm.wait_with_output().unwrap(); 8257 eprintln!( 8258 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8259 String::from_utf8_lossy(&dest_output.stdout) 8260 ); 8261 eprintln!( 8262 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8263 String::from_utf8_lossy(&dest_output.stderr) 8264 ); 8265 8266 if let Some(ovs_vm) = ovs_vm { 8267 let mut ovs_vm = ovs_vm; 8268 let _ = ovs_vm.kill(); 8269 let ovs_output = ovs_vm.wait_with_output().unwrap(); 8270 eprintln!( 8271 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 8272 String::from_utf8_lossy(&ovs_output.stdout) 8273 ); 8274 eprintln!( 8275 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 8276 String::from_utf8_lossy(&ovs_output.stderr) 8277 ); 8278 8279 cleanup_ovs_dpdk(); 8280 } 8281 8282 panic!("Test failed: {message}") 8283 } 8284 8285 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 8286 // same host. It ensures the following behaviors: 8287 // 1. The source VM is up and functional (including various virtio-devices are working properly); 8288 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 8289 // 3. The source VM terminated gracefully after live migration; 8290 // 4. The destination VM is functional (including various virtio-devices are working properly) after 8291 // live migration; 8292 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 8293 fn _test_live_migration(upgrade_test: bool, local: bool) { 8294 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8295 let guest = Guest::new(Box::new(focal)); 8296 let kernel_path = direct_kernel_boot_path(); 8297 let console_text = String::from("On a branch floating down river a cricket, singing."); 8298 let net_id = "net123"; 8299 let net_params = format!( 8300 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8301 net_id, guest.network.guest_mac, guest.network.host_ip 8302 ); 8303 8304 let memory_param: &[&str] = if local { 8305 &["--memory", "size=4G,shared=on"] 8306 } else { 8307 &["--memory", "size=4G"] 8308 }; 8309 8310 let boot_vcpus = 2; 8311 let max_vcpus = 4; 8312 8313 let pmem_temp_file = TempFile::new().unwrap(); 8314 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8315 std::process::Command::new("mkfs.ext4") 8316 .arg(pmem_temp_file.as_path()) 8317 .output() 8318 .expect("Expect creating disk image to succeed"); 8319 let pmem_path = String::from("/dev/pmem0"); 8320 8321 // Start the source VM 8322 let src_vm_path = if !upgrade_test { 8323 clh_command("cloud-hypervisor") 8324 } else { 8325 cloud_hypervisor_release_path() 8326 }; 8327 let src_api_socket = temp_api_path(&guest.tmp_dir); 8328 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8329 src_vm_cmd 8330 .args([ 8331 "--cpus", 8332 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8333 ]) 8334 .args(memory_param) 8335 .args(["--kernel", kernel_path.to_str().unwrap()]) 8336 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8337 .default_disks() 8338 .args(["--net", net_params.as_str()]) 8339 .args(["--api-socket", &src_api_socket]) 8340 .args([ 8341 "--pmem", 8342 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8343 ]); 8344 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8345 8346 // Start the destination VM 8347 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8348 dest_api_socket.push_str(".dest"); 8349 let mut dest_child = GuestCommand::new(&guest) 8350 .args(["--api-socket", &dest_api_socket]) 8351 .capture_output() 8352 .spawn() 8353 .unwrap(); 8354 8355 let r = std::panic::catch_unwind(|| { 8356 guest.wait_vm_boot(None).unwrap(); 8357 8358 // Make sure the source VM is functaionl 8359 // Check the number of vCPUs 8360 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8361 8362 // Check the guest RAM 8363 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8364 8365 // Check the guest virtio-devices, e.g. block, rng, console, and net 8366 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8367 8368 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8369 // to make sure that removing and adding back the virtio-net device does 8370 // not break the live-migration support for virtio-pci. 8371 #[cfg(target_arch = "x86_64")] 8372 { 8373 assert!(remote_command( 8374 &src_api_socket, 8375 "remove-device", 8376 Some(net_id), 8377 )); 8378 thread::sleep(std::time::Duration::new(10, 0)); 8379 8380 // Plug the virtio-net device again 8381 assert!(remote_command( 8382 &src_api_socket, 8383 "add-net", 8384 Some(net_params.as_str()), 8385 )); 8386 thread::sleep(std::time::Duration::new(10, 0)); 8387 } 8388 8389 // Start the live-migration 8390 let migration_socket = String::from( 8391 guest 8392 .tmp_dir 8393 .as_path() 8394 .join("live-migration.sock") 8395 .to_str() 8396 .unwrap(), 8397 ); 8398 8399 assert!( 8400 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8401 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8402 ); 8403 }); 8404 8405 // Check and report any errors occured during the live-migration 8406 if r.is_err() { 8407 print_and_panic( 8408 src_child, 8409 dest_child, 8410 None, 8411 "Error occured during live-migration", 8412 ); 8413 } 8414 8415 // Check the source vm has been terminated successful (give it '3s' to settle) 8416 thread::sleep(std::time::Duration::new(3, 0)); 8417 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8418 print_and_panic( 8419 src_child, 8420 dest_child, 8421 None, 8422 "source VM was not terminated successfully.", 8423 ); 8424 }; 8425 8426 // Post live-migration check to make sure the destination VM is funcational 8427 let r = std::panic::catch_unwind(|| { 8428 // Perform same checks to validate VM has been properly migrated 8429 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8430 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8431 8432 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8433 }); 8434 8435 // Clean-up the destination VM and make sure it terminated correctly 8436 let _ = dest_child.kill(); 8437 let dest_output = dest_child.wait_with_output().unwrap(); 8438 handle_child_output(r, &dest_output); 8439 8440 // Check the destination VM has the expected 'concole_text' from its output 8441 let r = std::panic::catch_unwind(|| { 8442 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8443 }); 8444 handle_child_output(r, &dest_output); 8445 } 8446 8447 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 8448 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8449 let guest = Guest::new(Box::new(focal)); 8450 let kernel_path = direct_kernel_boot_path(); 8451 let console_text = String::from("On a branch floating down river a cricket, singing."); 8452 let net_id = "net123"; 8453 let net_params = format!( 8454 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8455 net_id, guest.network.guest_mac, guest.network.host_ip 8456 ); 8457 8458 let memory_param: &[&str] = if local { 8459 &[ 8460 "--memory", 8461 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 8462 "--balloon", 8463 "size=0", 8464 ] 8465 } else { 8466 &[ 8467 "--memory", 8468 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 8469 "--balloon", 8470 "size=0", 8471 ] 8472 }; 8473 8474 let boot_vcpus = 2; 8475 let max_vcpus = 4; 8476 8477 let pmem_temp_file = TempFile::new().unwrap(); 8478 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8479 std::process::Command::new("mkfs.ext4") 8480 .arg(pmem_temp_file.as_path()) 8481 .output() 8482 .expect("Expect creating disk image to succeed"); 8483 let pmem_path = String::from("/dev/pmem0"); 8484 8485 // Start the source VM 8486 let src_vm_path = if !upgrade_test { 8487 clh_command("cloud-hypervisor") 8488 } else { 8489 cloud_hypervisor_release_path() 8490 }; 8491 let src_api_socket = temp_api_path(&guest.tmp_dir); 8492 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8493 src_vm_cmd 8494 .args([ 8495 "--cpus", 8496 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8497 ]) 8498 .args(memory_param) 8499 .args(["--kernel", kernel_path.to_str().unwrap()]) 8500 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8501 .default_disks() 8502 .args(["--net", net_params.as_str()]) 8503 .args(["--api-socket", &src_api_socket]) 8504 .args([ 8505 "--pmem", 8506 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8507 ]); 8508 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8509 8510 // Start the destination VM 8511 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8512 dest_api_socket.push_str(".dest"); 8513 let mut dest_child = GuestCommand::new(&guest) 8514 .args(["--api-socket", &dest_api_socket]) 8515 .capture_output() 8516 .spawn() 8517 .unwrap(); 8518 8519 let r = std::panic::catch_unwind(|| { 8520 guest.wait_vm_boot(None).unwrap(); 8521 8522 // Make sure the source VM is functaionl 8523 // Check the number of vCPUs 8524 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8525 8526 // Check the guest RAM 8527 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8528 // Increase the guest RAM 8529 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 8530 thread::sleep(std::time::Duration::new(5, 0)); 8531 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8532 // Use balloon to remove RAM from the VM 8533 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 8534 thread::sleep(std::time::Duration::new(5, 0)); 8535 let total_memory = guest.get_total_memory().unwrap_or_default(); 8536 assert!(total_memory > 4_800_000); 8537 assert!(total_memory < 5_760_000); 8538 8539 // Check the guest virtio-devices, e.g. block, rng, console, and net 8540 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8541 8542 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8543 // to make sure that removing and adding back the virtio-net device does 8544 // not break the live-migration support for virtio-pci. 8545 #[cfg(target_arch = "x86_64")] 8546 { 8547 assert!(remote_command( 8548 &src_api_socket, 8549 "remove-device", 8550 Some(net_id), 8551 )); 8552 thread::sleep(std::time::Duration::new(10, 0)); 8553 8554 // Plug the virtio-net device again 8555 assert!(remote_command( 8556 &src_api_socket, 8557 "add-net", 8558 Some(net_params.as_str()), 8559 )); 8560 thread::sleep(std::time::Duration::new(10, 0)); 8561 } 8562 8563 // Start the live-migration 8564 let migration_socket = String::from( 8565 guest 8566 .tmp_dir 8567 .as_path() 8568 .join("live-migration.sock") 8569 .to_str() 8570 .unwrap(), 8571 ); 8572 8573 assert!( 8574 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8575 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8576 ); 8577 }); 8578 8579 // Check and report any errors occured during the live-migration 8580 if r.is_err() { 8581 print_and_panic( 8582 src_child, 8583 dest_child, 8584 None, 8585 "Error occured during live-migration", 8586 ); 8587 } 8588 8589 // Check the source vm has been terminated successful (give it '3s' to settle) 8590 thread::sleep(std::time::Duration::new(3, 0)); 8591 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8592 print_and_panic( 8593 src_child, 8594 dest_child, 8595 None, 8596 "source VM was not terminated successfully.", 8597 ); 8598 }; 8599 8600 // Post live-migration check to make sure the destination VM is funcational 8601 let r = std::panic::catch_unwind(|| { 8602 // Perform same checks to validate VM has been properly migrated 8603 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8604 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8605 8606 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8607 8608 // Perform checks on guest RAM using balloon 8609 let total_memory = guest.get_total_memory().unwrap_or_default(); 8610 assert!(total_memory > 4_800_000); 8611 assert!(total_memory < 5_760_000); 8612 // Deflate balloon to restore entire RAM to the VM 8613 resize_command(&dest_api_socket, None, None, Some(0), None); 8614 thread::sleep(std::time::Duration::new(5, 0)); 8615 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8616 // Decrease guest RAM with virtio-mem 8617 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 8618 thread::sleep(std::time::Duration::new(5, 0)); 8619 let total_memory = guest.get_total_memory().unwrap_or_default(); 8620 assert!(total_memory > 4_800_000); 8621 assert!(total_memory < 5_760_000); 8622 }); 8623 8624 // Clean-up the destination VM and make sure it terminated correctly 8625 let _ = dest_child.kill(); 8626 let dest_output = dest_child.wait_with_output().unwrap(); 8627 handle_child_output(r, &dest_output); 8628 8629 // Check the destination VM has the expected 'concole_text' from its output 8630 let r = std::panic::catch_unwind(|| { 8631 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8632 }); 8633 handle_child_output(r, &dest_output); 8634 } 8635 8636 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 8637 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8638 let guest = Guest::new(Box::new(focal)); 8639 let kernel_path = direct_kernel_boot_path(); 8640 let console_text = String::from("On a branch floating down river a cricket, singing."); 8641 let net_id = "net123"; 8642 let net_params = format!( 8643 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8644 net_id, guest.network.guest_mac, guest.network.host_ip 8645 ); 8646 8647 let memory_param: &[&str] = if local { 8648 &[ 8649 "--memory", 8650 "size=0,hotplug_method=virtio-mem,shared=on", 8651 "--memory-zone", 8652 "id=mem0,size=1G,hotplug_size=4G,shared=on", 8653 "id=mem1,size=1G,hotplug_size=4G,shared=on", 8654 "id=mem2,size=2G,hotplug_size=4G,shared=on", 8655 "--numa", 8656 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 8657 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 8658 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 8659 ] 8660 } else { 8661 &[ 8662 "--memory", 8663 "size=0,hotplug_method=virtio-mem", 8664 "--memory-zone", 8665 "id=mem0,size=1G,hotplug_size=4G", 8666 "id=mem1,size=1G,hotplug_size=4G", 8667 "id=mem2,size=2G,hotplug_size=4G", 8668 "--numa", 8669 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 8670 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 8671 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 8672 ] 8673 }; 8674 8675 let boot_vcpus = 6; 8676 let max_vcpus = 12; 8677 8678 let pmem_temp_file = TempFile::new().unwrap(); 8679 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8680 std::process::Command::new("mkfs.ext4") 8681 .arg(pmem_temp_file.as_path()) 8682 .output() 8683 .expect("Expect creating disk image to succeed"); 8684 let pmem_path = String::from("/dev/pmem0"); 8685 8686 // Start the source VM 8687 let src_vm_path = if !upgrade_test { 8688 clh_command("cloud-hypervisor") 8689 } else { 8690 cloud_hypervisor_release_path() 8691 }; 8692 let src_api_socket = temp_api_path(&guest.tmp_dir); 8693 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8694 src_vm_cmd 8695 .args([ 8696 "--cpus", 8697 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8698 ]) 8699 .args(memory_param) 8700 .args(["--kernel", kernel_path.to_str().unwrap()]) 8701 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8702 .default_disks() 8703 .args(["--net", net_params.as_str()]) 8704 .args(["--api-socket", &src_api_socket]) 8705 .args([ 8706 "--pmem", 8707 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8708 ]); 8709 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8710 8711 // Start the destination VM 8712 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8713 dest_api_socket.push_str(".dest"); 8714 let mut dest_child = GuestCommand::new(&guest) 8715 .args(["--api-socket", &dest_api_socket]) 8716 .capture_output() 8717 .spawn() 8718 .unwrap(); 8719 8720 let r = std::panic::catch_unwind(|| { 8721 guest.wait_vm_boot(None).unwrap(); 8722 8723 // Make sure the source VM is functaionl 8724 // Check the number of vCPUs 8725 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8726 8727 // Check the guest RAM 8728 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 8729 8730 // Check the guest virtio-devices, e.g. block, rng, console, and net 8731 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8732 8733 // Check the NUMA parameters are applied correctly and resize 8734 // each zone to test the case where we migrate a VM with the 8735 // virtio-mem regions being used. 8736 { 8737 guest.check_numa_common( 8738 Some(&[960_000, 960_000, 1_920_000]), 8739 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8740 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8741 ); 8742 8743 // AArch64 currently does not support hotplug, and therefore we only 8744 // test hotplug-related function on x86_64 here. 8745 #[cfg(target_arch = "x86_64")] 8746 { 8747 guest.enable_memory_hotplug(); 8748 8749 // Resize every memory zone and check each associated NUMA node 8750 // has been assigned the right amount of memory. 8751 resize_zone_command(&src_api_socket, "mem0", "2G"); 8752 resize_zone_command(&src_api_socket, "mem1", "2G"); 8753 resize_zone_command(&src_api_socket, "mem2", "3G"); 8754 thread::sleep(std::time::Duration::new(5, 0)); 8755 8756 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 8757 } 8758 } 8759 8760 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8761 // to make sure that removing and adding back the virtio-net device does 8762 // not break the live-migration support for virtio-pci. 8763 #[cfg(target_arch = "x86_64")] 8764 { 8765 assert!(remote_command( 8766 &src_api_socket, 8767 "remove-device", 8768 Some(net_id), 8769 )); 8770 thread::sleep(std::time::Duration::new(10, 0)); 8771 8772 // Plug the virtio-net device again 8773 assert!(remote_command( 8774 &src_api_socket, 8775 "add-net", 8776 Some(net_params.as_str()), 8777 )); 8778 thread::sleep(std::time::Duration::new(10, 0)); 8779 } 8780 8781 // Start the live-migration 8782 let migration_socket = String::from( 8783 guest 8784 .tmp_dir 8785 .as_path() 8786 .join("live-migration.sock") 8787 .to_str() 8788 .unwrap(), 8789 ); 8790 8791 assert!( 8792 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8793 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8794 ); 8795 }); 8796 8797 // Check and report any errors occured during the live-migration 8798 if r.is_err() { 8799 print_and_panic( 8800 src_child, 8801 dest_child, 8802 None, 8803 "Error occured during live-migration", 8804 ); 8805 } 8806 8807 // Check the source vm has been terminated successful (give it '3s' to settle) 8808 thread::sleep(std::time::Duration::new(3, 0)); 8809 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8810 print_and_panic( 8811 src_child, 8812 dest_child, 8813 None, 8814 "source VM was not terminated successfully.", 8815 ); 8816 }; 8817 8818 // Post live-migration check to make sure the destination VM is funcational 8819 let r = std::panic::catch_unwind(|| { 8820 // Perform same checks to validate VM has been properly migrated 8821 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8822 #[cfg(target_arch = "x86_64")] 8823 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 8824 #[cfg(target_arch = "aarch64")] 8825 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8826 8827 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8828 8829 // Perform NUMA related checks 8830 { 8831 #[cfg(target_arch = "aarch64")] 8832 { 8833 guest.check_numa_common( 8834 Some(&[960_000, 960_000, 1_920_000]), 8835 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8836 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8837 ); 8838 } 8839 8840 // AArch64 currently does not support hotplug, and therefore we only 8841 // test hotplug-related function on x86_64 here. 8842 #[cfg(target_arch = "x86_64")] 8843 { 8844 guest.check_numa_common( 8845 Some(&[1_920_000, 1_920_000, 2_880_000]), 8846 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8847 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8848 ); 8849 8850 guest.enable_memory_hotplug(); 8851 8852 // Resize every memory zone and check each associated NUMA node 8853 // has been assigned the right amount of memory. 8854 resize_zone_command(&dest_api_socket, "mem0", "4G"); 8855 resize_zone_command(&dest_api_socket, "mem1", "4G"); 8856 resize_zone_command(&dest_api_socket, "mem2", "4G"); 8857 // Resize to the maximum amount of CPUs and check each NUMA 8858 // node has been assigned the right CPUs set. 8859 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 8860 thread::sleep(std::time::Duration::new(5, 0)); 8861 8862 guest.check_numa_common( 8863 Some(&[3_840_000, 3_840_000, 3_840_000]), 8864 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 8865 None, 8866 ); 8867 } 8868 } 8869 }); 8870 8871 // Clean-up the destination VM and make sure it terminated correctly 8872 let _ = dest_child.kill(); 8873 let dest_output = dest_child.wait_with_output().unwrap(); 8874 handle_child_output(r, &dest_output); 8875 8876 // Check the destination VM has the expected 'concole_text' from its output 8877 let r = std::panic::catch_unwind(|| { 8878 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8879 }); 8880 handle_child_output(r, &dest_output); 8881 } 8882 8883 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 8884 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8885 let guest = Guest::new(Box::new(focal)); 8886 let kernel_path = direct_kernel_boot_path(); 8887 let console_text = String::from("On a branch floating down river a cricket, singing."); 8888 let net_id = "net123"; 8889 let net_params = format!( 8890 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8891 net_id, guest.network.guest_mac, guest.network.host_ip 8892 ); 8893 8894 let memory_param: &[&str] = if local { 8895 &["--memory", "size=4G,shared=on"] 8896 } else { 8897 &["--memory", "size=4G"] 8898 }; 8899 8900 let boot_vcpus = 2; 8901 let max_vcpus = 4; 8902 8903 let pmem_temp_file = TempFile::new().unwrap(); 8904 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8905 std::process::Command::new("mkfs.ext4") 8906 .arg(pmem_temp_file.as_path()) 8907 .output() 8908 .expect("Expect creating disk image to succeed"); 8909 let pmem_path = String::from("/dev/pmem0"); 8910 8911 // Start the source VM 8912 let src_vm_path = if !upgrade_test { 8913 clh_command("cloud-hypervisor") 8914 } else { 8915 cloud_hypervisor_release_path() 8916 }; 8917 let src_api_socket = temp_api_path(&guest.tmp_dir); 8918 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8919 src_vm_cmd 8920 .args([ 8921 "--cpus", 8922 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8923 ]) 8924 .args(memory_param) 8925 .args(["--kernel", kernel_path.to_str().unwrap()]) 8926 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8927 .default_disks() 8928 .args(["--net", net_params.as_str()]) 8929 .args(["--api-socket", &src_api_socket]) 8930 .args([ 8931 "--pmem", 8932 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8933 ]) 8934 .args(["--watchdog"]); 8935 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8936 8937 // Start the destination VM 8938 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8939 dest_api_socket.push_str(".dest"); 8940 let mut dest_child = GuestCommand::new(&guest) 8941 .args(["--api-socket", &dest_api_socket]) 8942 .capture_output() 8943 .spawn() 8944 .unwrap(); 8945 8946 let r = std::panic::catch_unwind(|| { 8947 guest.wait_vm_boot(None).unwrap(); 8948 8949 // Make sure the source VM is functaionl 8950 // Check the number of vCPUs 8951 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8952 // Check the guest RAM 8953 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8954 // Check the guest virtio-devices, e.g. block, rng, console, and net 8955 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8956 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8957 // to make sure that removing and adding back the virtio-net device does 8958 // not break the live-migration support for virtio-pci. 8959 #[cfg(target_arch = "x86_64")] 8960 { 8961 assert!(remote_command( 8962 &src_api_socket, 8963 "remove-device", 8964 Some(net_id), 8965 )); 8966 thread::sleep(std::time::Duration::new(10, 0)); 8967 8968 // Plug the virtio-net device again 8969 assert!(remote_command( 8970 &src_api_socket, 8971 "add-net", 8972 Some(net_params.as_str()), 8973 )); 8974 thread::sleep(std::time::Duration::new(10, 0)); 8975 } 8976 8977 // Enable watchdog and ensure its functional 8978 let mut expected_reboot_count = 1; 8979 // Enable the watchdog with a 15s timeout 8980 enable_guest_watchdog(&guest, 15); 8981 // Reboot and check that systemd has activated the watchdog 8982 guest.ssh_command("sudo reboot").unwrap(); 8983 guest.wait_vm_boot(None).unwrap(); 8984 expected_reboot_count += 1; 8985 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 8986 assert_eq!( 8987 guest 8988 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 8989 .unwrap() 8990 .trim() 8991 .parse::<u32>() 8992 .unwrap_or_default(), 8993 2 8994 ); 8995 // Allow some normal time to elapse to check we don't get spurious reboots 8996 thread::sleep(std::time::Duration::new(40, 0)); 8997 // Check no reboot 8998 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 8999 9000 // Start the live-migration 9001 let migration_socket = String::from( 9002 guest 9003 .tmp_dir 9004 .as_path() 9005 .join("live-migration.sock") 9006 .to_str() 9007 .unwrap(), 9008 ); 9009 9010 assert!( 9011 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9012 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9013 ); 9014 }); 9015 9016 // Check and report any errors occured during the live-migration 9017 if r.is_err() { 9018 print_and_panic( 9019 src_child, 9020 dest_child, 9021 None, 9022 "Error occured during live-migration", 9023 ); 9024 } 9025 9026 // Check the source vm has been terminated successful (give it '3s' to settle) 9027 thread::sleep(std::time::Duration::new(3, 0)); 9028 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9029 print_and_panic( 9030 src_child, 9031 dest_child, 9032 None, 9033 "source VM was not terminated successfully.", 9034 ); 9035 }; 9036 9037 // Post live-migration check to make sure the destination VM is funcational 9038 let r = std::panic::catch_unwind(|| { 9039 // Perform same checks to validate VM has been properly migrated 9040 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9041 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9042 9043 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9044 9045 // Perform checks on watchdog 9046 let mut expected_reboot_count = 2; 9047 9048 // Allow some normal time to elapse to check we don't get spurious reboots 9049 thread::sleep(std::time::Duration::new(40, 0)); 9050 // Check no reboot 9051 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9052 9053 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 9054 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 9055 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 9056 guest.wait_vm_boot(Some(50)).unwrap(); 9057 // Check a reboot is triggerred by the watchdog 9058 expected_reboot_count += 1; 9059 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9060 9061 #[cfg(target_arch = "x86_64")] 9062 { 9063 // Now pause the VM and remain offline for 30s 9064 assert!(remote_command(&dest_api_socket, "pause", None)); 9065 thread::sleep(std::time::Duration::new(30, 0)); 9066 assert!(remote_command(&dest_api_socket, "resume", None)); 9067 9068 // Check no reboot 9069 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9070 } 9071 }); 9072 9073 // Clean-up the destination VM and make sure it terminated correctly 9074 let _ = dest_child.kill(); 9075 let dest_output = dest_child.wait_with_output().unwrap(); 9076 handle_child_output(r, &dest_output); 9077 9078 // Check the destination VM has the expected 'concole_text' from its output 9079 let r = std::panic::catch_unwind(|| { 9080 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9081 }); 9082 handle_child_output(r, &dest_output); 9083 } 9084 9085 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 9086 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9087 let ovs_guest = Guest::new(Box::new(ovs_focal)); 9088 9089 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9090 let migration_guest = Guest::new(Box::new(migration_focal)); 9091 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 9092 9093 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 9094 let (mut ovs_child, mut src_child) = 9095 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 9096 9097 // Start the destination VM 9098 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 9099 dest_api_socket.push_str(".dest"); 9100 let mut dest_child = GuestCommand::new(&migration_guest) 9101 .args(["--api-socket", &dest_api_socket]) 9102 .capture_output() 9103 .spawn() 9104 .unwrap(); 9105 9106 let r = std::panic::catch_unwind(|| { 9107 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 9108 thread::sleep(std::time::Duration::new(1, 0)); 9109 9110 // Start the live-migration 9111 let migration_socket = String::from( 9112 migration_guest 9113 .tmp_dir 9114 .as_path() 9115 .join("live-migration.sock") 9116 .to_str() 9117 .unwrap(), 9118 ); 9119 9120 assert!( 9121 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9122 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9123 ); 9124 }); 9125 9126 // Check and report any errors occured during the live-migration 9127 if r.is_err() { 9128 print_and_panic( 9129 src_child, 9130 dest_child, 9131 Some(ovs_child), 9132 "Error occured during live-migration", 9133 ); 9134 } 9135 9136 // Check the source vm has been terminated successful (give it '3s' to settle) 9137 thread::sleep(std::time::Duration::new(3, 0)); 9138 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9139 print_and_panic( 9140 src_child, 9141 dest_child, 9142 Some(ovs_child), 9143 "source VM was not terminated successfully.", 9144 ); 9145 }; 9146 9147 // Post live-migration check to make sure the destination VM is funcational 9148 let r = std::panic::catch_unwind(|| { 9149 // Perform same checks to validate VM has been properly migrated 9150 // Spawn a new netcat listener in the OVS VM 9151 let guest_ip = ovs_guest.network.guest_ip.clone(); 9152 thread::spawn(move || { 9153 ssh_command_ip( 9154 "nc -l 12345", 9155 &guest_ip, 9156 DEFAULT_SSH_RETRIES, 9157 DEFAULT_SSH_TIMEOUT, 9158 ) 9159 .unwrap(); 9160 }); 9161 9162 // Wait for the server to be listening 9163 thread::sleep(std::time::Duration::new(5, 0)); 9164 9165 // And check the connection is still functional after live-migration 9166 migration_guest 9167 .ssh_command("nc -vz 172.100.0.1 12345") 9168 .unwrap(); 9169 }); 9170 9171 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 9172 let _ = dest_child.kill(); 9173 let _ = ovs_child.kill(); 9174 let dest_output = dest_child.wait_with_output().unwrap(); 9175 let ovs_output = ovs_child.wait_with_output().unwrap(); 9176 9177 cleanup_ovs_dpdk(); 9178 9179 handle_child_output(r, &dest_output); 9180 handle_child_output(Ok(()), &ovs_output); 9181 } 9182 9183 mod live_migration_parallel { 9184 use super::*; 9185 #[test] 9186 fn test_live_migration_basic() { 9187 _test_live_migration(false, false) 9188 } 9189 9190 #[test] 9191 fn test_live_migration_local() { 9192 _test_live_migration(false, true) 9193 } 9194 9195 #[test] 9196 #[cfg(not(feature = "mshv"))] 9197 fn test_live_migration_numa() { 9198 _test_live_migration_numa(false, false) 9199 } 9200 9201 #[test] 9202 #[cfg(not(feature = "mshv"))] 9203 fn test_live_migration_numa_local() { 9204 _test_live_migration_numa(false, true) 9205 } 9206 9207 #[test] 9208 fn test_live_migration_watchdog() { 9209 _test_live_migration_watchdog(false, false) 9210 } 9211 9212 #[test] 9213 fn test_live_migration_watchdog_local() { 9214 _test_live_migration_watchdog(false, true) 9215 } 9216 9217 #[test] 9218 fn test_live_migration_balloon() { 9219 _test_live_migration_balloon(false, false) 9220 } 9221 9222 #[test] 9223 fn test_live_migration_balloon_local() { 9224 _test_live_migration_balloon(false, true) 9225 } 9226 9227 #[test] 9228 #[ignore] 9229 fn test_live_upgrade_basic() { 9230 _test_live_migration(true, false) 9231 } 9232 9233 #[test] 9234 #[ignore] 9235 fn test_live_upgrade_local() { 9236 _test_live_migration(true, true) 9237 } 9238 9239 #[test] 9240 #[ignore] 9241 #[cfg(not(feature = "mshv"))] 9242 fn test_live_upgrade_numa() { 9243 _test_live_migration_numa(true, false) 9244 } 9245 9246 #[test] 9247 #[ignore] 9248 #[cfg(not(feature = "mshv"))] 9249 fn test_live_upgrade_numa_local() { 9250 _test_live_migration_numa(true, true) 9251 } 9252 9253 #[test] 9254 #[ignore] 9255 fn test_live_upgrade_watchdog() { 9256 _test_live_migration_watchdog(true, false) 9257 } 9258 9259 #[test] 9260 #[ignore] 9261 fn test_live_upgrade_watchdog_local() { 9262 _test_live_migration_watchdog(true, true) 9263 } 9264 9265 #[test] 9266 #[ignore] 9267 fn test_live_upgrade_balloon() { 9268 _test_live_migration_balloon(true, false) 9269 } 9270 9271 #[test] 9272 #[ignore] 9273 fn test_live_upgrade_balloon_local() { 9274 _test_live_migration_balloon(true, true) 9275 } 9276 } 9277 9278 mod live_migration_sequential { 9279 #[cfg(target_arch = "x86_64")] 9280 #[cfg(not(feature = "mshv"))] 9281 use super::*; 9282 9283 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 9284 #[test] 9285 #[cfg(target_arch = "x86_64")] 9286 #[cfg(not(feature = "mshv"))] 9287 fn test_live_migration_ovs_dpdk() { 9288 _test_live_migration_ovs_dpdk(false, false); 9289 } 9290 9291 #[test] 9292 #[cfg(target_arch = "x86_64")] 9293 #[cfg(not(feature = "mshv"))] 9294 fn test_live_migration_ovs_dpdk_local() { 9295 _test_live_migration_ovs_dpdk(false, true); 9296 } 9297 9298 #[test] 9299 #[ignore] 9300 #[cfg(target_arch = "x86_64")] 9301 #[cfg(not(feature = "mshv"))] 9302 fn test_live_upgrade_ovs_dpdk() { 9303 _test_live_migration_ovs_dpdk(true, false); 9304 } 9305 9306 #[test] 9307 #[ignore] 9308 #[cfg(target_arch = "x86_64")] 9309 #[cfg(not(feature = "mshv"))] 9310 fn test_live_upgrade_ovs_dpdk_local() { 9311 _test_live_migration_ovs_dpdk(true, true); 9312 } 9313 } 9314 } 9315 9316 #[cfg(target_arch = "aarch64")] 9317 mod aarch64_acpi { 9318 use crate::*; 9319 9320 #[test] 9321 fn test_simple_launch_acpi() { 9322 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9323 9324 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 9325 let guest = Guest::new(disk_config); 9326 9327 let mut child = GuestCommand::new(&guest) 9328 .args(["--cpus", "boot=1"]) 9329 .args(["--memory", "size=512M"]) 9330 .args(["--kernel", edk2_path().to_str().unwrap()]) 9331 .default_disks() 9332 .default_net() 9333 .args(["--serial", "tty", "--console", "off"]) 9334 .capture_output() 9335 .spawn() 9336 .unwrap(); 9337 9338 let r = std::panic::catch_unwind(|| { 9339 guest.wait_vm_boot(Some(120)).unwrap(); 9340 9341 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 9342 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 9343 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 9344 }); 9345 9346 let _ = child.kill(); 9347 let output = child.wait_with_output().unwrap(); 9348 9349 handle_child_output(r, &output); 9350 }); 9351 } 9352 9353 #[test] 9354 fn test_guest_numa_nodes_acpi() { 9355 _test_guest_numa_nodes(true); 9356 } 9357 9358 #[test] 9359 fn test_cpu_topology_421_acpi() { 9360 test_cpu_topology(4, 2, 1, true); 9361 } 9362 9363 #[test] 9364 fn test_cpu_topology_142_acpi() { 9365 test_cpu_topology(1, 4, 2, true); 9366 } 9367 9368 #[test] 9369 fn test_cpu_topology_262_acpi() { 9370 test_cpu_topology(2, 6, 2, true); 9371 } 9372 9373 #[test] 9374 fn test_power_button_acpi() { 9375 _test_power_button(true); 9376 } 9377 9378 #[test] 9379 fn test_virtio_iommu() { 9380 _test_virtio_iommu(true) 9381 } 9382 } 9383 9384 mod rate_limiter { 9385 use super::*; 9386 9387 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 9388 // compared to given 'limit' rate. 9389 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 9390 let upper_limit = limit * (1_f64 + difference); 9391 let lower_limit = limit * (1_f64 - difference); 9392 9393 if measured > lower_limit && measured < upper_limit { 9394 return true; 9395 } 9396 9397 eprintln!( 9398 "\n\n==== check_rate_limit failed! ====\n\nmeasured={measured}, , lower_limit={lower_limit}, upper_limit={upper_limit}\n\n" 9399 ); 9400 9401 false 9402 } 9403 9404 fn _test_rate_limiter_net(rx: bool) { 9405 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9406 let guest = Guest::new(Box::new(focal)); 9407 9408 let test_timeout = 10; 9409 let num_queues = 2; 9410 let queue_size = 256; 9411 let bw_size = 10485760_u64; // bytes 9412 let bw_refill_time = 100; // ms 9413 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 9414 9415 let net_params = format!( 9416 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 9417 guest.network.guest_mac, 9418 guest.network.host_ip, 9419 num_queues, 9420 queue_size, 9421 bw_size, 9422 bw_refill_time, 9423 ); 9424 9425 let mut child = GuestCommand::new(&guest) 9426 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 9427 .args(["--memory", "size=4G"]) 9428 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9429 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9430 .default_disks() 9431 .args(["--net", net_params.as_str()]) 9432 .capture_output() 9433 .spawn() 9434 .unwrap(); 9435 9436 let r = std::panic::catch_unwind(|| { 9437 guest.wait_vm_boot(None).unwrap(); 9438 let measured_bps = 9439 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx).unwrap(); 9440 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 9441 }); 9442 9443 let _ = child.kill(); 9444 let output = child.wait_with_output().unwrap(); 9445 handle_child_output(r, &output); 9446 } 9447 9448 #[test] 9449 fn test_rate_limiter_net_rx() { 9450 _test_rate_limiter_net(true); 9451 } 9452 9453 #[test] 9454 fn test_rate_limiter_net_tx() { 9455 _test_rate_limiter_net(false); 9456 } 9457 9458 fn _test_rate_limiter_block(bandwidth: bool) { 9459 let test_timeout = 10; 9460 let num_queues = 1; 9461 let fio_ops = FioOps::RandRW; 9462 9463 let bw_size = if bandwidth { 9464 10485760_u64 // bytes 9465 } else { 9466 100_u64 // I/O 9467 }; 9468 let bw_refill_time = 100; // ms 9469 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 9470 9471 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9472 let guest = Guest::new(Box::new(focal)); 9473 let api_socket = temp_api_path(&guest.tmp_dir); 9474 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 9475 let blk_rate_limiter_test_img = 9476 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 9477 9478 // Create the test block image 9479 assert!(exec_host_command_output(&format!( 9480 "dd if=/dev/zero of={blk_rate_limiter_test_img} bs=1M count=1024" 9481 )) 9482 .status 9483 .success()); 9484 9485 let test_blk_params = if bandwidth { 9486 format!( 9487 "path={blk_rate_limiter_test_img},bw_size={bw_size},bw_refill_time={bw_refill_time}" 9488 ) 9489 } else { 9490 format!( 9491 "path={blk_rate_limiter_test_img},ops_size={bw_size},ops_refill_time={bw_refill_time}" 9492 ) 9493 }; 9494 9495 let mut child = GuestCommand::new(&guest) 9496 .args(["--cpus", &format!("boot={num_queues}")]) 9497 .args(["--memory", "size=4G"]) 9498 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9499 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9500 .args([ 9501 "--disk", 9502 format!( 9503 "path={}", 9504 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 9505 ) 9506 .as_str(), 9507 format!( 9508 "path={}", 9509 guest.disk_config.disk(DiskType::CloudInit).unwrap() 9510 ) 9511 .as_str(), 9512 test_blk_params.as_str(), 9513 ]) 9514 .default_net() 9515 .args(["--api-socket", &api_socket]) 9516 .capture_output() 9517 .spawn() 9518 .unwrap(); 9519 9520 let r = std::panic::catch_unwind(|| { 9521 guest.wait_vm_boot(None).unwrap(); 9522 9523 let fio_command = format!( 9524 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 9525 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 9526 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 9527 ); 9528 let output = guest.ssh_command(&fio_command).unwrap(); 9529 9530 // Parse fio output 9531 let measured_rate = if bandwidth { 9532 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 9533 } else { 9534 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 9535 }; 9536 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 9537 }); 9538 9539 let _ = child.kill(); 9540 let output = child.wait_with_output().unwrap(); 9541 handle_child_output(r, &output); 9542 } 9543 9544 #[test] 9545 fn test_rate_limiter_block_bandwidth() { 9546 _test_rate_limiter_block(true) 9547 } 9548 9549 #[test] 9550 fn test_rate_limiter_block_iops() { 9551 _test_rate_limiter_block(false) 9552 } 9553 } 9554