1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 use std::sync::Mutex; 28 use std::thread; 29 use test_infra::*; 30 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 31 use wait_timeout::ChildExt; 32 33 #[cfg(target_arch = "x86_64")] 34 mod x86_64 { 35 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw"; 36 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 37 pub const FOCAL_SGX_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-sgx.raw"; 38 pub const JAMMY_NVIDIA_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-nvidia.raw"; 39 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 40 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 41 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 42 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20221118-1.raw"; 43 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2019.raw"; 44 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 45 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 46 } 47 48 #[cfg(target_arch = "x86_64")] 49 use x86_64::*; 50 51 #[cfg(target_arch = "aarch64")] 52 mod aarch64 { 53 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 54 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 55 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 56 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 57 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 58 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 59 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 60 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 61 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 62 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 63 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 64 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 65 } 66 67 #[cfg(target_arch = "aarch64")] 68 use aarch64::*; 69 70 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 71 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 72 73 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 74 75 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 76 let mut workload_path = dirs::home_dir().unwrap(); 77 workload_path.push("workloads"); 78 79 let mut virtiofsd_path = workload_path; 80 virtiofsd_path.push("virtiofsd"); 81 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 82 83 let virtiofsd_socket_path = 84 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 85 86 // Start the daemon 87 let child = Command::new(virtiofsd_path.as_str()) 88 .args(["--shared-dir", shared_dir]) 89 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 90 .args(["--cache", "never"]) 91 .spawn() 92 .unwrap(); 93 94 thread::sleep(std::time::Duration::new(10, 0)); 95 96 (child, virtiofsd_socket_path) 97 } 98 99 fn prepare_vubd( 100 tmp_dir: &TempDir, 101 blk_img: &str, 102 num_queues: usize, 103 rdonly: bool, 104 direct: bool, 105 ) -> (std::process::Child, String) { 106 let mut workload_path = dirs::home_dir().unwrap(); 107 workload_path.push("workloads"); 108 109 let mut blk_file_path = workload_path; 110 blk_file_path.push(blk_img); 111 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 112 113 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 114 115 // Start the daemon 116 let child = Command::new(clh_command("vhost_user_block")) 117 .args([ 118 "--block-backend", 119 format!( 120 "path={},socket={},num_queues={},readonly={},direct={}", 121 blk_file_path, vubd_socket_path, num_queues, rdonly, direct 122 ) 123 .as_str(), 124 ]) 125 .spawn() 126 .unwrap(); 127 128 thread::sleep(std::time::Duration::new(10, 0)); 129 130 (child, vubd_socket_path) 131 } 132 133 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 134 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 135 } 136 137 fn temp_api_path(tmp_dir: &TempDir) -> String { 138 String::from( 139 tmp_dir 140 .as_path() 141 .join("cloud-hypervisor.sock") 142 .to_str() 143 .unwrap(), 144 ) 145 } 146 147 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 148 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 149 } 150 151 // Creates the directory and returns the path. 152 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 153 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 154 std::fs::create_dir(&snapshot_dir).unwrap(); 155 snapshot_dir 156 } 157 158 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 159 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 160 vmcore_file 161 } 162 163 // Creates the path for direct kernel boot and return the path. 164 // For x86_64, this function returns the vmlinux kernel path. 165 // For AArch64, this function returns the PE kernel path. 166 fn direct_kernel_boot_path() -> PathBuf { 167 let mut workload_path = dirs::home_dir().unwrap(); 168 workload_path.push("workloads"); 169 170 let mut kernel_path = workload_path; 171 #[cfg(target_arch = "x86_64")] 172 kernel_path.push("vmlinux"); 173 #[cfg(target_arch = "aarch64")] 174 kernel_path.push("Image"); 175 176 kernel_path 177 } 178 179 fn edk2_path() -> PathBuf { 180 let mut workload_path = dirs::home_dir().unwrap(); 181 workload_path.push("workloads"); 182 let mut edk2_path = workload_path; 183 edk2_path.push(OVMF_NAME); 184 185 edk2_path 186 } 187 188 fn cloud_hypervisor_release_path() -> String { 189 let mut workload_path = dirs::home_dir().unwrap(); 190 workload_path.push("workloads"); 191 192 let mut ch_release_path = workload_path; 193 #[cfg(target_arch = "x86_64")] 194 ch_release_path.push("cloud-hypervisor-static"); 195 #[cfg(target_arch = "aarch64")] 196 ch_release_path.push("cloud-hypervisor-static-aarch64"); 197 198 ch_release_path.into_os_string().into_string().unwrap() 199 } 200 201 fn prepare_vhost_user_net_daemon( 202 tmp_dir: &TempDir, 203 ip: &str, 204 tap: Option<&str>, 205 mtu: Option<u16>, 206 num_queues: usize, 207 client_mode: bool, 208 ) -> (std::process::Command, String) { 209 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 210 211 // Start the daemon 212 let mut net_params = format!( 213 "ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 214 ip, vunet_socket_path, num_queues, client_mode 215 ); 216 217 if let Some(tap) = tap { 218 net_params.push_str(format!(",tap={}", tap).as_str()); 219 } 220 221 if let Some(mtu) = mtu { 222 net_params.push_str(format!(",mtu={}", mtu).as_str()); 223 } 224 225 let mut command = Command::new(clh_command("vhost_user_net")); 226 command.args(["--net-backend", net_params.as_str()]); 227 228 (command, vunet_socket_path) 229 } 230 231 fn curl_command(api_socket: &str, method: &str, url: &str, http_body: Option<&str>) { 232 let mut curl_args: Vec<&str> = ["--unix-socket", api_socket, "-i", "-X", method, url].to_vec(); 233 234 if let Some(body) = http_body { 235 curl_args.push("-H"); 236 curl_args.push("Accept: application/json"); 237 curl_args.push("-H"); 238 curl_args.push("Content-Type: application/json"); 239 curl_args.push("-d"); 240 curl_args.push(body); 241 } 242 243 let status = Command::new("curl") 244 .args(curl_args) 245 .status() 246 .expect("Failed to launch curl command"); 247 248 assert!(status.success()); 249 } 250 251 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 252 let mut cmd = Command::new(clh_command("ch-remote")); 253 cmd.args([&format!("--api-socket={}", api_socket), command]); 254 255 if let Some(arg) = arg { 256 cmd.arg(arg); 257 } 258 let output = cmd.output().unwrap(); 259 if output.status.success() { 260 true 261 } else { 262 eprintln!("Error running ch-remote command: {:?}", &cmd); 263 let stderr = String::from_utf8_lossy(&output.stderr); 264 eprintln!("stderr: {}", stderr); 265 false 266 } 267 } 268 269 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 270 let mut cmd = Command::new(clh_command("ch-remote")); 271 cmd.args([&format!("--api-socket={}", api_socket), command]); 272 273 if let Some(arg) = arg { 274 cmd.arg(arg); 275 } 276 277 let output = cmd.output().expect("Failed to launch ch-remote"); 278 279 (output.status.success(), output.stdout) 280 } 281 282 fn resize_command( 283 api_socket: &str, 284 desired_vcpus: Option<u8>, 285 desired_ram: Option<usize>, 286 desired_balloon: Option<usize>, 287 event_file: Option<&str>, 288 ) -> bool { 289 let mut cmd = Command::new(clh_command("ch-remote")); 290 cmd.args([&format!("--api-socket={}", api_socket), "resize"]); 291 292 if let Some(desired_vcpus) = desired_vcpus { 293 cmd.arg(format!("--cpus={}", desired_vcpus)); 294 } 295 296 if let Some(desired_ram) = desired_ram { 297 cmd.arg(format!("--memory={}", desired_ram)); 298 } 299 300 if let Some(desired_balloon) = desired_balloon { 301 cmd.arg(format!("--balloon={}", desired_balloon)); 302 } 303 304 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 305 306 if let Some(event_path) = event_file { 307 let latest_events = [ 308 &MetaEvent { 309 event: "resizing".to_string(), 310 device_id: None, 311 }, 312 &MetaEvent { 313 event: "resized".to_string(), 314 device_id: None, 315 }, 316 ]; 317 assert!(check_latest_events_exact(&latest_events, event_path)); 318 } 319 320 ret 321 } 322 323 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 324 let mut cmd = Command::new(clh_command("ch-remote")); 325 cmd.args([ 326 &format!("--api-socket={}", api_socket), 327 "resize-zone", 328 &format!("--id={}", id), 329 &format!("--size={}", desired_size), 330 ]); 331 332 cmd.status().expect("Failed to launch ch-remote").success() 333 } 334 335 // setup OVS-DPDK bridge and ports 336 fn setup_ovs_dpdk() { 337 // setup OVS-DPDK 338 assert!(exec_host_command_status("service openvswitch-switch start").success()); 339 assert!(exec_host_command_status("ovs-vsctl init").success()); 340 assert!( 341 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 342 .success() 343 ); 344 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 345 346 // Create OVS-DPDK bridge and ports 347 assert!(exec_host_command_status( 348 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 349 ) 350 .success()); 351 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 352 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 353 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 354 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 355 } 356 fn cleanup_ovs_dpdk() { 357 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 358 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 359 } 360 // Setup two guests and ensure they are connected through ovs-dpdk 361 fn setup_ovs_dpdk_guests( 362 guest1: &Guest, 363 guest2: &Guest, 364 api_socket: &str, 365 release_binary: bool, 366 ) -> (Child, Child) { 367 setup_ovs_dpdk(); 368 369 let clh_path = if !release_binary { 370 clh_command("cloud-hypervisor") 371 } else { 372 cloud_hypervisor_release_path() 373 }; 374 375 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 376 .args(["--cpus", "boot=2"]) 377 .args(["--memory", "size=0,shared=on"]) 378 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 379 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 380 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 381 .default_disks() 382 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 383 .capture_output() 384 .spawn() 385 .unwrap(); 386 387 #[cfg(target_arch = "x86_64")] 388 let guest_net_iface = "ens5"; 389 #[cfg(target_arch = "aarch64")] 390 let guest_net_iface = "enp0s5"; 391 392 let r = std::panic::catch_unwind(|| { 393 guest1.wait_vm_boot(None).unwrap(); 394 395 guest1 396 .ssh_command(&format!( 397 "sudo ip addr add 172.100.0.1/24 dev {}", 398 guest_net_iface 399 )) 400 .unwrap(); 401 guest1 402 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 403 .unwrap(); 404 405 let guest_ip = guest1.network.guest_ip.clone(); 406 thread::spawn(move || { 407 ssh_command_ip( 408 "nc -l 12345", 409 &guest_ip, 410 DEFAULT_SSH_RETRIES, 411 DEFAULT_SSH_TIMEOUT, 412 ) 413 .unwrap(); 414 }); 415 }); 416 if r.is_err() { 417 cleanup_ovs_dpdk(); 418 419 let _ = child1.kill(); 420 let output = child1.wait_with_output().unwrap(); 421 handle_child_output(r, &output); 422 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 423 } 424 425 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 426 .args(["--api-socket", api_socket]) 427 .args(["--cpus", "boot=2"]) 428 .args(["--memory", "size=0,shared=on"]) 429 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 430 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 431 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 432 .default_disks() 433 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 434 .capture_output() 435 .spawn() 436 .unwrap(); 437 438 let r = std::panic::catch_unwind(|| { 439 guest2.wait_vm_boot(None).unwrap(); 440 441 guest2 442 .ssh_command(&format!( 443 "sudo ip addr add 172.100.0.2/24 dev {}", 444 guest_net_iface 445 )) 446 .unwrap(); 447 guest2 448 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 449 .unwrap(); 450 451 // Check the connection works properly between the two VMs 452 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 453 }); 454 if r.is_err() { 455 cleanup_ovs_dpdk(); 456 457 let _ = child1.kill(); 458 let _ = child2.kill(); 459 let output = child2.wait_with_output().unwrap(); 460 handle_child_output(r, &output); 461 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 462 } 463 464 (child1, child2) 465 } 466 467 enum FwType { 468 Ovmf, 469 RustHypervisorFirmware, 470 } 471 472 fn fw_path(_fw_type: FwType) -> String { 473 let mut workload_path = dirs::home_dir().unwrap(); 474 workload_path.push("workloads"); 475 476 let mut fw_path = workload_path; 477 #[cfg(target_arch = "aarch64")] 478 fw_path.push("CLOUDHV_EFI.fd"); 479 #[cfg(target_arch = "x86_64")] 480 { 481 match _fw_type { 482 FwType::Ovmf => fw_path.push(OVMF_NAME), 483 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 484 } 485 } 486 487 fw_path.to_str().unwrap().to_string() 488 } 489 490 struct MetaEvent { 491 event: String, 492 device_id: Option<String>, 493 } 494 495 impl MetaEvent { 496 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 497 let mut matched = false; 498 if v["event"].as_str().unwrap() == self.event { 499 if let Some(device_id) = &self.device_id { 500 if v["properties"]["id"].as_str().unwrap() == device_id { 501 matched = true 502 } 503 } else { 504 matched = true; 505 } 506 } 507 matched 508 } 509 } 510 511 // Parse the event_monitor file based on the format that each event 512 // is followed by a double newline 513 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 514 let content = fs::read(event_file).unwrap(); 515 let mut ret = Vec::new(); 516 for entry in String::from_utf8_lossy(&content) 517 .trim() 518 .split("\n\n") 519 .collect::<Vec<&str>>() 520 { 521 ret.push(serde_json::from_str(entry).unwrap()); 522 } 523 524 ret 525 } 526 527 // Return true if all events from the input 'expected_events' are matched sequentially 528 // with events from the 'event_file' 529 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 530 let json_events = parse_event_file(event_file); 531 let len = expected_events.len(); 532 let mut idx = 0; 533 for e in &json_events { 534 if idx == len { 535 break; 536 } 537 if expected_events[idx].match_with_json_event(e) { 538 idx += 1; 539 } 540 } 541 542 idx == len 543 } 544 545 // Return true if all events from the input 'expected_events' are matched exactly 546 // with events from the 'event_file' 547 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 548 let json_events = parse_event_file(event_file); 549 assert!(expected_events.len() <= json_events.len()); 550 let json_events = &json_events[..expected_events.len()]; 551 552 for (idx, e) in json_events.iter().enumerate() { 553 if !expected_events[idx].match_with_json_event(e) { 554 return false; 555 } 556 } 557 558 true 559 } 560 561 // Return true if events from the input 'expected_events' are matched exactly 562 // with the most recent events from the 'event_file' 563 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 564 let json_events = parse_event_file(event_file); 565 assert!(latest_events.len() <= json_events.len()); 566 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 567 568 for (idx, e) in json_events.iter().enumerate() { 569 if !latest_events[idx].match_with_json_event(e) { 570 return false; 571 } 572 } 573 574 true 575 } 576 577 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 578 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 579 let guest = Guest::new(Box::new(focal)); 580 let total_vcpus = threads_per_core * cores_per_package * packages; 581 let direct_kernel_boot_path = direct_kernel_boot_path(); 582 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 583 let fw_path = fw_path(FwType::RustHypervisorFirmware); 584 if use_fw { 585 kernel_path = fw_path.as_str(); 586 } 587 588 let mut child = GuestCommand::new(&guest) 589 .args([ 590 "--cpus", 591 &format!( 592 "boot={},topology={}:{}:1:{}", 593 total_vcpus, threads_per_core, cores_per_package, packages 594 ), 595 ]) 596 .args(["--memory", "size=512M"]) 597 .args(["--kernel", kernel_path]) 598 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 599 .default_disks() 600 .default_net() 601 .capture_output() 602 .spawn() 603 .unwrap(); 604 605 let r = std::panic::catch_unwind(|| { 606 guest.wait_vm_boot(None).unwrap(); 607 assert_eq!( 608 guest.get_cpu_count().unwrap_or_default(), 609 u32::from(total_vcpus) 610 ); 611 assert_eq!( 612 guest 613 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 614 .unwrap() 615 .trim() 616 .parse::<u8>() 617 .unwrap_or(0), 618 threads_per_core 619 ); 620 621 assert_eq!( 622 guest 623 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 624 .unwrap() 625 .trim() 626 .parse::<u8>() 627 .unwrap_or(0), 628 cores_per_package 629 ); 630 631 assert_eq!( 632 guest 633 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 634 .unwrap() 635 .trim() 636 .parse::<u8>() 637 .unwrap_or(0), 638 packages 639 ); 640 }); 641 642 let _ = child.kill(); 643 let output = child.wait_with_output().unwrap(); 644 645 handle_child_output(r, &output); 646 } 647 648 #[allow(unused_variables)] 649 fn _test_guest_numa_nodes(acpi: bool) { 650 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 651 let guest = Guest::new(Box::new(focal)); 652 let api_socket = temp_api_path(&guest.tmp_dir); 653 #[cfg(target_arch = "x86_64")] 654 let kernel_path = direct_kernel_boot_path(); 655 #[cfg(target_arch = "aarch64")] 656 let kernel_path = if acpi { 657 edk2_path() 658 } else { 659 direct_kernel_boot_path() 660 }; 661 662 let mut child = GuestCommand::new(&guest) 663 .args(["--cpus", "boot=6,max=12"]) 664 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 665 .args([ 666 "--memory-zone", 667 "id=mem0,size=1G,hotplug_size=3G", 668 "id=mem1,size=2G,hotplug_size=3G", 669 "id=mem2,size=3G,hotplug_size=3G", 670 ]) 671 .args([ 672 "--numa", 673 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 674 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 675 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 676 ]) 677 .args(["--kernel", kernel_path.to_str().unwrap()]) 678 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 679 .args(["--api-socket", &api_socket]) 680 .capture_output() 681 .default_disks() 682 .default_net() 683 .spawn() 684 .unwrap(); 685 686 let r = std::panic::catch_unwind(|| { 687 guest.wait_vm_boot(None).unwrap(); 688 689 guest.check_numa_common( 690 Some(&[960_000, 1_920_000, 2_880_000]), 691 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 692 Some(&["10 15 20", "20 10 25", "25 30 10"]), 693 ); 694 695 // AArch64 currently does not support hotplug, and therefore we only 696 // test hotplug-related function on x86_64 here. 697 #[cfg(target_arch = "x86_64")] 698 { 699 guest.enable_memory_hotplug(); 700 701 // Resize every memory zone and check each associated NUMA node 702 // has been assigned the right amount of memory. 703 resize_zone_command(&api_socket, "mem0", "4G"); 704 resize_zone_command(&api_socket, "mem1", "4G"); 705 resize_zone_command(&api_socket, "mem2", "4G"); 706 // Resize to the maximum amount of CPUs and check each NUMA 707 // node has been assigned the right CPUs set. 708 resize_command(&api_socket, Some(12), None, None, None); 709 thread::sleep(std::time::Duration::new(5, 0)); 710 711 guest.check_numa_common( 712 Some(&[3_840_000, 3_840_000, 3_840_000]), 713 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 714 None, 715 ); 716 } 717 }); 718 719 let _ = child.kill(); 720 let output = child.wait_with_output().unwrap(); 721 722 handle_child_output(r, &output); 723 } 724 725 #[allow(unused_variables)] 726 fn _test_power_button(acpi: bool) { 727 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 728 let guest = Guest::new(Box::new(focal)); 729 let mut cmd = GuestCommand::new(&guest); 730 let api_socket = temp_api_path(&guest.tmp_dir); 731 732 #[cfg(target_arch = "x86_64")] 733 let kernel_path = direct_kernel_boot_path(); 734 #[cfg(target_arch = "aarch64")] 735 let kernel_path = if acpi { 736 edk2_path() 737 } else { 738 direct_kernel_boot_path() 739 }; 740 741 cmd.args(["--cpus", "boot=1"]) 742 .args(["--memory", "size=512M"]) 743 .args(["--kernel", kernel_path.to_str().unwrap()]) 744 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 745 .capture_output() 746 .default_disks() 747 .default_net() 748 .args(["--api-socket", &api_socket]); 749 750 let child = cmd.spawn().unwrap(); 751 752 let r = std::panic::catch_unwind(|| { 753 guest.wait_vm_boot(None).unwrap(); 754 assert!(remote_command(&api_socket, "power-button", None)); 755 }); 756 757 let output = child.wait_with_output().unwrap(); 758 assert!(output.status.success()); 759 handle_child_output(r, &output); 760 } 761 762 type PrepareNetDaemon = dyn Fn( 763 &TempDir, 764 &str, 765 Option<&str>, 766 Option<u16>, 767 usize, 768 bool, 769 ) -> (std::process::Command, String); 770 771 fn test_vhost_user_net( 772 tap: Option<&str>, 773 num_queues: usize, 774 prepare_daemon: &PrepareNetDaemon, 775 generate_host_mac: bool, 776 client_mode_daemon: bool, 777 ) { 778 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 779 let guest = Guest::new(Box::new(focal)); 780 let api_socket = temp_api_path(&guest.tmp_dir); 781 782 let kernel_path = direct_kernel_boot_path(); 783 784 let host_mac = if generate_host_mac { 785 Some(MacAddr::local_random()) 786 } else { 787 None 788 }; 789 790 let mtu = Some(3000); 791 792 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 793 &guest.tmp_dir, 794 &guest.network.host_ip, 795 tap, 796 mtu, 797 num_queues, 798 client_mode_daemon, 799 ); 800 801 let net_params = format!( 802 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 803 guest.network.guest_mac, 804 vunet_socket_path, 805 num_queues, 806 if let Some(host_mac) = host_mac { 807 format!(",host_mac={}", host_mac) 808 } else { 809 "".to_owned() 810 }, 811 if client_mode_daemon { 812 "server" 813 } else { 814 "client" 815 }, 816 ); 817 818 let mut ch_command = GuestCommand::new(&guest); 819 ch_command 820 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 821 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 822 .args(["--kernel", kernel_path.to_str().unwrap()]) 823 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 824 .default_disks() 825 .args(["--net", net_params.as_str()]) 826 .args(["--api-socket", &api_socket]) 827 .capture_output(); 828 829 let mut daemon_child: std::process::Child; 830 let mut child: std::process::Child; 831 832 if client_mode_daemon { 833 child = ch_command.spawn().unwrap(); 834 // Make sure the VMM is waiting for the backend to connect 835 thread::sleep(std::time::Duration::new(10, 0)); 836 daemon_child = daemon_command.spawn().unwrap(); 837 } else { 838 daemon_child = daemon_command.spawn().unwrap(); 839 // Make sure the backend is waiting for the VMM to connect 840 thread::sleep(std::time::Duration::new(10, 0)); 841 child = ch_command.spawn().unwrap(); 842 } 843 844 let r = std::panic::catch_unwind(|| { 845 guest.wait_vm_boot(None).unwrap(); 846 847 if let Some(tap_name) = tap { 848 let tap_count = exec_host_command_output(&format!("ip link | grep -c {}", tap_name)); 849 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 850 } 851 852 if let Some(host_mac) = tap { 853 let mac_count = exec_host_command_output(&format!("ip link | grep -c {}", host_mac)); 854 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 855 } 856 857 #[cfg(target_arch = "aarch64")] 858 let iface = "enp0s4"; 859 #[cfg(target_arch = "x86_64")] 860 let iface = "ens4"; 861 862 assert_eq!( 863 guest 864 .ssh_command(format!("cat /sys/class/net/{}/mtu", iface).as_str()) 865 .unwrap() 866 .trim(), 867 "3000" 868 ); 869 870 // 1 network interface + default localhost ==> 2 interfaces 871 // It's important to note that this test is fully exercising the 872 // vhost-user-net implementation and the associated backend since 873 // it does not define any --net network interface. That means all 874 // the ssh communication in that test happens through the network 875 // interface backed by vhost-user-net. 876 assert_eq!( 877 guest 878 .ssh_command("ip -o link | wc -l") 879 .unwrap() 880 .trim() 881 .parse::<u32>() 882 .unwrap_or_default(), 883 2 884 ); 885 886 // The following pci devices will appear on guest with PCI-MSI 887 // interrupt vectors assigned. 888 // 1 virtio-console with 3 vectors: config, Rx, Tx 889 // 1 virtio-blk with 2 vectors: config, Request 890 // 1 virtio-blk with 2 vectors: config, Request 891 // 1 virtio-rng with 2 vectors: config, Request 892 // Since virtio-net has 2 queue pairs, its vectors is as follows: 893 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 894 // Based on the above, the total vectors should 14. 895 #[cfg(target_arch = "x86_64")] 896 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 897 #[cfg(target_arch = "aarch64")] 898 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 899 assert_eq!( 900 guest 901 .ssh_command(grep_cmd) 902 .unwrap() 903 .trim() 904 .parse::<u32>() 905 .unwrap_or_default(), 906 10 + (num_queues as u32) 907 ); 908 909 // ACPI feature is needed. 910 #[cfg(target_arch = "x86_64")] 911 { 912 guest.enable_memory_hotplug(); 913 914 // Add RAM to the VM 915 let desired_ram = 1024 << 20; 916 resize_command(&api_socket, None, Some(desired_ram), None, None); 917 918 thread::sleep(std::time::Duration::new(10, 0)); 919 920 // Here by simply checking the size (through ssh), we validate 921 // the connection is still working, which means vhost-user-net 922 // keeps working after the resize. 923 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 924 } 925 }); 926 927 let _ = child.kill(); 928 let output = child.wait_with_output().unwrap(); 929 930 thread::sleep(std::time::Duration::new(5, 0)); 931 let _ = daemon_child.kill(); 932 let _ = daemon_child.wait(); 933 934 handle_child_output(r, &output); 935 } 936 937 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 938 939 fn test_vhost_user_blk( 940 num_queues: usize, 941 readonly: bool, 942 direct: bool, 943 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 944 ) { 945 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 946 let guest = Guest::new(Box::new(focal)); 947 let api_socket = temp_api_path(&guest.tmp_dir); 948 949 let kernel_path = direct_kernel_boot_path(); 950 951 let (blk_params, daemon_child) = { 952 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 953 // Start the daemon 954 let (daemon_child, vubd_socket_path) = 955 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 956 957 ( 958 format!( 959 "vhost_user=true,socket={},num_queues={},queue_size=128", 960 vubd_socket_path, num_queues, 961 ), 962 Some(daemon_child), 963 ) 964 }; 965 966 let mut child = GuestCommand::new(&guest) 967 .args(["--cpus", format!("boot={}", num_queues).as_str()]) 968 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 969 .args(["--kernel", kernel_path.to_str().unwrap()]) 970 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 971 .args([ 972 "--disk", 973 format!( 974 "path={}", 975 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 976 ) 977 .as_str(), 978 format!( 979 "path={}", 980 guest.disk_config.disk(DiskType::CloudInit).unwrap() 981 ) 982 .as_str(), 983 blk_params.as_str(), 984 ]) 985 .default_net() 986 .args(["--api-socket", &api_socket]) 987 .capture_output() 988 .spawn() 989 .unwrap(); 990 991 let r = std::panic::catch_unwind(|| { 992 guest.wait_vm_boot(None).unwrap(); 993 994 // Check both if /dev/vdc exists and if the block size is 16M. 995 assert_eq!( 996 guest 997 .ssh_command("lsblk | grep vdc | grep -c 16M") 998 .unwrap() 999 .trim() 1000 .parse::<u32>() 1001 .unwrap_or_default(), 1002 1 1003 ); 1004 1005 // Check if this block is RO or RW. 1006 assert_eq!( 1007 guest 1008 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1009 .unwrap() 1010 .trim() 1011 .parse::<u32>() 1012 .unwrap_or_default(), 1013 readonly as u32 1014 ); 1015 1016 // Check if the number of queues in /sys/block/vdc/mq matches the 1017 // expected num_queues. 1018 assert_eq!( 1019 guest 1020 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1021 .unwrap() 1022 .trim() 1023 .parse::<u32>() 1024 .unwrap_or_default(), 1025 num_queues as u32 1026 ); 1027 1028 // Mount the device 1029 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1030 guest.ssh_command("mkdir mount_image").unwrap(); 1031 guest 1032 .ssh_command( 1033 format!( 1034 "sudo mount -o {} -t ext4 /dev/vdc mount_image/", 1035 mount_ro_rw_flag 1036 ) 1037 .as_str(), 1038 ) 1039 .unwrap(); 1040 1041 // Check the content of the block device. The file "foo" should 1042 // contain "bar". 1043 assert_eq!( 1044 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1045 "bar" 1046 ); 1047 1048 // ACPI feature is needed. 1049 #[cfg(target_arch = "x86_64")] 1050 { 1051 guest.enable_memory_hotplug(); 1052 1053 // Add RAM to the VM 1054 let desired_ram = 1024 << 20; 1055 resize_command(&api_socket, None, Some(desired_ram), None, None); 1056 1057 thread::sleep(std::time::Duration::new(10, 0)); 1058 1059 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1060 1061 // Check again the content of the block device after the resize 1062 // has been performed. 1063 assert_eq!( 1064 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1065 "bar" 1066 ); 1067 } 1068 1069 // Unmount the device 1070 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1071 guest.ssh_command("rm -r mount_image").unwrap(); 1072 }); 1073 1074 let _ = child.kill(); 1075 let output = child.wait_with_output().unwrap(); 1076 1077 if let Some(mut daemon_child) = daemon_child { 1078 thread::sleep(std::time::Duration::new(5, 0)); 1079 let _ = daemon_child.kill(); 1080 let _ = daemon_child.wait(); 1081 } 1082 1083 handle_child_output(r, &output); 1084 } 1085 1086 fn test_boot_from_vhost_user_blk( 1087 num_queues: usize, 1088 readonly: bool, 1089 direct: bool, 1090 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1091 ) { 1092 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1093 let guest = Guest::new(Box::new(focal)); 1094 1095 let kernel_path = direct_kernel_boot_path(); 1096 1097 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1098 1099 let (blk_boot_params, daemon_child) = { 1100 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1101 // Start the daemon 1102 let (daemon_child, vubd_socket_path) = prepare_daemon( 1103 &guest.tmp_dir, 1104 disk_path.as_str(), 1105 num_queues, 1106 readonly, 1107 direct, 1108 ); 1109 1110 ( 1111 format!( 1112 "vhost_user=true,socket={},num_queues={},queue_size=128", 1113 vubd_socket_path, num_queues, 1114 ), 1115 Some(daemon_child), 1116 ) 1117 }; 1118 1119 let mut child = GuestCommand::new(&guest) 1120 .args(["--cpus", format!("boot={}", num_queues).as_str()]) 1121 .args(["--memory", "size=512M,shared=on"]) 1122 .args(["--kernel", kernel_path.to_str().unwrap()]) 1123 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1124 .args([ 1125 "--disk", 1126 blk_boot_params.as_str(), 1127 format!( 1128 "path={}", 1129 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1130 ) 1131 .as_str(), 1132 ]) 1133 .default_net() 1134 .capture_output() 1135 .spawn() 1136 .unwrap(); 1137 1138 let r = std::panic::catch_unwind(|| { 1139 guest.wait_vm_boot(None).unwrap(); 1140 1141 // Just check the VM booted correctly. 1142 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1143 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1144 }); 1145 let _ = child.kill(); 1146 let output = child.wait_with_output().unwrap(); 1147 1148 if let Some(mut daemon_child) = daemon_child { 1149 thread::sleep(std::time::Duration::new(5, 0)); 1150 let _ = daemon_child.kill(); 1151 let _ = daemon_child.wait(); 1152 } 1153 1154 handle_child_output(r, &output); 1155 } 1156 1157 fn _test_virtio_fs( 1158 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1159 hotplug: bool, 1160 pci_segment: Option<u16>, 1161 ) { 1162 #[cfg(target_arch = "aarch64")] 1163 let focal_image = if hotplug { 1164 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1165 } else { 1166 FOCAL_IMAGE_NAME.to_string() 1167 }; 1168 #[cfg(target_arch = "x86_64")] 1169 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1170 let focal = UbuntuDiskConfig::new(focal_image); 1171 let guest = Guest::new(Box::new(focal)); 1172 let api_socket = temp_api_path(&guest.tmp_dir); 1173 1174 let mut workload_path = dirs::home_dir().unwrap(); 1175 workload_path.push("workloads"); 1176 1177 let mut shared_dir = workload_path; 1178 shared_dir.push("shared_dir"); 1179 1180 #[cfg(target_arch = "x86_64")] 1181 let kernel_path = direct_kernel_boot_path(); 1182 #[cfg(target_arch = "aarch64")] 1183 let kernel_path = if hotplug { 1184 edk2_path() 1185 } else { 1186 direct_kernel_boot_path() 1187 }; 1188 1189 let (mut daemon_child, virtiofsd_socket_path) = 1190 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1191 1192 let mut guest_command = GuestCommand::new(&guest); 1193 guest_command 1194 .args(["--cpus", "boot=1"]) 1195 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1196 .args(["--kernel", kernel_path.to_str().unwrap()]) 1197 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1198 .default_disks() 1199 .default_net() 1200 .args(["--api-socket", &api_socket]); 1201 if pci_segment.is_some() { 1202 guest_command.args(["--platform", "num_pci_segments=16"]); 1203 } 1204 1205 let fs_params = format!( 1206 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1207 virtiofsd_socket_path, 1208 if let Some(pci_segment) = pci_segment { 1209 format!(",pci_segment={}", pci_segment) 1210 } else { 1211 "".to_owned() 1212 } 1213 ); 1214 1215 if !hotplug { 1216 guest_command.args(["--fs", fs_params.as_str()]); 1217 } 1218 1219 let mut child = guest_command.capture_output().spawn().unwrap(); 1220 1221 let r = std::panic::catch_unwind(|| { 1222 guest.wait_vm_boot(None).unwrap(); 1223 1224 if hotplug { 1225 // Add fs to the VM 1226 let (cmd_success, cmd_output) = 1227 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1228 assert!(cmd_success); 1229 1230 if let Some(pci_segment) = pci_segment { 1231 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1232 "{{\"id\":\"myfs0\",\"bdf\":\"{:04x}:00:01.0\"}}", 1233 pci_segment 1234 ))); 1235 } else { 1236 assert!(String::from_utf8_lossy(&cmd_output) 1237 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1238 } 1239 1240 thread::sleep(std::time::Duration::new(10, 0)); 1241 } 1242 1243 // Mount shared directory through virtio_fs filesystem 1244 guest 1245 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1246 .unwrap(); 1247 1248 // Check file1 exists and its content is "foo" 1249 assert_eq!( 1250 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1251 "foo" 1252 ); 1253 // Check file2 does not exist 1254 guest 1255 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1256 .unwrap(); 1257 1258 // Check file3 exists and its content is "bar" 1259 assert_eq!( 1260 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1261 "bar" 1262 ); 1263 1264 // ACPI feature is needed. 1265 #[cfg(target_arch = "x86_64")] 1266 { 1267 guest.enable_memory_hotplug(); 1268 1269 // Add RAM to the VM 1270 let desired_ram = 1024 << 20; 1271 resize_command(&api_socket, None, Some(desired_ram), None, None); 1272 1273 thread::sleep(std::time::Duration::new(30, 0)); 1274 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1275 1276 // After the resize, check again that file1 exists and its 1277 // content is "foo". 1278 assert_eq!( 1279 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1280 "foo" 1281 ); 1282 } 1283 1284 if hotplug { 1285 // Remove from VM 1286 guest.ssh_command("sudo umount mount_dir").unwrap(); 1287 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1288 } 1289 }); 1290 1291 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1292 thread::sleep(std::time::Duration::new(10, 0)); 1293 let (daemon_child, virtiofsd_socket_path) = 1294 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1295 1296 let r = std::panic::catch_unwind(|| { 1297 thread::sleep(std::time::Duration::new(10, 0)); 1298 let fs_params = format!( 1299 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1300 virtiofsd_socket_path, 1301 if let Some(pci_segment) = pci_segment { 1302 format!(",pci_segment={}", pci_segment) 1303 } else { 1304 "".to_owned() 1305 } 1306 ); 1307 1308 // Add back and check it works 1309 let (cmd_success, cmd_output) = 1310 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1311 assert!(cmd_success); 1312 if let Some(pci_segment) = pci_segment { 1313 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1314 "{{\"id\":\"myfs0\",\"bdf\":\"{:04x}:00:01.0\"}}", 1315 pci_segment 1316 ))); 1317 } else { 1318 assert!(String::from_utf8_lossy(&cmd_output) 1319 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1320 } 1321 1322 thread::sleep(std::time::Duration::new(10, 0)); 1323 // Mount shared directory through virtio_fs filesystem 1324 guest 1325 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1326 .unwrap(); 1327 1328 // Check file1 exists and its content is "foo" 1329 assert_eq!( 1330 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1331 "foo" 1332 ); 1333 }); 1334 1335 (r, Some(daemon_child)) 1336 } else { 1337 (r, None) 1338 }; 1339 1340 let _ = child.kill(); 1341 let output = child.wait_with_output().unwrap(); 1342 1343 let _ = daemon_child.kill(); 1344 let _ = daemon_child.wait(); 1345 1346 if let Some(mut daemon_child) = hotplug_daemon_child { 1347 let _ = daemon_child.kill(); 1348 let _ = daemon_child.wait(); 1349 } 1350 1351 handle_child_output(r, &output); 1352 } 1353 1354 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1355 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1356 let guest = Guest::new(Box::new(focal)); 1357 1358 let kernel_path = direct_kernel_boot_path(); 1359 1360 let pmem_temp_file = TempFile::new().unwrap(); 1361 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1362 1363 std::process::Command::new("mkfs.ext4") 1364 .arg(pmem_temp_file.as_path()) 1365 .output() 1366 .expect("Expect creating disk image to succeed"); 1367 1368 let mut child = GuestCommand::new(&guest) 1369 .args(["--cpus", "boot=1"]) 1370 .args(["--memory", "size=512M"]) 1371 .args(["--kernel", kernel_path.to_str().unwrap()]) 1372 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1373 .default_disks() 1374 .default_net() 1375 .args([ 1376 "--pmem", 1377 format!( 1378 "file={}{}{}", 1379 pmem_temp_file.as_path().to_str().unwrap(), 1380 if specify_size { ",size=128M" } else { "" }, 1381 if discard_writes { 1382 ",discard_writes=on" 1383 } else { 1384 "" 1385 } 1386 ) 1387 .as_str(), 1388 ]) 1389 .capture_output() 1390 .spawn() 1391 .unwrap(); 1392 1393 let r = std::panic::catch_unwind(|| { 1394 guest.wait_vm_boot(None).unwrap(); 1395 1396 // Check for the presence of /dev/pmem0 1397 assert_eq!( 1398 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1399 "/dev/pmem0" 1400 ); 1401 1402 // Check changes persist after reboot 1403 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1404 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1405 guest 1406 .ssh_command("echo test123 | sudo tee /mnt/test") 1407 .unwrap(); 1408 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1409 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1410 1411 guest.reboot_linux(0, None); 1412 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1413 assert_eq!( 1414 guest 1415 .ssh_command("sudo cat /mnt/test || true") 1416 .unwrap() 1417 .trim(), 1418 if discard_writes { "" } else { "test123" } 1419 ); 1420 }); 1421 1422 let _ = child.kill(); 1423 let output = child.wait_with_output().unwrap(); 1424 1425 handle_child_output(r, &output); 1426 } 1427 1428 fn get_fd_count(pid: u32) -> usize { 1429 fs::read_dir(format!("/proc/{}/fd", pid)).unwrap().count() 1430 } 1431 1432 fn _test_virtio_vsock(hotplug: bool) { 1433 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1434 let guest = Guest::new(Box::new(focal)); 1435 1436 #[cfg(target_arch = "x86_64")] 1437 let kernel_path = direct_kernel_boot_path(); 1438 #[cfg(target_arch = "aarch64")] 1439 let kernel_path = if hotplug { 1440 edk2_path() 1441 } else { 1442 direct_kernel_boot_path() 1443 }; 1444 1445 let socket = temp_vsock_path(&guest.tmp_dir); 1446 let api_socket = temp_api_path(&guest.tmp_dir); 1447 1448 let mut cmd = GuestCommand::new(&guest); 1449 cmd.args(["--api-socket", &api_socket]); 1450 cmd.args(["--cpus", "boot=1"]); 1451 cmd.args(["--memory", "size=512M"]); 1452 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1453 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1454 cmd.default_disks(); 1455 cmd.default_net(); 1456 1457 if !hotplug { 1458 cmd.args(["--vsock", format!("cid=3,socket={}", socket).as_str()]); 1459 } 1460 1461 let mut child = cmd.capture_output().spawn().unwrap(); 1462 1463 let r = std::panic::catch_unwind(|| { 1464 guest.wait_vm_boot(None).unwrap(); 1465 1466 if hotplug { 1467 let (cmd_success, cmd_output) = remote_command_w_output( 1468 &api_socket, 1469 "add-vsock", 1470 Some(format!("cid=3,socket={},id=test0", socket).as_str()), 1471 ); 1472 assert!(cmd_success); 1473 assert!(String::from_utf8_lossy(&cmd_output) 1474 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1475 thread::sleep(std::time::Duration::new(10, 0)); 1476 // Check adding a second one fails 1477 assert!(!remote_command( 1478 &api_socket, 1479 "add-vsock", 1480 Some("cid=1234,socket=/tmp/fail") 1481 )); 1482 } 1483 1484 // Validate vsock works as expected. 1485 guest.check_vsock(socket.as_str()); 1486 guest.reboot_linux(0, None); 1487 // Validate vsock still works after a reboot. 1488 guest.check_vsock(socket.as_str()); 1489 1490 if hotplug { 1491 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1492 } 1493 }); 1494 1495 let _ = child.kill(); 1496 let output = child.wait_with_output().unwrap(); 1497 1498 handle_child_output(r, &output); 1499 } 1500 1501 fn get_ksm_pages_shared() -> u32 { 1502 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1503 .unwrap() 1504 .trim() 1505 .parse::<u32>() 1506 .unwrap() 1507 } 1508 1509 fn test_memory_mergeable(mergeable: bool) { 1510 let memory_param = if mergeable { 1511 "mergeable=on" 1512 } else { 1513 "mergeable=off" 1514 }; 1515 1516 // We are assuming the rest of the system in our CI is not using mergeable memeory 1517 let ksm_ps_init = get_ksm_pages_shared(); 1518 assert!(ksm_ps_init == 0); 1519 1520 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1521 let guest1 = Guest::new(Box::new(focal1)); 1522 let mut child1 = GuestCommand::new(&guest1) 1523 .args(["--cpus", "boot=1"]) 1524 .args(["--memory", format!("size=512M,{}", memory_param).as_str()]) 1525 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1526 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1527 .default_disks() 1528 .args(["--net", guest1.default_net_string().as_str()]) 1529 .args(["--serial", "tty", "--console", "off"]) 1530 .capture_output() 1531 .spawn() 1532 .unwrap(); 1533 1534 let r = std::panic::catch_unwind(|| { 1535 guest1.wait_vm_boot(None).unwrap(); 1536 }); 1537 if r.is_err() { 1538 let _ = child1.kill(); 1539 let output = child1.wait_with_output().unwrap(); 1540 handle_child_output(r, &output); 1541 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1542 } 1543 1544 let ksm_ps_guest1 = get_ksm_pages_shared(); 1545 1546 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1547 let guest2 = Guest::new(Box::new(focal2)); 1548 let mut child2 = GuestCommand::new(&guest2) 1549 .args(["--cpus", "boot=1"]) 1550 .args(["--memory", format!("size=512M,{}", memory_param).as_str()]) 1551 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1552 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1553 .default_disks() 1554 .args(["--net", guest2.default_net_string().as_str()]) 1555 .args(["--serial", "tty", "--console", "off"]) 1556 .capture_output() 1557 .spawn() 1558 .unwrap(); 1559 1560 let r = std::panic::catch_unwind(|| { 1561 guest2.wait_vm_boot(None).unwrap(); 1562 let ksm_ps_guest2 = get_ksm_pages_shared(); 1563 1564 if mergeable { 1565 println!( 1566 "ksm pages_shared after vm1 booted '{}', ksm pages_shared after vm2 booted '{}'", 1567 ksm_ps_guest1, ksm_ps_guest2 1568 ); 1569 // We are expecting the number of shared pages to increase as the number of VM increases 1570 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1571 } else { 1572 assert!(ksm_ps_guest1 == 0); 1573 assert!(ksm_ps_guest2 == 0); 1574 } 1575 }); 1576 1577 let _ = child1.kill(); 1578 let _ = child2.kill(); 1579 1580 let output = child1.wait_with_output().unwrap(); 1581 child2.wait().unwrap(); 1582 1583 handle_child_output(r, &output); 1584 } 1585 1586 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 1587 let smaps = fs::File::open(format!("/proc/{}/smaps", pid)).unwrap(); 1588 let reader = io::BufReader::new(smaps); 1589 1590 let mut skip_map: bool = false; 1591 let mut region_name: String = "".to_string(); 1592 let mut region_maps = HashMap::new(); 1593 for line in reader.lines() { 1594 let l = line.unwrap(); 1595 1596 if l.contains('-') { 1597 let values: Vec<&str> = l.split_whitespace().collect(); 1598 region_name = values.last().unwrap().trim().to_string(); 1599 if region_name == "0" { 1600 region_name = "anonymous".to_string() 1601 } 1602 } 1603 1604 // Each section begins with something that looks like: 1605 // Size: 2184 kB 1606 if l.starts_with("Size:") { 1607 let values: Vec<&str> = l.split_whitespace().collect(); 1608 let map_size = values[1].parse::<u32>().unwrap(); 1609 // We skip the assigned guest RAM map, its RSS is only 1610 // dependent on the guest actual memory usage. 1611 // Everything else can be added to the VMM overhead. 1612 skip_map = map_size >= guest_memory_size; 1613 continue; 1614 } 1615 1616 // If this is a map we're taking into account, then we only 1617 // count the RSS. The sum of all counted RSS is the VMM overhead. 1618 if !skip_map && l.starts_with("Rss:") { 1619 let values: Vec<&str> = l.split_whitespace().collect(); 1620 let value = values[1].trim().parse::<u32>().unwrap(); 1621 *region_maps.entry(region_name.clone()).or_insert(0) += value; 1622 } 1623 } 1624 1625 region_maps 1626 } 1627 1628 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 1629 let mut total = 0; 1630 1631 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 1632 eprintln!("{}: {}", region_name, value); 1633 total += value; 1634 } 1635 1636 total 1637 } 1638 1639 fn process_rss_kib(pid: u32) -> usize { 1640 let command = format!("ps -q {} -o rss=", pid); 1641 let rss = exec_host_command_output(&command); 1642 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 1643 } 1644 1645 // 10MB is our maximum accepted overhead. 1646 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 1647 1648 #[derive(PartialEq, Eq, PartialOrd)] 1649 struct Counters { 1650 rx_bytes: u64, 1651 rx_frames: u64, 1652 tx_bytes: u64, 1653 tx_frames: u64, 1654 read_bytes: u64, 1655 write_bytes: u64, 1656 read_ops: u64, 1657 write_ops: u64, 1658 } 1659 1660 fn get_counters(api_socket: &str) -> Counters { 1661 // Get counters 1662 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 1663 assert!(cmd_success); 1664 1665 let counters: HashMap<&str, HashMap<&str, u64>> = 1666 serde_json::from_slice(&cmd_output).unwrap_or_default(); 1667 1668 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 1669 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 1670 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 1671 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 1672 1673 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 1674 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 1675 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 1676 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 1677 1678 Counters { 1679 rx_bytes, 1680 rx_frames, 1681 tx_bytes, 1682 tx_frames, 1683 read_bytes, 1684 write_bytes, 1685 read_ops, 1686 write_ops, 1687 } 1688 } 1689 1690 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 1691 let (tx, rx) = mpsc::channel::<String>(); 1692 thread::spawn(move || loop { 1693 thread::sleep(std::time::Duration::new(1, 0)); 1694 let mut buf = [0; 512]; 1695 match pty.read(&mut buf) { 1696 Ok(_) => { 1697 let output = std::str::from_utf8(&buf).unwrap().to_string(); 1698 match tx.send(output) { 1699 Ok(_) => (), 1700 Err(_) => break, 1701 } 1702 } 1703 Err(_) => break, 1704 } 1705 }); 1706 rx 1707 } 1708 1709 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 1710 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1711 assert!(cmd_success); 1712 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1713 assert_eq!("Pty", info["config"][pty_type]["mode"]); 1714 PathBuf::from( 1715 info["config"][pty_type]["file"] 1716 .as_str() 1717 .expect("Missing pty path"), 1718 ) 1719 } 1720 1721 // VFIO test network setup. 1722 // We reserve a different IP class for it: 172.18.0.0/24. 1723 #[cfg(target_arch = "x86_64")] 1724 fn setup_vfio_network_interfaces() { 1725 // 'vfio-br0' 1726 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 1727 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 1728 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 1729 // 'vfio-tap0' 1730 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 1731 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 1732 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 1733 // 'vfio-tap1' 1734 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 1735 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 1736 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 1737 // 'vfio-tap2' 1738 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 1739 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 1740 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 1741 // 'vfio-tap3' 1742 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 1743 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 1744 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 1745 } 1746 1747 // Tear VFIO test network down 1748 #[cfg(target_arch = "x86_64")] 1749 fn cleanup_vfio_network_interfaces() { 1750 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 1751 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 1752 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 1753 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 1754 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 1755 } 1756 1757 fn balloon_size(api_socket: &str) -> u64 { 1758 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1759 assert!(cmd_success); 1760 1761 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1762 let total_mem = &info["config"]["memory"]["size"] 1763 .to_string() 1764 .parse::<u64>() 1765 .unwrap(); 1766 let actual_mem = &info["memory_actual_size"] 1767 .to_string() 1768 .parse::<u64>() 1769 .unwrap(); 1770 total_mem - actual_mem 1771 } 1772 1773 // This test validates that it can find the virtio-iommu device at first. 1774 // It also verifies that both disks and the network card are attached to 1775 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 1776 // The last interesting part of this test is that it exercises the network 1777 // interface attached to the virtual IOMMU since this is the one used to 1778 // send all commands through SSH. 1779 fn _test_virtio_iommu(acpi: bool) { 1780 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 1781 // Focal image is still old. 1782 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 1783 // the kernel binary has been updated. 1784 #[cfg(target_arch = "aarch64")] 1785 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 1786 #[cfg(target_arch = "x86_64")] 1787 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1788 let focal = UbuntuDiskConfig::new(focal_image); 1789 let guest = Guest::new(Box::new(focal)); 1790 1791 #[cfg(target_arch = "x86_64")] 1792 let kernel_path = direct_kernel_boot_path(); 1793 #[cfg(target_arch = "aarch64")] 1794 let kernel_path = if acpi { 1795 edk2_path() 1796 } else { 1797 direct_kernel_boot_path() 1798 }; 1799 1800 let mut child = GuestCommand::new(&guest) 1801 .args(["--cpus", "boot=1"]) 1802 .args(["--memory", "size=512M"]) 1803 .args(["--kernel", kernel_path.to_str().unwrap()]) 1804 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1805 .args([ 1806 "--disk", 1807 format!( 1808 "path={},iommu=on", 1809 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1810 ) 1811 .as_str(), 1812 format!( 1813 "path={},iommu=on", 1814 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1815 ) 1816 .as_str(), 1817 ]) 1818 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 1819 .capture_output() 1820 .spawn() 1821 .unwrap(); 1822 1823 let r = std::panic::catch_unwind(|| { 1824 guest.wait_vm_boot(None).unwrap(); 1825 1826 // Verify the virtio-iommu device is present. 1827 assert!(guest 1828 .does_device_vendor_pair_match("0x1057", "0x1af4") 1829 .unwrap_or_default()); 1830 1831 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 1832 // different with ACPI. 1833 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 1834 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 1835 // and appear under folder '/sys/kernel/iommu_groups/'. 1836 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 1837 // which is the console. The first disk "0000:00:02.0" is in group '1'. 1838 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 1839 // contains "0000:00:02.0" which is the first disk. 1840 // 1841 // Verify the iommu group of the first disk. 1842 let iommu_group = !acpi as i32; 1843 assert_eq!( 1844 guest 1845 .ssh_command( 1846 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1847 ) 1848 .unwrap() 1849 .trim(), 1850 "0000:00:02.0" 1851 ); 1852 1853 // Verify the iommu group of the second disk. 1854 let iommu_group = if acpi { 1 } else { 2 }; 1855 assert_eq!( 1856 guest 1857 .ssh_command( 1858 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1859 ) 1860 .unwrap() 1861 .trim(), 1862 "0000:00:03.0" 1863 ); 1864 1865 // Verify the iommu group of the network card. 1866 let iommu_group = if acpi { 2 } else { 3 }; 1867 assert_eq!( 1868 guest 1869 .ssh_command( 1870 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1871 ) 1872 .unwrap() 1873 .trim(), 1874 "0000:00:04.0" 1875 ); 1876 }); 1877 1878 let _ = child.kill(); 1879 let output = child.wait_with_output().unwrap(); 1880 1881 handle_child_output(r, &output); 1882 } 1883 1884 fn get_reboot_count(guest: &Guest) -> u32 { 1885 guest 1886 .ssh_command("sudo last | grep -c reboot") 1887 .unwrap() 1888 .trim() 1889 .parse::<u32>() 1890 .unwrap_or_default() 1891 } 1892 1893 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 1894 // Check for PCI device 1895 assert!(guest 1896 .does_device_vendor_pair_match("0x1063", "0x1af4") 1897 .unwrap_or_default()); 1898 1899 // Enable systemd watchdog 1900 guest 1901 .ssh_command(&format!( 1902 "echo RuntimeWatchdogSec={}s | sudo tee -a /etc/systemd/system.conf", 1903 watchdog_sec 1904 )) 1905 .unwrap(); 1906 } 1907 1908 mod common_parallel { 1909 use std::{fs::OpenOptions, io::SeekFrom}; 1910 1911 use crate::*; 1912 1913 #[test] 1914 #[cfg(target_arch = "x86_64")] 1915 fn test_bionic_hypervisor_fw() { 1916 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), BIONIC_IMAGE_NAME) 1917 } 1918 1919 #[test] 1920 #[cfg(target_arch = "x86_64")] 1921 fn test_focal_hypervisor_fw() { 1922 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 1923 } 1924 1925 #[test] 1926 #[cfg(target_arch = "x86_64")] 1927 fn test_bionic_ovmf() { 1928 test_simple_launch(fw_path(FwType::Ovmf), BIONIC_IMAGE_NAME) 1929 } 1930 1931 #[test] 1932 #[cfg(target_arch = "x86_64")] 1933 fn test_focal_ovmf() { 1934 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 1935 } 1936 1937 #[cfg(target_arch = "x86_64")] 1938 fn test_simple_launch(fw_path: String, disk_path: &str) { 1939 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 1940 let guest = Guest::new(disk_config); 1941 let event_path = temp_event_monitor_path(&guest.tmp_dir); 1942 1943 let mut child = GuestCommand::new(&guest) 1944 .args(["--cpus", "boot=1"]) 1945 .args(["--memory", "size=512M"]) 1946 .args(["--kernel", fw_path.as_str()]) 1947 .default_disks() 1948 .default_net() 1949 .args(["--serial", "tty", "--console", "off"]) 1950 .args(["--event-monitor", format!("path={}", event_path).as_str()]) 1951 .capture_output() 1952 .spawn() 1953 .unwrap(); 1954 1955 let r = std::panic::catch_unwind(|| { 1956 guest.wait_vm_boot(Some(120)).unwrap(); 1957 1958 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 1959 assert_eq!(guest.get_initial_apicid().unwrap_or(1), 0); 1960 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1961 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 1962 1963 let expected_sequential_events = [ 1964 &MetaEvent { 1965 event: "starting".to_string(), 1966 device_id: None, 1967 }, 1968 &MetaEvent { 1969 event: "booting".to_string(), 1970 device_id: None, 1971 }, 1972 &MetaEvent { 1973 event: "booted".to_string(), 1974 device_id: None, 1975 }, 1976 &MetaEvent { 1977 event: "activated".to_string(), 1978 device_id: Some("_disk0".to_string()), 1979 }, 1980 &MetaEvent { 1981 event: "reset".to_string(), 1982 device_id: Some("_disk0".to_string()), 1983 }, 1984 ]; 1985 assert!(check_sequential_events( 1986 &expected_sequential_events, 1987 &event_path 1988 )); 1989 1990 // It's been observed on the Bionic image that udev and snapd 1991 // services can cause some delay in the VM's shutdown. Disabling 1992 // them improves the reliability of this test. 1993 let _ = guest.ssh_command("sudo systemctl disable udev"); 1994 let _ = guest.ssh_command("sudo systemctl stop udev"); 1995 let _ = guest.ssh_command("sudo systemctl disable snapd"); 1996 let _ = guest.ssh_command("sudo systemctl stop snapd"); 1997 1998 guest.ssh_command("sudo poweroff").unwrap(); 1999 thread::sleep(std::time::Duration::new(20, 0)); 2000 let latest_events = [ 2001 &MetaEvent { 2002 event: "shutdown".to_string(), 2003 device_id: None, 2004 }, 2005 &MetaEvent { 2006 event: "deleted".to_string(), 2007 device_id: None, 2008 }, 2009 &MetaEvent { 2010 event: "shutdown".to_string(), 2011 device_id: None, 2012 }, 2013 ]; 2014 assert!(check_latest_events_exact(&latest_events, &event_path)); 2015 }); 2016 2017 let _ = child.kill(); 2018 let output = child.wait_with_output().unwrap(); 2019 2020 handle_child_output(r, &output); 2021 } 2022 2023 #[test] 2024 fn test_multi_cpu() { 2025 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2026 let jammy = UbuntuDiskConfig::new(jammy_image); 2027 let guest = Guest::new(Box::new(jammy)); 2028 2029 let mut cmd = GuestCommand::new(&guest); 2030 cmd.args(["--cpus", "boot=2,max=4"]) 2031 .args(["--memory", "size=512M"]) 2032 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2033 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2034 .capture_output() 2035 .default_disks() 2036 .default_net(); 2037 2038 let mut child = cmd.spawn().unwrap(); 2039 2040 let r = std::panic::catch_unwind(|| { 2041 guest.wait_vm_boot(Some(120)).unwrap(); 2042 2043 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2044 2045 assert_eq!( 2046 guest 2047 .ssh_command( 2048 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2049 ) 2050 .unwrap() 2051 .trim(), 2052 "smp: Brought up 1 node, 2 CPUs" 2053 ); 2054 }); 2055 2056 let _ = child.kill(); 2057 let output = child.wait_with_output().unwrap(); 2058 2059 handle_child_output(r, &output); 2060 } 2061 2062 #[test] 2063 #[cfg(not(feature = "mshv"))] 2064 fn test_cpu_topology_421() { 2065 test_cpu_topology(4, 2, 1, false); 2066 } 2067 2068 #[test] 2069 #[cfg(not(feature = "mshv"))] 2070 fn test_cpu_topology_142() { 2071 test_cpu_topology(1, 4, 2, false); 2072 } 2073 2074 #[test] 2075 #[cfg(not(feature = "mshv"))] 2076 fn test_cpu_topology_262() { 2077 test_cpu_topology(2, 6, 2, false); 2078 } 2079 2080 #[test] 2081 #[cfg(target_arch = "x86_64")] 2082 #[cfg(not(feature = "mshv"))] 2083 fn test_cpu_physical_bits() { 2084 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2085 let guest = Guest::new(Box::new(focal)); 2086 let max_phys_bits: u8 = 36; 2087 let mut child = GuestCommand::new(&guest) 2088 .args(["--cpus", &format!("max_phys_bits={}", max_phys_bits)]) 2089 .args(["--memory", "size=512M"]) 2090 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2091 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2092 .default_disks() 2093 .default_net() 2094 .capture_output() 2095 .spawn() 2096 .unwrap(); 2097 2098 let r = std::panic::catch_unwind(|| { 2099 guest.wait_vm_boot(None).unwrap(); 2100 2101 assert!( 2102 guest 2103 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2104 .unwrap() 2105 .trim() 2106 .parse::<u8>() 2107 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2108 ); 2109 }); 2110 2111 let _ = child.kill(); 2112 let output = child.wait_with_output().unwrap(); 2113 2114 handle_child_output(r, &output); 2115 } 2116 2117 #[test] 2118 fn test_cpu_affinity() { 2119 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2120 let guest = Guest::new(Box::new(focal)); 2121 2122 // We need the host to have at least 4 CPUs if we want to be able 2123 // to run this test. 2124 let host_cpus_count = exec_host_command_output("nproc"); 2125 assert!( 2126 String::from_utf8_lossy(&host_cpus_count.stdout) 2127 .trim() 2128 .parse::<u8>() 2129 .unwrap_or(0) 2130 >= 4 2131 ); 2132 2133 let mut child = GuestCommand::new(&guest) 2134 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2135 .args(["--memory", "size=512M"]) 2136 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2137 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2138 .default_disks() 2139 .default_net() 2140 .capture_output() 2141 .spawn() 2142 .unwrap(); 2143 2144 let r = std::panic::catch_unwind(|| { 2145 guest.wait_vm_boot(None).unwrap(); 2146 let pid = child.id(); 2147 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"", pid).as_str()); 2148 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2149 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"", pid).as_str()); 2150 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2151 }); 2152 2153 let _ = child.kill(); 2154 let output = child.wait_with_output().unwrap(); 2155 2156 handle_child_output(r, &output); 2157 } 2158 2159 #[test] 2160 #[cfg(not(feature = "mshv"))] 2161 fn test_large_vm() { 2162 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2163 let guest = Guest::new(Box::new(focal)); 2164 let mut cmd = GuestCommand::new(&guest); 2165 cmd.args(["--cpus", "boot=48"]) 2166 .args(["--memory", "size=5120M"]) 2167 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2168 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2169 .args(["--serial", "tty"]) 2170 .args(["--console", "off"]) 2171 .capture_output() 2172 .default_disks() 2173 .default_net(); 2174 2175 let mut child = cmd.spawn().unwrap(); 2176 2177 guest.wait_vm_boot(None).unwrap(); 2178 2179 let r = std::panic::catch_unwind(|| { 2180 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2181 assert_eq!( 2182 guest 2183 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2184 .unwrap() 2185 .trim(), 2186 "0-47" 2187 ); 2188 2189 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2190 }); 2191 2192 let _ = child.kill(); 2193 let output = child.wait_with_output().unwrap(); 2194 2195 handle_child_output(r, &output); 2196 } 2197 2198 #[test] 2199 #[cfg(not(feature = "mshv"))] 2200 fn test_huge_memory() { 2201 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2202 let guest = Guest::new(Box::new(focal)); 2203 let mut cmd = GuestCommand::new(&guest); 2204 cmd.args(["--cpus", "boot=1"]) 2205 .args(["--memory", "size=128G"]) 2206 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2207 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2208 .capture_output() 2209 .default_disks() 2210 .default_net(); 2211 2212 let mut child = cmd.spawn().unwrap(); 2213 2214 guest.wait_vm_boot(Some(120)).unwrap(); 2215 2216 let r = std::panic::catch_unwind(|| { 2217 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2218 }); 2219 2220 let _ = child.kill(); 2221 let output = child.wait_with_output().unwrap(); 2222 2223 handle_child_output(r, &output); 2224 } 2225 2226 #[test] 2227 fn test_power_button() { 2228 _test_power_button(false); 2229 } 2230 2231 #[test] 2232 #[cfg(not(feature = "mshv"))] 2233 fn test_user_defined_memory_regions() { 2234 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2235 let guest = Guest::new(Box::new(focal)); 2236 let api_socket = temp_api_path(&guest.tmp_dir); 2237 2238 let kernel_path = direct_kernel_boot_path(); 2239 2240 let mut child = GuestCommand::new(&guest) 2241 .args(["--cpus", "boot=1"]) 2242 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2243 .args([ 2244 "--memory-zone", 2245 "id=mem0,size=1G,hotplug_size=2G", 2246 "id=mem1,size=1G,file=/dev/shm", 2247 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2248 ]) 2249 .args(["--kernel", kernel_path.to_str().unwrap()]) 2250 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2251 .args(["--api-socket", &api_socket]) 2252 .capture_output() 2253 .default_disks() 2254 .default_net() 2255 .spawn() 2256 .unwrap(); 2257 2258 let r = std::panic::catch_unwind(|| { 2259 guest.wait_vm_boot(None).unwrap(); 2260 2261 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2262 2263 guest.enable_memory_hotplug(); 2264 2265 resize_zone_command(&api_socket, "mem0", "3G"); 2266 thread::sleep(std::time::Duration::new(5, 0)); 2267 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2268 resize_zone_command(&api_socket, "mem2", "3G"); 2269 thread::sleep(std::time::Duration::new(5, 0)); 2270 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2271 resize_zone_command(&api_socket, "mem0", "2G"); 2272 thread::sleep(std::time::Duration::new(5, 0)); 2273 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2274 resize_zone_command(&api_socket, "mem2", "2G"); 2275 thread::sleep(std::time::Duration::new(5, 0)); 2276 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2277 2278 guest.reboot_linux(0, None); 2279 2280 // Check the amount of RAM after reboot 2281 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2282 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2283 2284 // Check if we can still resize down to the initial 'boot'size 2285 resize_zone_command(&api_socket, "mem0", "1G"); 2286 thread::sleep(std::time::Duration::new(5, 0)); 2287 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2288 resize_zone_command(&api_socket, "mem2", "1G"); 2289 thread::sleep(std::time::Duration::new(5, 0)); 2290 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2291 }); 2292 2293 let _ = child.kill(); 2294 let output = child.wait_with_output().unwrap(); 2295 2296 handle_child_output(r, &output); 2297 } 2298 2299 #[test] 2300 #[cfg(not(feature = "mshv"))] 2301 fn test_guest_numa_nodes() { 2302 _test_guest_numa_nodes(false); 2303 } 2304 2305 #[test] 2306 #[cfg(target_arch = "x86_64")] 2307 fn test_iommu_segments() { 2308 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2309 let focal = UbuntuDiskConfig::new(focal_image); 2310 let guest = Guest::new(Box::new(focal)); 2311 2312 // Prepare another disk file for the virtio-disk device 2313 let test_disk_path = String::from( 2314 guest 2315 .tmp_dir 2316 .as_path() 2317 .join("test-disk.raw") 2318 .to_str() 2319 .unwrap(), 2320 ); 2321 assert!( 2322 exec_host_command_status(format!("truncate {} -s 4M", test_disk_path).as_str()) 2323 .success() 2324 ); 2325 assert!( 2326 exec_host_command_status(format!("mkfs.ext4 {}", test_disk_path).as_str()).success() 2327 ); 2328 2329 let api_socket = temp_api_path(&guest.tmp_dir); 2330 let mut cmd = GuestCommand::new(&guest); 2331 2332 cmd.args(["--cpus", "boot=1"]) 2333 .args(["--api-socket", &api_socket]) 2334 .args(["--memory", "size=512M"]) 2335 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2336 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2337 .args(["--platform", "num_pci_segments=16,iommu_segments=[1]"]) 2338 .default_disks() 2339 .capture_output() 2340 .default_net(); 2341 2342 let mut child = cmd.spawn().unwrap(); 2343 2344 guest.wait_vm_boot(None).unwrap(); 2345 2346 let r = std::panic::catch_unwind(|| { 2347 let (cmd_success, cmd_output) = remote_command_w_output( 2348 &api_socket, 2349 "add-disk", 2350 Some( 2351 format!( 2352 "path={},id=test0,pci_segment=1,iommu=on", 2353 test_disk_path.as_str() 2354 ) 2355 .as_str(), 2356 ), 2357 ); 2358 assert!(cmd_success); 2359 assert!(String::from_utf8_lossy(&cmd_output) 2360 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2361 2362 // Check IOMMU setup 2363 assert!(guest 2364 .does_device_vendor_pair_match("0x1057", "0x1af4") 2365 .unwrap_or_default()); 2366 assert_eq!( 2367 guest 2368 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2369 .unwrap() 2370 .trim(), 2371 "0001:00:01.0" 2372 ); 2373 }); 2374 2375 let _ = child.kill(); 2376 let output = child.wait_with_output().unwrap(); 2377 2378 handle_child_output(r, &output); 2379 } 2380 2381 #[test] 2382 fn test_pci_msi() { 2383 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2384 let guest = Guest::new(Box::new(focal)); 2385 let mut cmd = GuestCommand::new(&guest); 2386 cmd.args(["--cpus", "boot=1"]) 2387 .args(["--memory", "size=512M"]) 2388 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2389 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2390 .capture_output() 2391 .default_disks() 2392 .default_net(); 2393 2394 let mut child = cmd.spawn().unwrap(); 2395 2396 guest.wait_vm_boot(None).unwrap(); 2397 2398 #[cfg(target_arch = "x86_64")] 2399 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2400 #[cfg(target_arch = "aarch64")] 2401 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2402 2403 let r = std::panic::catch_unwind(|| { 2404 assert_eq!( 2405 guest 2406 .ssh_command(grep_cmd) 2407 .unwrap() 2408 .trim() 2409 .parse::<u32>() 2410 .unwrap_or_default(), 2411 12 2412 ); 2413 }); 2414 2415 let _ = child.kill(); 2416 let output = child.wait_with_output().unwrap(); 2417 2418 handle_child_output(r, &output); 2419 } 2420 2421 #[test] 2422 fn test_virtio_net_ctrl_queue() { 2423 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2424 let guest = Guest::new(Box::new(focal)); 2425 let mut cmd = GuestCommand::new(&guest); 2426 cmd.args(["--cpus", "boot=1"]) 2427 .args(["--memory", "size=512M"]) 2428 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2429 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2430 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2431 .capture_output() 2432 .default_disks(); 2433 2434 let mut child = cmd.spawn().unwrap(); 2435 2436 guest.wait_vm_boot(None).unwrap(); 2437 2438 #[cfg(target_arch = "aarch64")] 2439 let iface = "enp0s4"; 2440 #[cfg(target_arch = "x86_64")] 2441 let iface = "ens4"; 2442 2443 let r = std::panic::catch_unwind(|| { 2444 assert_eq!( 2445 guest 2446 .ssh_command( 2447 format!("sudo ethtool -K {} rx-gro-hw off && echo success", iface).as_str() 2448 ) 2449 .unwrap() 2450 .trim(), 2451 "success" 2452 ); 2453 assert_eq!( 2454 guest 2455 .ssh_command(format!("cat /sys/class/net/{}/mtu", iface).as_str()) 2456 .unwrap() 2457 .trim(), 2458 "3000" 2459 ); 2460 }); 2461 2462 let _ = child.kill(); 2463 let output = child.wait_with_output().unwrap(); 2464 2465 handle_child_output(r, &output); 2466 } 2467 2468 #[test] 2469 #[cfg(not(feature = "mshv"))] 2470 fn test_pci_multiple_segments() { 2471 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2472 let guest = Guest::new(Box::new(focal)); 2473 2474 // Prepare another disk file for the virtio-disk device 2475 let test_disk_path = String::from( 2476 guest 2477 .tmp_dir 2478 .as_path() 2479 .join("test-disk.raw") 2480 .to_str() 2481 .unwrap(), 2482 ); 2483 assert!( 2484 exec_host_command_status(format!("truncate {} -s 4M", test_disk_path).as_str()) 2485 .success() 2486 ); 2487 assert!( 2488 exec_host_command_status(format!("mkfs.ext4 {}", test_disk_path).as_str()).success() 2489 ); 2490 2491 let mut cmd = GuestCommand::new(&guest); 2492 cmd.args(["--cpus", "boot=1"]) 2493 .args(["--memory", "size=512M"]) 2494 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2495 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2496 .args(["--platform", "num_pci_segments=16"]) 2497 .args([ 2498 "--disk", 2499 format!( 2500 "path={}", 2501 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2502 ) 2503 .as_str(), 2504 format!( 2505 "path={}", 2506 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2507 ) 2508 .as_str(), 2509 format!("path={},pci_segment=15", test_disk_path).as_str(), 2510 ]) 2511 .capture_output() 2512 .default_net(); 2513 2514 let mut child = cmd.spawn().unwrap(); 2515 2516 guest.wait_vm_boot(None).unwrap(); 2517 2518 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2519 2520 let r = std::panic::catch_unwind(|| { 2521 // There should be 16 PCI host bridges in the guest. 2522 assert_eq!( 2523 guest 2524 .ssh_command(grep_cmd) 2525 .unwrap() 2526 .trim() 2527 .parse::<u32>() 2528 .unwrap_or_default(), 2529 16 2530 ); 2531 2532 // Check both if /dev/vdc exists and if the block size is 4M. 2533 assert_eq!( 2534 guest 2535 .ssh_command("lsblk | grep vdc | grep -c 4M") 2536 .unwrap() 2537 .trim() 2538 .parse::<u32>() 2539 .unwrap_or_default(), 2540 1 2541 ); 2542 2543 // Mount the device. 2544 guest.ssh_command("mkdir mount_image").unwrap(); 2545 guest 2546 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 2547 .unwrap(); 2548 // Grant all users with write permission. 2549 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 2550 2551 // Write something to the device. 2552 guest 2553 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 2554 .unwrap(); 2555 2556 // Check the content of the block device. The file "foo" should 2557 // contain "bar". 2558 assert_eq!( 2559 guest 2560 .ssh_command("sudo cat mount_image/foo") 2561 .unwrap() 2562 .trim(), 2563 "bar" 2564 ); 2565 }); 2566 2567 let _ = child.kill(); 2568 let output = child.wait_with_output().unwrap(); 2569 2570 handle_child_output(r, &output); 2571 } 2572 2573 #[test] 2574 fn test_direct_kernel_boot() { 2575 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2576 let guest = Guest::new(Box::new(focal)); 2577 2578 let kernel_path = direct_kernel_boot_path(); 2579 2580 let mut child = GuestCommand::new(&guest) 2581 .args(["--cpus", "boot=1"]) 2582 .args(["--memory", "size=512M"]) 2583 .args(["--kernel", kernel_path.to_str().unwrap()]) 2584 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2585 .default_disks() 2586 .default_net() 2587 .capture_output() 2588 .spawn() 2589 .unwrap(); 2590 2591 let r = std::panic::catch_unwind(|| { 2592 guest.wait_vm_boot(None).unwrap(); 2593 2594 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2595 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2596 2597 let grep_cmd = if cfg!(target_arch = "x86_64") { 2598 "grep -c PCI-MSI /proc/interrupts" 2599 } else { 2600 "grep -c ITS-MSI /proc/interrupts" 2601 }; 2602 assert_eq!( 2603 guest 2604 .ssh_command(grep_cmd) 2605 .unwrap() 2606 .trim() 2607 .parse::<u32>() 2608 .unwrap_or_default(), 2609 12 2610 ); 2611 }); 2612 2613 let _ = child.kill(); 2614 let output = child.wait_with_output().unwrap(); 2615 2616 handle_child_output(r, &output); 2617 } 2618 2619 fn _test_virtio_block(image_name: &str, disable_io_uring: bool) { 2620 let focal = UbuntuDiskConfig::new(image_name.to_string()); 2621 let guest = Guest::new(Box::new(focal)); 2622 2623 let mut workload_path = dirs::home_dir().unwrap(); 2624 workload_path.push("workloads"); 2625 2626 let mut blk_file_path = workload_path; 2627 blk_file_path.push("blk.img"); 2628 2629 let kernel_path = direct_kernel_boot_path(); 2630 2631 let mut cloud_child = GuestCommand::new(&guest) 2632 .args(["--cpus", "boot=4"]) 2633 .args(["--memory", "size=512M,shared=on"]) 2634 .args(["--kernel", kernel_path.to_str().unwrap()]) 2635 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2636 .args([ 2637 "--disk", 2638 format!( 2639 "path={}", 2640 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2641 ) 2642 .as_str(), 2643 format!( 2644 "path={}", 2645 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2646 ) 2647 .as_str(), 2648 format!( 2649 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={}", 2650 blk_file_path.to_str().unwrap(), 2651 disable_io_uring 2652 ) 2653 .as_str(), 2654 ]) 2655 .default_net() 2656 .capture_output() 2657 .spawn() 2658 .unwrap(); 2659 2660 let r = std::panic::catch_unwind(|| { 2661 guest.wait_vm_boot(None).unwrap(); 2662 2663 // Check both if /dev/vdc exists and if the block size is 16M. 2664 assert_eq!( 2665 guest 2666 .ssh_command("lsblk | grep vdc | grep -c 16M") 2667 .unwrap() 2668 .trim() 2669 .parse::<u32>() 2670 .unwrap_or_default(), 2671 1 2672 ); 2673 2674 // Check both if /dev/vdc exists and if this block is RO. 2675 assert_eq!( 2676 guest 2677 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 2678 .unwrap() 2679 .trim() 2680 .parse::<u32>() 2681 .unwrap_or_default(), 2682 1 2683 ); 2684 2685 // Check if the number of queues is 4. 2686 assert_eq!( 2687 guest 2688 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 2689 .unwrap() 2690 .trim() 2691 .parse::<u32>() 2692 .unwrap_or_default(), 2693 4 2694 ); 2695 }); 2696 2697 let _ = cloud_child.kill(); 2698 let output = cloud_child.wait_with_output().unwrap(); 2699 2700 handle_child_output(r, &output); 2701 } 2702 2703 #[test] 2704 fn test_virtio_block() { 2705 _test_virtio_block(FOCAL_IMAGE_NAME, false) 2706 } 2707 2708 #[test] 2709 fn test_virtio_block_disable_io_uring() { 2710 _test_virtio_block(FOCAL_IMAGE_NAME, true) 2711 } 2712 2713 #[test] 2714 fn test_virtio_block_qcow2() { 2715 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false) 2716 } 2717 2718 #[test] 2719 fn test_virtio_block_vhd() { 2720 let mut workload_path = dirs::home_dir().unwrap(); 2721 workload_path.push("workloads"); 2722 2723 let mut raw_file_path = workload_path.clone(); 2724 let mut vhd_file_path = workload_path; 2725 raw_file_path.push(FOCAL_IMAGE_NAME); 2726 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 2727 2728 // Generate VHD file from RAW file 2729 std::process::Command::new("qemu-img") 2730 .arg("convert") 2731 .arg("-p") 2732 .args(["-f", "raw"]) 2733 .args(["-O", "vpc"]) 2734 .args(["-o", "subformat=fixed"]) 2735 .arg(raw_file_path.to_str().unwrap()) 2736 .arg(vhd_file_path.to_str().unwrap()) 2737 .output() 2738 .expect("Expect generating VHD image from RAW image"); 2739 2740 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false) 2741 } 2742 2743 #[test] 2744 fn test_virtio_block_vhdx() { 2745 let mut workload_path = dirs::home_dir().unwrap(); 2746 workload_path.push("workloads"); 2747 2748 let mut raw_file_path = workload_path.clone(); 2749 let mut vhdx_file_path = workload_path; 2750 raw_file_path.push(FOCAL_IMAGE_NAME); 2751 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 2752 2753 // Generate dynamic VHDX file from RAW file 2754 std::process::Command::new("qemu-img") 2755 .arg("convert") 2756 .arg("-p") 2757 .args(["-f", "raw"]) 2758 .args(["-O", "vhdx"]) 2759 .arg(raw_file_path.to_str().unwrap()) 2760 .arg(vhdx_file_path.to_str().unwrap()) 2761 .output() 2762 .expect("Expect generating dynamic VHDx image from RAW image"); 2763 2764 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false) 2765 } 2766 2767 #[test] 2768 fn test_virtio_block_dynamic_vhdx_expand() { 2769 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 2770 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 2771 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 2772 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 2773 2774 let mut workload_path = dirs::home_dir().unwrap(); 2775 workload_path.push("workloads"); 2776 2777 let mut vhdx_file_path = workload_path; 2778 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 2779 let vhdx_path = vhdx_file_path.to_str().unwrap(); 2780 2781 // Generate a 100 MiB dynamic VHDX file 2782 std::process::Command::new("qemu-img") 2783 .arg("create") 2784 .args(["-f", "vhdx"]) 2785 .arg(vhdx_path) 2786 .arg(VIRTUAL_DISK_SIZE.to_string()) 2787 .output() 2788 .expect("Expect generating dynamic VHDx image from RAW image"); 2789 2790 // Check if the size matches with empty VHDx file size 2791 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 2792 2793 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2794 let guest = Guest::new(Box::new(focal)); 2795 let kernel_path = direct_kernel_boot_path(); 2796 2797 let mut cloud_child = GuestCommand::new(&guest) 2798 .args(["--cpus", "boot=1"]) 2799 .args(["--memory", "size=512M"]) 2800 .args(["--kernel", kernel_path.to_str().unwrap()]) 2801 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2802 .args([ 2803 "--disk", 2804 format!( 2805 "path={}", 2806 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2807 ) 2808 .as_str(), 2809 format!( 2810 "path={}", 2811 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2812 ) 2813 .as_str(), 2814 format!("path={}", vhdx_path).as_str(), 2815 ]) 2816 .default_net() 2817 .capture_output() 2818 .spawn() 2819 .unwrap(); 2820 2821 let r = std::panic::catch_unwind(|| { 2822 guest.wait_vm_boot(None).unwrap(); 2823 2824 // Check both if /dev/vdc exists and if the block size is 100 MiB. 2825 assert_eq!( 2826 guest 2827 .ssh_command("lsblk | grep vdc | grep -c 100M") 2828 .unwrap() 2829 .trim() 2830 .parse::<u32>() 2831 .unwrap_or_default(), 2832 1 2833 ); 2834 2835 // Write 100 MB of data to the VHDx disk 2836 guest 2837 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 2838 .unwrap(); 2839 }); 2840 2841 // Check if the size matches with expected expanded VHDx file size 2842 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 2843 2844 let _ = cloud_child.kill(); 2845 let output = cloud_child.wait_with_output().unwrap(); 2846 2847 handle_child_output(r, &output); 2848 } 2849 2850 fn vhdx_image_size(disk_name: &str) -> u64 { 2851 std::fs::File::open(disk_name) 2852 .unwrap() 2853 .seek(SeekFrom::End(0)) 2854 .unwrap() 2855 } 2856 2857 #[test] 2858 fn test_virtio_block_direct_and_firmware() { 2859 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2860 let guest = Guest::new(Box::new(focal)); 2861 2862 // The OS disk must be copied to a location that is not backed by 2863 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 2864 // with EINVAL because tmpfs doesn't support this flag. 2865 let mut workloads_path = dirs::home_dir().unwrap(); 2866 workloads_path.push("workloads"); 2867 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 2868 let mut os_path = os_dir.as_path().to_path_buf(); 2869 os_path.push("osdisk.img"); 2870 rate_limited_copy( 2871 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 2872 os_path.as_path(), 2873 ) 2874 .expect("copying of OS disk failed"); 2875 2876 let mut child = GuestCommand::new(&guest) 2877 .args(["--cpus", "boot=1"]) 2878 .args(["--memory", "size=512M"]) 2879 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 2880 .args([ 2881 "--disk", 2882 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 2883 format!( 2884 "path={}", 2885 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2886 ) 2887 .as_str(), 2888 ]) 2889 .default_net() 2890 .capture_output() 2891 .spawn() 2892 .unwrap(); 2893 2894 let r = std::panic::catch_unwind(|| { 2895 guest.wait_vm_boot(Some(120)).unwrap(); 2896 }); 2897 2898 let _ = child.kill(); 2899 let output = child.wait_with_output().unwrap(); 2900 2901 handle_child_output(r, &output); 2902 } 2903 2904 #[test] 2905 fn test_vhost_user_net_default() { 2906 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 2907 } 2908 2909 #[test] 2910 fn test_vhost_user_net_named_tap() { 2911 test_vhost_user_net( 2912 Some("mytap0"), 2913 2, 2914 &prepare_vhost_user_net_daemon, 2915 false, 2916 false, 2917 ) 2918 } 2919 2920 #[test] 2921 fn test_vhost_user_net_existing_tap() { 2922 test_vhost_user_net( 2923 Some("vunet-tap0"), 2924 2, 2925 &prepare_vhost_user_net_daemon, 2926 false, 2927 false, 2928 ) 2929 } 2930 2931 #[test] 2932 fn test_vhost_user_net_multiple_queues() { 2933 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 2934 } 2935 2936 #[test] 2937 fn test_vhost_user_net_tap_multiple_queues() { 2938 test_vhost_user_net( 2939 Some("vunet-tap1"), 2940 4, 2941 &prepare_vhost_user_net_daemon, 2942 false, 2943 false, 2944 ) 2945 } 2946 2947 #[test] 2948 fn test_vhost_user_net_host_mac() { 2949 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 2950 } 2951 2952 #[test] 2953 fn test_vhost_user_net_client_mode() { 2954 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 2955 } 2956 2957 #[test] 2958 fn test_vhost_user_blk_default() { 2959 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 2960 } 2961 2962 #[test] 2963 fn test_vhost_user_blk_readonly() { 2964 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 2965 } 2966 2967 #[test] 2968 fn test_vhost_user_blk_direct() { 2969 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 2970 } 2971 2972 #[test] 2973 fn test_boot_from_vhost_user_blk_default() { 2974 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 2975 } 2976 2977 #[test] 2978 #[cfg(target_arch = "x86_64")] 2979 fn test_split_irqchip() { 2980 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2981 let guest = Guest::new(Box::new(focal)); 2982 2983 let mut child = GuestCommand::new(&guest) 2984 .args(["--cpus", "boot=1"]) 2985 .args(["--memory", "size=512M"]) 2986 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2987 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2988 .default_disks() 2989 .default_net() 2990 .capture_output() 2991 .spawn() 2992 .unwrap(); 2993 2994 let r = std::panic::catch_unwind(|| { 2995 guest.wait_vm_boot(None).unwrap(); 2996 2997 assert_eq!( 2998 guest 2999 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3000 .unwrap() 3001 .trim() 3002 .parse::<u32>() 3003 .unwrap_or(1), 3004 0 3005 ); 3006 assert_eq!( 3007 guest 3008 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3009 .unwrap() 3010 .trim() 3011 .parse::<u32>() 3012 .unwrap_or(1), 3013 0 3014 ); 3015 }); 3016 3017 let _ = child.kill(); 3018 let output = child.wait_with_output().unwrap(); 3019 3020 handle_child_output(r, &output); 3021 } 3022 3023 #[test] 3024 #[cfg(target_arch = "x86_64")] 3025 fn test_dmi_serial_number() { 3026 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3027 let guest = Guest::new(Box::new(focal)); 3028 3029 let mut child = GuestCommand::new(&guest) 3030 .args(["--cpus", "boot=1"]) 3031 .args(["--memory", "size=512M"]) 3032 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3033 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3034 .args(["--platform", "serial_number=a=b;c=d"]) 3035 .default_disks() 3036 .default_net() 3037 .capture_output() 3038 .spawn() 3039 .unwrap(); 3040 3041 let r = std::panic::catch_unwind(|| { 3042 guest.wait_vm_boot(None).unwrap(); 3043 3044 assert_eq!( 3045 guest 3046 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3047 .unwrap() 3048 .trim(), 3049 "a=b;c=d" 3050 ); 3051 }); 3052 3053 let _ = child.kill(); 3054 let output = child.wait_with_output().unwrap(); 3055 3056 handle_child_output(r, &output); 3057 } 3058 3059 #[test] 3060 #[cfg(target_arch = "x86_64")] 3061 fn test_dmi_uuid() { 3062 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3063 let guest = Guest::new(Box::new(focal)); 3064 3065 let mut child = GuestCommand::new(&guest) 3066 .args(["--cpus", "boot=1"]) 3067 .args(["--memory", "size=512M"]) 3068 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3069 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3070 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3071 .default_disks() 3072 .default_net() 3073 .capture_output() 3074 .spawn() 3075 .unwrap(); 3076 3077 let r = std::panic::catch_unwind(|| { 3078 guest.wait_vm_boot(None).unwrap(); 3079 3080 assert_eq!( 3081 guest 3082 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3083 .unwrap() 3084 .trim(), 3085 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3086 ); 3087 }); 3088 3089 let _ = child.kill(); 3090 let output = child.wait_with_output().unwrap(); 3091 3092 handle_child_output(r, &output); 3093 } 3094 3095 #[test] 3096 #[cfg(target_arch = "x86_64")] 3097 fn test_dmi_oem_strings() { 3098 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3099 let guest = Guest::new(Box::new(focal)); 3100 3101 let s1 = "io.systemd.credential:xx=yy"; 3102 let s2 = "This is a test string"; 3103 3104 let oem_strings = format!("oem_strings=[{},{}]", s1, s2); 3105 3106 let mut child = GuestCommand::new(&guest) 3107 .args(["--cpus", "boot=1"]) 3108 .args(["--memory", "size=512M"]) 3109 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3110 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3111 .args(["--platform", &oem_strings]) 3112 .default_disks() 3113 .default_net() 3114 .capture_output() 3115 .spawn() 3116 .unwrap(); 3117 3118 let r = std::panic::catch_unwind(|| { 3119 guest.wait_vm_boot(None).unwrap(); 3120 3121 assert_eq!( 3122 guest 3123 .ssh_command("sudo dmidecode --oem-string count") 3124 .unwrap() 3125 .trim(), 3126 "2" 3127 ); 3128 3129 assert_eq!( 3130 guest 3131 .ssh_command("sudo dmidecode --oem-string 1") 3132 .unwrap() 3133 .trim(), 3134 s1 3135 ); 3136 3137 assert_eq!( 3138 guest 3139 .ssh_command("sudo dmidecode --oem-string 2") 3140 .unwrap() 3141 .trim(), 3142 s2 3143 ); 3144 }); 3145 3146 let _ = child.kill(); 3147 let output = child.wait_with_output().unwrap(); 3148 3149 handle_child_output(r, &output); 3150 } 3151 3152 #[test] 3153 fn test_virtio_fs() { 3154 _test_virtio_fs(&prepare_virtiofsd, false, None) 3155 } 3156 3157 #[test] 3158 fn test_virtio_fs_hotplug() { 3159 _test_virtio_fs(&prepare_virtiofsd, true, None) 3160 } 3161 3162 #[test] 3163 #[cfg(not(feature = "mshv"))] 3164 fn test_virtio_fs_multi_segment_hotplug() { 3165 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3166 } 3167 3168 #[test] 3169 #[cfg(not(feature = "mshv"))] 3170 fn test_virtio_fs_multi_segment() { 3171 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3172 } 3173 3174 #[test] 3175 fn test_virtio_pmem_persist_writes() { 3176 test_virtio_pmem(false, false) 3177 } 3178 3179 #[test] 3180 fn test_virtio_pmem_discard_writes() { 3181 test_virtio_pmem(true, false) 3182 } 3183 3184 #[test] 3185 fn test_virtio_pmem_with_size() { 3186 test_virtio_pmem(true, true) 3187 } 3188 3189 #[test] 3190 fn test_boot_from_virtio_pmem() { 3191 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3192 let guest = Guest::new(Box::new(focal)); 3193 3194 let kernel_path = direct_kernel_boot_path(); 3195 3196 let mut child = GuestCommand::new(&guest) 3197 .args(["--cpus", "boot=1"]) 3198 .args(["--memory", "size=512M"]) 3199 .args(["--kernel", kernel_path.to_str().unwrap()]) 3200 .args([ 3201 "--disk", 3202 format!( 3203 "path={}", 3204 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3205 ) 3206 .as_str(), 3207 ]) 3208 .default_net() 3209 .args([ 3210 "--pmem", 3211 format!( 3212 "file={},size={}", 3213 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3214 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3215 .unwrap() 3216 .len() 3217 ) 3218 .as_str(), 3219 ]) 3220 .args([ 3221 "--cmdline", 3222 DIRECT_KERNEL_BOOT_CMDLINE 3223 .replace("vda1", "pmem0p1") 3224 .as_str(), 3225 ]) 3226 .capture_output() 3227 .spawn() 3228 .unwrap(); 3229 3230 let r = std::panic::catch_unwind(|| { 3231 guest.wait_vm_boot(None).unwrap(); 3232 3233 // Simple checks to validate the VM booted properly 3234 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3235 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3236 }); 3237 3238 let _ = child.kill(); 3239 let output = child.wait_with_output().unwrap(); 3240 3241 handle_child_output(r, &output); 3242 } 3243 3244 #[test] 3245 fn test_multiple_network_interfaces() { 3246 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3247 let guest = Guest::new(Box::new(focal)); 3248 3249 let kernel_path = direct_kernel_boot_path(); 3250 3251 let mut child = GuestCommand::new(&guest) 3252 .args(["--cpus", "boot=1"]) 3253 .args(["--memory", "size=512M"]) 3254 .args(["--kernel", kernel_path.to_str().unwrap()]) 3255 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3256 .default_disks() 3257 .args([ 3258 "--net", 3259 guest.default_net_string().as_str(), 3260 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3261 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3262 ]) 3263 .capture_output() 3264 .spawn() 3265 .unwrap(); 3266 3267 let r = std::panic::catch_unwind(|| { 3268 guest.wait_vm_boot(None).unwrap(); 3269 3270 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3271 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3272 3273 // 3 network interfaces + default localhost ==> 4 interfaces 3274 assert_eq!( 3275 guest 3276 .ssh_command("ip -o link | wc -l") 3277 .unwrap() 3278 .trim() 3279 .parse::<u32>() 3280 .unwrap_or_default(), 3281 4 3282 ); 3283 }); 3284 3285 let _ = child.kill(); 3286 let output = child.wait_with_output().unwrap(); 3287 3288 handle_child_output(r, &output); 3289 } 3290 3291 #[test] 3292 #[cfg(target_arch = "aarch64")] 3293 fn test_pmu_on() { 3294 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3295 let guest = Guest::new(Box::new(focal)); 3296 let mut child = GuestCommand::new(&guest) 3297 .args(["--cpus", "boot=1"]) 3298 .args(["--memory", "size=512M"]) 3299 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3300 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3301 .default_disks() 3302 .default_net() 3303 .capture_output() 3304 .spawn() 3305 .unwrap(); 3306 3307 let r = std::panic::catch_unwind(|| { 3308 guest.wait_vm_boot(None).unwrap(); 3309 3310 // Test that PMU exists. 3311 assert_eq!( 3312 guest 3313 .ssh_command(GREP_PMU_IRQ_CMD) 3314 .unwrap() 3315 .trim() 3316 .parse::<u32>() 3317 .unwrap_or_default(), 3318 1 3319 ); 3320 }); 3321 3322 let _ = child.kill(); 3323 let output = child.wait_with_output().unwrap(); 3324 3325 handle_child_output(r, &output); 3326 } 3327 3328 #[test] 3329 fn test_serial_off() { 3330 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3331 let guest = Guest::new(Box::new(focal)); 3332 let mut child = GuestCommand::new(&guest) 3333 .args(["--cpus", "boot=1"]) 3334 .args(["--memory", "size=512M"]) 3335 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3336 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3337 .default_disks() 3338 .default_net() 3339 .args(["--serial", "off"]) 3340 .capture_output() 3341 .spawn() 3342 .unwrap(); 3343 3344 let r = std::panic::catch_unwind(|| { 3345 guest.wait_vm_boot(None).unwrap(); 3346 3347 // Test that there is no ttyS0 3348 assert_eq!( 3349 guest 3350 .ssh_command(GREP_SERIAL_IRQ_CMD) 3351 .unwrap() 3352 .trim() 3353 .parse::<u32>() 3354 .unwrap_or(1), 3355 0 3356 ); 3357 }); 3358 3359 let _ = child.kill(); 3360 let output = child.wait_with_output().unwrap(); 3361 3362 handle_child_output(r, &output); 3363 } 3364 3365 #[test] 3366 fn test_serial_null() { 3367 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3368 let guest = Guest::new(Box::new(focal)); 3369 let mut cmd = GuestCommand::new(&guest); 3370 #[cfg(target_arch = "x86_64")] 3371 let console_str: &str = "console=ttyS0"; 3372 #[cfg(target_arch = "aarch64")] 3373 let console_str: &str = "console=ttyAMA0"; 3374 3375 cmd.args(["--cpus", "boot=1"]) 3376 .args(["--memory", "size=512M"]) 3377 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3378 .args([ 3379 "--cmdline", 3380 DIRECT_KERNEL_BOOT_CMDLINE 3381 .replace("console=hvc0 ", console_str) 3382 .as_str(), 3383 ]) 3384 .default_disks() 3385 .default_net() 3386 .args(["--serial", "null"]) 3387 .args(["--console", "off"]) 3388 .capture_output(); 3389 3390 let mut child = cmd.spawn().unwrap(); 3391 3392 let r = std::panic::catch_unwind(|| { 3393 guest.wait_vm_boot(None).unwrap(); 3394 3395 // Test that there is a ttyS0 3396 assert_eq!( 3397 guest 3398 .ssh_command(GREP_SERIAL_IRQ_CMD) 3399 .unwrap() 3400 .trim() 3401 .parse::<u32>() 3402 .unwrap_or_default(), 3403 1 3404 ); 3405 }); 3406 3407 let _ = child.kill(); 3408 let output = child.wait_with_output().unwrap(); 3409 handle_child_output(r, &output); 3410 3411 let r = std::panic::catch_unwind(|| { 3412 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3413 }); 3414 3415 handle_child_output(r, &output); 3416 } 3417 3418 #[test] 3419 fn test_serial_tty() { 3420 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3421 let guest = Guest::new(Box::new(focal)); 3422 3423 let kernel_path = direct_kernel_boot_path(); 3424 3425 #[cfg(target_arch = "x86_64")] 3426 let console_str: &str = "console=ttyS0"; 3427 #[cfg(target_arch = "aarch64")] 3428 let console_str: &str = "console=ttyAMA0"; 3429 3430 let mut child = GuestCommand::new(&guest) 3431 .args(["--cpus", "boot=1"]) 3432 .args(["--memory", "size=512M"]) 3433 .args(["--kernel", kernel_path.to_str().unwrap()]) 3434 .args([ 3435 "--cmdline", 3436 DIRECT_KERNEL_BOOT_CMDLINE 3437 .replace("console=hvc0 ", console_str) 3438 .as_str(), 3439 ]) 3440 .default_disks() 3441 .default_net() 3442 .args(["--serial", "tty"]) 3443 .args(["--console", "off"]) 3444 .capture_output() 3445 .spawn() 3446 .unwrap(); 3447 3448 let r = std::panic::catch_unwind(|| { 3449 guest.wait_vm_boot(None).unwrap(); 3450 3451 // Test that there is a ttyS0 3452 assert_eq!( 3453 guest 3454 .ssh_command(GREP_SERIAL_IRQ_CMD) 3455 .unwrap() 3456 .trim() 3457 .parse::<u32>() 3458 .unwrap_or_default(), 3459 1 3460 ); 3461 }); 3462 3463 // This sleep is needed to wait for the login prompt 3464 thread::sleep(std::time::Duration::new(2, 0)); 3465 3466 let _ = child.kill(); 3467 let output = child.wait_with_output().unwrap(); 3468 handle_child_output(r, &output); 3469 3470 let r = std::panic::catch_unwind(|| { 3471 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3472 }); 3473 3474 handle_child_output(r, &output); 3475 } 3476 3477 #[test] 3478 fn test_serial_file() { 3479 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3480 let guest = Guest::new(Box::new(focal)); 3481 3482 let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output"); 3483 #[cfg(target_arch = "x86_64")] 3484 let console_str: &str = "console=ttyS0"; 3485 #[cfg(target_arch = "aarch64")] 3486 let console_str: &str = "console=ttyAMA0"; 3487 3488 let mut child = GuestCommand::new(&guest) 3489 .args(["--cpus", "boot=1"]) 3490 .args(["--memory", "size=512M"]) 3491 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3492 .args([ 3493 "--cmdline", 3494 DIRECT_KERNEL_BOOT_CMDLINE 3495 .replace("console=hvc0 ", console_str) 3496 .as_str(), 3497 ]) 3498 .default_disks() 3499 .default_net() 3500 .args([ 3501 "--serial", 3502 format!("file={}", serial_path.to_str().unwrap()).as_str(), 3503 ]) 3504 .capture_output() 3505 .spawn() 3506 .unwrap(); 3507 3508 let r = std::panic::catch_unwind(|| { 3509 guest.wait_vm_boot(None).unwrap(); 3510 3511 // Test that there is a ttyS0 3512 assert_eq!( 3513 guest 3514 .ssh_command(GREP_SERIAL_IRQ_CMD) 3515 .unwrap() 3516 .trim() 3517 .parse::<u32>() 3518 .unwrap_or_default(), 3519 1 3520 ); 3521 3522 guest.ssh_command("sudo shutdown -h now").unwrap(); 3523 }); 3524 3525 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3526 let _ = child.kill(); 3527 let output = child.wait_with_output().unwrap(); 3528 handle_child_output(r, &output); 3529 3530 let r = std::panic::catch_unwind(|| { 3531 // Check that the cloud-hypervisor binary actually terminated 3532 assert!(output.status.success()); 3533 3534 // Do this check after shutdown of the VM as an easy way to ensure 3535 // all writes are flushed to disk 3536 let mut f = std::fs::File::open(serial_path).unwrap(); 3537 let mut buf = String::new(); 3538 f.read_to_string(&mut buf).unwrap(); 3539 assert!(buf.contains(CONSOLE_TEST_STRING)); 3540 }); 3541 3542 handle_child_output(r, &output); 3543 } 3544 3545 #[test] 3546 fn test_pty_interaction() { 3547 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3548 let guest = Guest::new(Box::new(focal)); 3549 let api_socket = temp_api_path(&guest.tmp_dir); 3550 let serial_option = if cfg!(target_arch = "x86_64") { 3551 " console=ttyS0" 3552 } else { 3553 " console=ttyAMA0" 3554 }; 3555 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 3556 3557 let mut child = GuestCommand::new(&guest) 3558 .args(["--cpus", "boot=1"]) 3559 .args(["--memory", "size=512M"]) 3560 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3561 .args(["--cmdline", &cmdline]) 3562 .default_disks() 3563 .default_net() 3564 .args(["--serial", "null"]) 3565 .args(["--console", "pty"]) 3566 .args(["--api-socket", &api_socket]) 3567 .spawn() 3568 .unwrap(); 3569 3570 let r = std::panic::catch_unwind(|| { 3571 guest.wait_vm_boot(None).unwrap(); 3572 // Get pty fd for console 3573 let console_path = get_pty_path(&api_socket, "console"); 3574 // TODO: Get serial pty test working 3575 let mut cf = std::fs::OpenOptions::new() 3576 .write(true) 3577 .read(true) 3578 .open(console_path) 3579 .unwrap(); 3580 3581 // Some dumb sleeps but we don't want to write 3582 // before the console is up and we don't want 3583 // to try and write the next line before the 3584 // login process is ready. 3585 thread::sleep(std::time::Duration::new(5, 0)); 3586 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 3587 thread::sleep(std::time::Duration::new(2, 0)); 3588 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 3589 thread::sleep(std::time::Duration::new(2, 0)); 3590 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 3591 thread::sleep(std::time::Duration::new(2, 0)); 3592 3593 // read pty and ensure they have a login shell 3594 // some fairly hacky workarounds to avoid looping 3595 // forever in case the channel is blocked getting output 3596 let ptyc = pty_read(cf); 3597 let mut empty = 0; 3598 let mut prev = String::new(); 3599 loop { 3600 thread::sleep(std::time::Duration::new(2, 0)); 3601 match ptyc.try_recv() { 3602 Ok(line) => { 3603 empty = 0; 3604 prev = prev + &line; 3605 if prev.contains("test_pty_console") { 3606 break; 3607 } 3608 } 3609 Err(mpsc::TryRecvError::Empty) => { 3610 empty += 1; 3611 assert!(empty <= 5, "No login on pty"); 3612 } 3613 _ => panic!("No login on pty"), 3614 } 3615 } 3616 3617 guest.ssh_command("sudo shutdown -h now").unwrap(); 3618 }); 3619 3620 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3621 let _ = child.kill(); 3622 let output = child.wait_with_output().unwrap(); 3623 handle_child_output(r, &output); 3624 3625 let r = std::panic::catch_unwind(|| { 3626 // Check that the cloud-hypervisor binary actually terminated 3627 assert!(output.status.success()) 3628 }); 3629 handle_child_output(r, &output); 3630 } 3631 3632 #[test] 3633 fn test_virtio_console() { 3634 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3635 let guest = Guest::new(Box::new(focal)); 3636 3637 let kernel_path = direct_kernel_boot_path(); 3638 3639 let mut child = GuestCommand::new(&guest) 3640 .args(["--cpus", "boot=1"]) 3641 .args(["--memory", "size=512M"]) 3642 .args(["--kernel", kernel_path.to_str().unwrap()]) 3643 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3644 .default_disks() 3645 .default_net() 3646 .args(["--console", "tty"]) 3647 .args(["--serial", "null"]) 3648 .capture_output() 3649 .spawn() 3650 .unwrap(); 3651 3652 let text = String::from("On a branch floating down river a cricket, singing."); 3653 let cmd = format!("echo {} | sudo tee /dev/hvc0", text); 3654 3655 let r = std::panic::catch_unwind(|| { 3656 guest.wait_vm_boot(None).unwrap(); 3657 3658 assert!(guest 3659 .does_device_vendor_pair_match("0x1043", "0x1af4") 3660 .unwrap_or_default()); 3661 3662 guest.ssh_command(&cmd).unwrap(); 3663 }); 3664 3665 let _ = child.kill(); 3666 let output = child.wait_with_output().unwrap(); 3667 handle_child_output(r, &output); 3668 3669 let r = std::panic::catch_unwind(|| { 3670 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 3671 }); 3672 3673 handle_child_output(r, &output); 3674 } 3675 3676 #[test] 3677 fn test_console_file() { 3678 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3679 let guest = Guest::new(Box::new(focal)); 3680 3681 let console_path = guest.tmp_dir.as_path().join("/tmp/console-output"); 3682 let mut child = GuestCommand::new(&guest) 3683 .args(["--cpus", "boot=1"]) 3684 .args(["--memory", "size=512M"]) 3685 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3686 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3687 .default_disks() 3688 .default_net() 3689 .args([ 3690 "--console", 3691 format!("file={}", console_path.to_str().unwrap()).as_str(), 3692 ]) 3693 .capture_output() 3694 .spawn() 3695 .unwrap(); 3696 3697 guest.wait_vm_boot(None).unwrap(); 3698 3699 guest.ssh_command("sudo shutdown -h now").unwrap(); 3700 3701 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3702 let _ = child.kill(); 3703 let output = child.wait_with_output().unwrap(); 3704 3705 let r = std::panic::catch_unwind(|| { 3706 // Check that the cloud-hypervisor binary actually terminated 3707 assert!(output.status.success()); 3708 3709 // Do this check after shutdown of the VM as an easy way to ensure 3710 // all writes are flushed to disk 3711 let mut f = std::fs::File::open(console_path).unwrap(); 3712 let mut buf = String::new(); 3713 f.read_to_string(&mut buf).unwrap(); 3714 3715 if !buf.contains(CONSOLE_TEST_STRING) { 3716 eprintln!( 3717 "\n\n==== Console file output ====\n\n{}\n\n==== End console file output ====", 3718 buf 3719 ); 3720 } 3721 assert!(buf.contains(CONSOLE_TEST_STRING)); 3722 }); 3723 3724 handle_child_output(r, &output); 3725 } 3726 3727 #[test] 3728 #[cfg(target_arch = "x86_64")] 3729 #[cfg(not(feature = "mshv"))] 3730 #[ignore = "See #4324"] 3731 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 3732 // backed networking interfaces, bound through a simple bridge on the host. 3733 // So if the nested cloud-hypervisor succeeds in getting a directly 3734 // assigned interface from its cloud-hypervisor host, we should be able to 3735 // ssh into it, and verify that it's running with the right kernel command 3736 // line (We tag the command line from cloud-hypervisor for that purpose). 3737 // The third device is added to validate that hotplug works correctly since 3738 // it is being added to the L2 VM through hotplugging mechanism. 3739 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 3740 // vfio device support 3741 fn test_vfio() { 3742 setup_vfio_network_interfaces(); 3743 3744 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3745 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 3746 3747 let mut workload_path = dirs::home_dir().unwrap(); 3748 workload_path.push("workloads"); 3749 3750 let kernel_path = direct_kernel_boot_path(); 3751 3752 let mut vfio_path = workload_path.clone(); 3753 vfio_path.push("vfio"); 3754 3755 let mut cloud_init_vfio_base_path = vfio_path.clone(); 3756 cloud_init_vfio_base_path.push("cloudinit.img"); 3757 3758 // We copy our cloudinit into the vfio mount point, for the nested 3759 // cloud-hypervisor guest to use. 3760 rate_limited_copy( 3761 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 3762 &cloud_init_vfio_base_path, 3763 ) 3764 .expect("copying of cloud-init disk failed"); 3765 3766 let mut vfio_disk_path = workload_path.clone(); 3767 vfio_disk_path.push("vfio.img"); 3768 3769 // Create the vfio disk image 3770 let output = Command::new("mkfs.ext4") 3771 .arg("-d") 3772 .arg(vfio_path.to_str().unwrap()) 3773 .arg(vfio_disk_path.to_str().unwrap()) 3774 .arg("2g") 3775 .output() 3776 .unwrap(); 3777 if !output.status.success() { 3778 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 3779 panic!("mkfs.ext4 command generated an error"); 3780 } 3781 3782 let mut blk_file_path = workload_path; 3783 blk_file_path.push("blk.img"); 3784 3785 let vfio_tap0 = "vfio-tap0"; 3786 let vfio_tap1 = "vfio-tap1"; 3787 let vfio_tap2 = "vfio-tap2"; 3788 let vfio_tap3 = "vfio-tap3"; 3789 3790 let mut child = GuestCommand::new(&guest) 3791 .args(["--cpus", "boot=4"]) 3792 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 3793 .args(["--kernel", kernel_path.to_str().unwrap()]) 3794 .args([ 3795 "--disk", 3796 format!( 3797 "path={}", 3798 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3799 ) 3800 .as_str(), 3801 format!( 3802 "path={}", 3803 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3804 ) 3805 .as_str(), 3806 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 3807 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 3808 ]) 3809 .args([ 3810 "--cmdline", 3811 format!( 3812 "{} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts", 3813 DIRECT_KERNEL_BOOT_CMDLINE 3814 ) 3815 .as_str(), 3816 ]) 3817 .args([ 3818 "--net", 3819 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 3820 format!( 3821 "tap={},mac={},iommu=on", 3822 vfio_tap1, guest.network.l2_guest_mac1 3823 ) 3824 .as_str(), 3825 format!( 3826 "tap={},mac={},iommu=on", 3827 vfio_tap2, guest.network.l2_guest_mac2 3828 ) 3829 .as_str(), 3830 format!( 3831 "tap={},mac={},iommu=on", 3832 vfio_tap3, guest.network.l2_guest_mac3 3833 ) 3834 .as_str(), 3835 ]) 3836 .capture_output() 3837 .spawn() 3838 .unwrap(); 3839 3840 thread::sleep(std::time::Duration::new(30, 0)); 3841 3842 let r = std::panic::catch_unwind(|| { 3843 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 3844 thread::sleep(std::time::Duration::new(120, 0)); 3845 3846 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 3847 // added to its kernel command line. 3848 // Let's ssh into it and verify that it's there. If it is it means 3849 // we're in the right guest (The L2 one) because the QEMU L1 guest 3850 // does not have this command line tag. 3851 assert_eq!( 3852 guest 3853 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 3854 .unwrap() 3855 .trim() 3856 .parse::<u32>() 3857 .unwrap_or_default(), 3858 1 3859 ); 3860 3861 // Let's also verify from the second virtio-net device passed to 3862 // the L2 VM. 3863 assert_eq!( 3864 guest 3865 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 3866 .unwrap() 3867 .trim() 3868 .parse::<u32>() 3869 .unwrap_or_default(), 3870 1 3871 ); 3872 3873 // Check the amount of PCI devices appearing in L2 VM. 3874 assert_eq!( 3875 guest 3876 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3877 .unwrap() 3878 .trim() 3879 .parse::<u32>() 3880 .unwrap_or_default(), 3881 8, 3882 ); 3883 3884 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 3885 assert_eq!( 3886 guest 3887 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 3888 .unwrap() 3889 .trim() 3890 .parse::<u32>() 3891 .unwrap_or_default(), 3892 1 3893 ); 3894 3895 // Hotplug an extra virtio-net device through L2 VM. 3896 guest 3897 .ssh_command_l1( 3898 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 3899 ) 3900 .unwrap(); 3901 guest 3902 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 3903 .unwrap(); 3904 let vfio_hotplug_output = guest 3905 .ssh_command_l1( 3906 "sudo /mnt/ch-remote \ 3907 --api-socket=/tmp/ch_api.sock \ 3908 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 3909 ) 3910 .unwrap(); 3911 assert!(vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}")); 3912 3913 thread::sleep(std::time::Duration::new(10, 0)); 3914 3915 // Let's also verify from the third virtio-net device passed to 3916 // the L2 VM. This third device has been hotplugged through the L2 3917 // VM, so this is our way to validate hotplug works for VFIO PCI. 3918 assert_eq!( 3919 guest 3920 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 3921 .unwrap() 3922 .trim() 3923 .parse::<u32>() 3924 .unwrap_or_default(), 3925 1 3926 ); 3927 3928 // Check the amount of PCI devices appearing in L2 VM. 3929 // There should be one more device than before, raising the count 3930 // up to 9 PCI devices. 3931 assert_eq!( 3932 guest 3933 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3934 .unwrap() 3935 .trim() 3936 .parse::<u32>() 3937 .unwrap_or_default(), 3938 9, 3939 ); 3940 3941 // Let's now verify that we can correctly remove the virtio-net 3942 // device through the "remove-device" command responsible for 3943 // unplugging VFIO devices. 3944 guest 3945 .ssh_command_l1( 3946 "sudo /mnt/ch-remote \ 3947 --api-socket=/tmp/ch_api.sock \ 3948 remove-device vfio123", 3949 ) 3950 .unwrap(); 3951 thread::sleep(std::time::Duration::new(10, 0)); 3952 3953 // Check the amount of PCI devices appearing in L2 VM is back down 3954 // to 8 devices. 3955 assert_eq!( 3956 guest 3957 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3958 .unwrap() 3959 .trim() 3960 .parse::<u32>() 3961 .unwrap_or_default(), 3962 8, 3963 ); 3964 3965 // Perform memory hotplug in L2 and validate the memory is showing 3966 // up as expected. In order to check, we will use the virtio-net 3967 // device already passed through L2 as a VFIO device, this will 3968 // verify that VFIO devices are functional with memory hotplug. 3969 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 3970 guest 3971 .ssh_command_l2_1( 3972 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 3973 ) 3974 .unwrap(); 3975 guest 3976 .ssh_command_l1( 3977 "sudo /mnt/ch-remote \ 3978 --api-socket=/tmp/ch_api.sock \ 3979 resize --memory=1073741824", 3980 ) 3981 .unwrap(); 3982 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 3983 }); 3984 3985 let _ = child.kill(); 3986 let output = child.wait_with_output().unwrap(); 3987 3988 cleanup_vfio_network_interfaces(); 3989 3990 handle_child_output(r, &output); 3991 } 3992 3993 #[test] 3994 fn test_direct_kernel_boot_noacpi() { 3995 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3996 let guest = Guest::new(Box::new(focal)); 3997 3998 let kernel_path = direct_kernel_boot_path(); 3999 4000 let mut child = GuestCommand::new(&guest) 4001 .args(["--cpus", "boot=1"]) 4002 .args(["--memory", "size=512M"]) 4003 .args(["--kernel", kernel_path.to_str().unwrap()]) 4004 .args([ 4005 "--cmdline", 4006 format!("{} acpi=off", DIRECT_KERNEL_BOOT_CMDLINE).as_str(), 4007 ]) 4008 .default_disks() 4009 .default_net() 4010 .capture_output() 4011 .spawn() 4012 .unwrap(); 4013 4014 let r = std::panic::catch_unwind(|| { 4015 guest.wait_vm_boot(None).unwrap(); 4016 4017 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4018 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4019 }); 4020 4021 let _ = child.kill(); 4022 let output = child.wait_with_output().unwrap(); 4023 4024 handle_child_output(r, &output); 4025 } 4026 4027 #[test] 4028 fn test_virtio_vsock() { 4029 _test_virtio_vsock(false) 4030 } 4031 4032 #[test] 4033 fn test_virtio_vsock_hotplug() { 4034 _test_virtio_vsock(true); 4035 } 4036 4037 #[test] 4038 // Start cloud-hypervisor with no VM parameters, only the API server running. 4039 // From the API: Create a VM, boot it and check that it looks as expected. 4040 fn test_api_create_boot() { 4041 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4042 let guest = Guest::new(Box::new(focal)); 4043 4044 let api_socket = temp_api_path(&guest.tmp_dir); 4045 4046 let mut child = GuestCommand::new(&guest) 4047 .args(["--api-socket", &api_socket]) 4048 .capture_output() 4049 .spawn() 4050 .unwrap(); 4051 4052 thread::sleep(std::time::Duration::new(1, 0)); 4053 4054 // Verify API server is running 4055 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4056 4057 // Create the VM first 4058 let cpu_count: u8 = 4; 4059 let http_body = guest.api_create_body( 4060 cpu_count, 4061 direct_kernel_boot_path().to_str().unwrap(), 4062 DIRECT_KERNEL_BOOT_CMDLINE, 4063 ); 4064 4065 let temp_config_path = guest.tmp_dir.as_path().join("config"); 4066 std::fs::write(&temp_config_path, http_body).unwrap(); 4067 4068 remote_command( 4069 &api_socket, 4070 "create", 4071 Some(temp_config_path.as_os_str().to_str().unwrap()), 4072 ); 4073 4074 // Then boot it 4075 remote_command(&api_socket, "boot", None); 4076 thread::sleep(std::time::Duration::new(20, 0)); 4077 4078 let r = std::panic::catch_unwind(|| { 4079 // Check that the VM booted as expected 4080 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4081 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4082 }); 4083 4084 let _ = child.kill(); 4085 let output = child.wait_with_output().unwrap(); 4086 4087 handle_child_output(r, &output); 4088 } 4089 4090 #[test] 4091 // Start cloud-hypervisor with no VM parameters, only the API server running. 4092 // From the API: Create a VM, boot it and check it can be shutdown and then 4093 // booted again 4094 fn test_api_shutdown() { 4095 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4096 let guest = Guest::new(Box::new(focal)); 4097 4098 let api_socket = temp_api_path(&guest.tmp_dir); 4099 4100 let mut child = GuestCommand::new(&guest) 4101 .args(["--api-socket", &api_socket]) 4102 .capture_output() 4103 .spawn() 4104 .unwrap(); 4105 4106 thread::sleep(std::time::Duration::new(1, 0)); 4107 4108 // Verify API server is running 4109 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4110 4111 // Create the VM first 4112 let cpu_count: u8 = 4; 4113 let http_body = guest.api_create_body( 4114 cpu_count, 4115 direct_kernel_boot_path().to_str().unwrap(), 4116 DIRECT_KERNEL_BOOT_CMDLINE, 4117 ); 4118 4119 let r = std::panic::catch_unwind(|| { 4120 curl_command( 4121 &api_socket, 4122 "PUT", 4123 "http://localhost/api/v1/vm.create", 4124 Some(&http_body), 4125 ); 4126 4127 // Then boot it 4128 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4129 4130 guest.wait_vm_boot(None).unwrap(); 4131 4132 // Check that the VM booted as expected 4133 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4134 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4135 4136 // Sync and shutdown without powering off to prevent filesystem 4137 // corruption. 4138 guest.ssh_command("sync").unwrap(); 4139 guest.ssh_command("sudo shutdown -H now").unwrap(); 4140 4141 // Wait for the guest to be fully shutdown 4142 thread::sleep(std::time::Duration::new(20, 0)); 4143 4144 // Then shut it down 4145 curl_command( 4146 &api_socket, 4147 "PUT", 4148 "http://localhost/api/v1/vm.shutdown", 4149 None, 4150 ); 4151 4152 // Then boot it again 4153 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4154 4155 guest.wait_vm_boot(None).unwrap(); 4156 4157 // Check that the VM booted as expected 4158 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4159 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4160 }); 4161 4162 let _ = child.kill(); 4163 let output = child.wait_with_output().unwrap(); 4164 4165 handle_child_output(r, &output); 4166 } 4167 4168 #[test] 4169 // Start cloud-hypervisor with no VM parameters, only the API server running. 4170 // From the API: Create a VM, boot it and check it can be deleted and then recreated 4171 // booted again. 4172 fn test_api_delete() { 4173 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4174 let guest = Guest::new(Box::new(focal)); 4175 4176 let api_socket = temp_api_path(&guest.tmp_dir); 4177 4178 let mut child = GuestCommand::new(&guest) 4179 .args(["--api-socket", &api_socket]) 4180 .capture_output() 4181 .spawn() 4182 .unwrap(); 4183 4184 thread::sleep(std::time::Duration::new(1, 0)); 4185 4186 // Verify API server is running 4187 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4188 4189 // Create the VM first 4190 let cpu_count: u8 = 4; 4191 let http_body = guest.api_create_body( 4192 cpu_count, 4193 direct_kernel_boot_path().to_str().unwrap(), 4194 DIRECT_KERNEL_BOOT_CMDLINE, 4195 ); 4196 4197 let r = std::panic::catch_unwind(|| { 4198 curl_command( 4199 &api_socket, 4200 "PUT", 4201 "http://localhost/api/v1/vm.create", 4202 Some(&http_body), 4203 ); 4204 4205 // Then boot it 4206 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4207 4208 guest.wait_vm_boot(None).unwrap(); 4209 4210 // Check that the VM booted as expected 4211 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4212 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4213 4214 // Sync and shutdown without powering off to prevent filesystem 4215 // corruption. 4216 guest.ssh_command("sync").unwrap(); 4217 guest.ssh_command("sudo shutdown -H now").unwrap(); 4218 4219 // Wait for the guest to be fully shutdown 4220 thread::sleep(std::time::Duration::new(20, 0)); 4221 4222 // Then delete it 4223 curl_command( 4224 &api_socket, 4225 "PUT", 4226 "http://localhost/api/v1/vm.delete", 4227 None, 4228 ); 4229 4230 curl_command( 4231 &api_socket, 4232 "PUT", 4233 "http://localhost/api/v1/vm.create", 4234 Some(&http_body), 4235 ); 4236 4237 // Then boot it again 4238 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4239 4240 guest.wait_vm_boot(None).unwrap(); 4241 4242 // Check that the VM booted as expected 4243 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4244 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4245 }); 4246 4247 let _ = child.kill(); 4248 let output = child.wait_with_output().unwrap(); 4249 4250 handle_child_output(r, &output); 4251 } 4252 4253 #[test] 4254 // Start cloud-hypervisor with no VM parameters, only the API server running. 4255 // From the API: Create a VM, boot it and check that it looks as expected. 4256 // Then we pause the VM, check that it's no longer available. 4257 // Finally we resume the VM and check that it's available. 4258 fn test_api_pause_resume() { 4259 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4260 let guest = Guest::new(Box::new(focal)); 4261 4262 let api_socket = temp_api_path(&guest.tmp_dir); 4263 4264 let mut child = GuestCommand::new(&guest) 4265 .args(["--api-socket", &api_socket]) 4266 .capture_output() 4267 .spawn() 4268 .unwrap(); 4269 4270 thread::sleep(std::time::Duration::new(1, 0)); 4271 4272 // Verify API server is running 4273 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4274 4275 // Create the VM first 4276 let cpu_count: u8 = 4; 4277 let http_body = guest.api_create_body( 4278 cpu_count, 4279 direct_kernel_boot_path().to_str().unwrap(), 4280 DIRECT_KERNEL_BOOT_CMDLINE, 4281 ); 4282 curl_command( 4283 &api_socket, 4284 "PUT", 4285 "http://localhost/api/v1/vm.create", 4286 Some(&http_body), 4287 ); 4288 4289 // Then boot it 4290 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4291 thread::sleep(std::time::Duration::new(20, 0)); 4292 4293 let r = std::panic::catch_unwind(|| { 4294 // Check that the VM booted as expected 4295 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4296 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4297 4298 // We now pause the VM 4299 assert!(remote_command(&api_socket, "pause", None)); 4300 4301 // Check pausing again fails 4302 assert!(!remote_command(&api_socket, "pause", None)); 4303 4304 thread::sleep(std::time::Duration::new(2, 0)); 4305 4306 // SSH into the VM should fail 4307 assert!(ssh_command_ip( 4308 "grep -c processor /proc/cpuinfo", 4309 &guest.network.guest_ip, 4310 2, 4311 5 4312 ) 4313 .is_err()); 4314 4315 // Resume the VM 4316 assert!(remote_command(&api_socket, "resume", None)); 4317 4318 // Check resuming again fails 4319 assert!(!remote_command(&api_socket, "resume", None)); 4320 4321 thread::sleep(std::time::Duration::new(2, 0)); 4322 4323 // Now we should be able to SSH back in and get the right number of CPUs 4324 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4325 }); 4326 4327 let _ = child.kill(); 4328 let output = child.wait_with_output().unwrap(); 4329 4330 handle_child_output(r, &output); 4331 } 4332 4333 #[test] 4334 fn test_virtio_iommu() { 4335 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4336 } 4337 4338 #[test] 4339 // We cannot force the software running in the guest to reprogram the BAR 4340 // with some different addresses, but we have a reliable way of testing it 4341 // with a standard Linux kernel. 4342 // By removing a device from the PCI tree, and then rescanning the tree, 4343 // Linux consistently chooses to reorganize the PCI device BARs to other 4344 // locations in the guest address space. 4345 // This test creates a dedicated PCI network device to be checked as being 4346 // properly probed first, then removing it, and adding it again by doing a 4347 // rescan. 4348 fn test_pci_bar_reprogramming() { 4349 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4350 let guest = Guest::new(Box::new(focal)); 4351 4352 #[cfg(target_arch = "x86_64")] 4353 let kernel_path = direct_kernel_boot_path(); 4354 #[cfg(target_arch = "aarch64")] 4355 let kernel_path = edk2_path(); 4356 4357 let mut child = GuestCommand::new(&guest) 4358 .args(["--cpus", "boot=1"]) 4359 .args(["--memory", "size=512M"]) 4360 .args(["--kernel", kernel_path.to_str().unwrap()]) 4361 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4362 .default_disks() 4363 .args([ 4364 "--net", 4365 guest.default_net_string().as_str(), 4366 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4367 ]) 4368 .capture_output() 4369 .spawn() 4370 .unwrap(); 4371 4372 let r = std::panic::catch_unwind(|| { 4373 guest.wait_vm_boot(None).unwrap(); 4374 4375 // 2 network interfaces + default localhost ==> 3 interfaces 4376 assert_eq!( 4377 guest 4378 .ssh_command("ip -o link | wc -l") 4379 .unwrap() 4380 .trim() 4381 .parse::<u32>() 4382 .unwrap_or_default(), 4383 3 4384 ); 4385 4386 let init_bar_addr = guest 4387 .ssh_command( 4388 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4389 ) 4390 .unwrap(); 4391 4392 // Remove the PCI device 4393 guest 4394 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4395 .unwrap(); 4396 4397 // Only 1 network interface left + default localhost ==> 2 interfaces 4398 assert_eq!( 4399 guest 4400 .ssh_command("ip -o link | wc -l") 4401 .unwrap() 4402 .trim() 4403 .parse::<u32>() 4404 .unwrap_or_default(), 4405 2 4406 ); 4407 4408 // Remove the PCI device 4409 guest 4410 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4411 .unwrap(); 4412 4413 // Back to 2 network interface + default localhost ==> 3 interfaces 4414 assert_eq!( 4415 guest 4416 .ssh_command("ip -o link | wc -l") 4417 .unwrap() 4418 .trim() 4419 .parse::<u32>() 4420 .unwrap_or_default(), 4421 3 4422 ); 4423 4424 let new_bar_addr = guest 4425 .ssh_command( 4426 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4427 ) 4428 .unwrap(); 4429 4430 // Let's compare the BAR addresses for our virtio-net device. 4431 // They should be different as we expect the BAR reprogramming 4432 // to have happened. 4433 assert_ne!(init_bar_addr, new_bar_addr); 4434 }); 4435 4436 let _ = child.kill(); 4437 let output = child.wait_with_output().unwrap(); 4438 4439 handle_child_output(r, &output); 4440 } 4441 4442 #[test] 4443 fn test_memory_mergeable_off() { 4444 test_memory_mergeable(false) 4445 } 4446 4447 #[test] 4448 #[cfg(target_arch = "x86_64")] 4449 fn test_cpu_hotplug() { 4450 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4451 let guest = Guest::new(Box::new(focal)); 4452 let api_socket = temp_api_path(&guest.tmp_dir); 4453 4454 let kernel_path = direct_kernel_boot_path(); 4455 4456 let mut child = GuestCommand::new(&guest) 4457 .args(["--cpus", "boot=2,max=4"]) 4458 .args(["--memory", "size=512M"]) 4459 .args(["--kernel", kernel_path.to_str().unwrap()]) 4460 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4461 .default_disks() 4462 .default_net() 4463 .args(["--api-socket", &api_socket]) 4464 .capture_output() 4465 .spawn() 4466 .unwrap(); 4467 4468 let r = std::panic::catch_unwind(|| { 4469 guest.wait_vm_boot(None).unwrap(); 4470 4471 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4472 4473 // Resize the VM 4474 let desired_vcpus = 4; 4475 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4476 4477 guest 4478 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4479 .unwrap(); 4480 guest 4481 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4482 .unwrap(); 4483 thread::sleep(std::time::Duration::new(10, 0)); 4484 assert_eq!( 4485 guest.get_cpu_count().unwrap_or_default(), 4486 u32::from(desired_vcpus) 4487 ); 4488 4489 guest.reboot_linux(0, None); 4490 4491 assert_eq!( 4492 guest.get_cpu_count().unwrap_or_default(), 4493 u32::from(desired_vcpus) 4494 ); 4495 4496 // Resize the VM 4497 let desired_vcpus = 2; 4498 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4499 4500 thread::sleep(std::time::Duration::new(10, 0)); 4501 assert_eq!( 4502 guest.get_cpu_count().unwrap_or_default(), 4503 u32::from(desired_vcpus) 4504 ); 4505 4506 // Resize the VM back up to 4 4507 let desired_vcpus = 4; 4508 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4509 4510 guest 4511 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4512 .unwrap(); 4513 guest 4514 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4515 .unwrap(); 4516 thread::sleep(std::time::Duration::new(10, 0)); 4517 assert_eq!( 4518 guest.get_cpu_count().unwrap_or_default(), 4519 u32::from(desired_vcpus) 4520 ); 4521 }); 4522 4523 let _ = child.kill(); 4524 let output = child.wait_with_output().unwrap(); 4525 4526 handle_child_output(r, &output); 4527 } 4528 4529 #[test] 4530 fn test_memory_hotplug() { 4531 #[cfg(target_arch = "aarch64")] 4532 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4533 #[cfg(target_arch = "x86_64")] 4534 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4535 let focal = UbuntuDiskConfig::new(focal_image); 4536 let guest = Guest::new(Box::new(focal)); 4537 let api_socket = temp_api_path(&guest.tmp_dir); 4538 4539 #[cfg(target_arch = "aarch64")] 4540 let kernel_path = edk2_path(); 4541 #[cfg(target_arch = "x86_64")] 4542 let kernel_path = direct_kernel_boot_path(); 4543 4544 let mut child = GuestCommand::new(&guest) 4545 .args(["--cpus", "boot=2,max=4"]) 4546 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4547 .args(["--kernel", kernel_path.to_str().unwrap()]) 4548 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4549 .default_disks() 4550 .default_net() 4551 .args(["--balloon", "size=0"]) 4552 .args(["--api-socket", &api_socket]) 4553 .capture_output() 4554 .spawn() 4555 .unwrap(); 4556 4557 let r = std::panic::catch_unwind(|| { 4558 guest.wait_vm_boot(None).unwrap(); 4559 4560 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4561 4562 guest.enable_memory_hotplug(); 4563 4564 // Add RAM to the VM 4565 let desired_ram = 1024 << 20; 4566 resize_command(&api_socket, None, Some(desired_ram), None, None); 4567 4568 thread::sleep(std::time::Duration::new(10, 0)); 4569 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4570 4571 // Use balloon to remove RAM from the VM 4572 let desired_balloon = 512 << 20; 4573 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4574 4575 thread::sleep(std::time::Duration::new(10, 0)); 4576 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4577 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4578 4579 guest.reboot_linux(0, None); 4580 4581 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4582 4583 // Use balloon add RAM to the VM 4584 let desired_balloon = 0; 4585 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4586 4587 thread::sleep(std::time::Duration::new(10, 0)); 4588 4589 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4590 4591 guest.enable_memory_hotplug(); 4592 4593 // Add RAM to the VM 4594 let desired_ram = 2048 << 20; 4595 resize_command(&api_socket, None, Some(desired_ram), None, None); 4596 4597 thread::sleep(std::time::Duration::new(10, 0)); 4598 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4599 4600 // Remove RAM to the VM (only applies after reboot) 4601 let desired_ram = 1024 << 20; 4602 resize_command(&api_socket, None, Some(desired_ram), None, None); 4603 4604 guest.reboot_linux(1, None); 4605 4606 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4607 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4608 }); 4609 4610 let _ = child.kill(); 4611 let output = child.wait_with_output().unwrap(); 4612 4613 handle_child_output(r, &output); 4614 } 4615 4616 #[test] 4617 #[cfg(not(feature = "mshv"))] 4618 fn test_virtio_mem() { 4619 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4620 let guest = Guest::new(Box::new(focal)); 4621 let api_socket = temp_api_path(&guest.tmp_dir); 4622 4623 let kernel_path = direct_kernel_boot_path(); 4624 4625 let mut child = GuestCommand::new(&guest) 4626 .args(["--cpus", "boot=2,max=4"]) 4627 .args([ 4628 "--memory", 4629 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4630 ]) 4631 .args(["--kernel", kernel_path.to_str().unwrap()]) 4632 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4633 .default_disks() 4634 .default_net() 4635 .args(["--api-socket", &api_socket]) 4636 .capture_output() 4637 .spawn() 4638 .unwrap(); 4639 4640 let r = std::panic::catch_unwind(|| { 4641 guest.wait_vm_boot(None).unwrap(); 4642 4643 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4644 4645 guest.enable_memory_hotplug(); 4646 4647 // Add RAM to the VM 4648 let desired_ram = 1024 << 20; 4649 resize_command(&api_socket, None, Some(desired_ram), None, None); 4650 4651 thread::sleep(std::time::Duration::new(10, 0)); 4652 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4653 4654 // Add RAM to the VM 4655 let desired_ram = 2048 << 20; 4656 resize_command(&api_socket, None, Some(desired_ram), None, None); 4657 4658 thread::sleep(std::time::Duration::new(10, 0)); 4659 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4660 4661 // Remove RAM from the VM 4662 let desired_ram = 1024 << 20; 4663 resize_command(&api_socket, None, Some(desired_ram), None, None); 4664 4665 thread::sleep(std::time::Duration::new(10, 0)); 4666 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4667 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4668 4669 guest.reboot_linux(0, None); 4670 4671 // Check the amount of memory after reboot is 1GiB 4672 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4673 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4674 4675 // Check we can still resize to 512MiB 4676 let desired_ram = 512 << 20; 4677 resize_command(&api_socket, None, Some(desired_ram), None, None); 4678 thread::sleep(std::time::Duration::new(10, 0)); 4679 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4680 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4681 }); 4682 4683 let _ = child.kill(); 4684 let output = child.wait_with_output().unwrap(); 4685 4686 handle_child_output(r, &output); 4687 } 4688 4689 #[test] 4690 #[cfg(target_arch = "x86_64")] 4691 #[cfg(not(feature = "mshv"))] 4692 // Test both vCPU and memory resizing together 4693 fn test_resize() { 4694 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4695 let guest = Guest::new(Box::new(focal)); 4696 let api_socket = temp_api_path(&guest.tmp_dir); 4697 4698 let kernel_path = direct_kernel_boot_path(); 4699 4700 let mut child = GuestCommand::new(&guest) 4701 .args(["--cpus", "boot=2,max=4"]) 4702 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4703 .args(["--kernel", kernel_path.to_str().unwrap()]) 4704 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4705 .default_disks() 4706 .default_net() 4707 .args(["--api-socket", &api_socket]) 4708 .capture_output() 4709 .spawn() 4710 .unwrap(); 4711 4712 let r = std::panic::catch_unwind(|| { 4713 guest.wait_vm_boot(None).unwrap(); 4714 4715 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4716 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4717 4718 guest.enable_memory_hotplug(); 4719 4720 // Resize the VM 4721 let desired_vcpus = 4; 4722 let desired_ram = 1024 << 20; 4723 resize_command( 4724 &api_socket, 4725 Some(desired_vcpus), 4726 Some(desired_ram), 4727 None, 4728 None, 4729 ); 4730 4731 guest 4732 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4733 .unwrap(); 4734 guest 4735 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4736 .unwrap(); 4737 thread::sleep(std::time::Duration::new(10, 0)); 4738 assert_eq!( 4739 guest.get_cpu_count().unwrap_or_default(), 4740 u32::from(desired_vcpus) 4741 ); 4742 4743 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4744 }); 4745 4746 let _ = child.kill(); 4747 let output = child.wait_with_output().unwrap(); 4748 4749 handle_child_output(r, &output); 4750 } 4751 4752 #[test] 4753 fn test_memory_overhead() { 4754 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4755 let guest = Guest::new(Box::new(focal)); 4756 4757 let kernel_path = direct_kernel_boot_path(); 4758 4759 let guest_memory_size_kb = 512 * 1024; 4760 4761 let mut child = GuestCommand::new(&guest) 4762 .args(["--cpus", "boot=1"]) 4763 .args([ 4764 "--memory", 4765 format!("size={}K", guest_memory_size_kb).as_str(), 4766 ]) 4767 .args(["--kernel", kernel_path.to_str().unwrap()]) 4768 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4769 .default_disks() 4770 .capture_output() 4771 .spawn() 4772 .unwrap(); 4773 4774 thread::sleep(std::time::Duration::new(20, 0)); 4775 4776 let r = std::panic::catch_unwind(|| { 4777 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 4778 eprintln!( 4779 "Guest memory overhead: {} vs {}", 4780 overhead, MAXIMUM_VMM_OVERHEAD_KB 4781 ); 4782 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 4783 }); 4784 4785 let _ = child.kill(); 4786 let output = child.wait_with_output().unwrap(); 4787 4788 handle_child_output(r, &output); 4789 } 4790 4791 #[test] 4792 fn test_disk_hotplug() { 4793 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4794 let guest = Guest::new(Box::new(focal)); 4795 4796 #[cfg(target_arch = "x86_64")] 4797 let kernel_path = direct_kernel_boot_path(); 4798 #[cfg(target_arch = "aarch64")] 4799 let kernel_path = edk2_path(); 4800 4801 let api_socket = temp_api_path(&guest.tmp_dir); 4802 4803 let mut child = GuestCommand::new(&guest) 4804 .args(["--api-socket", &api_socket]) 4805 .args(["--cpus", "boot=1"]) 4806 .args(["--memory", "size=512M"]) 4807 .args(["--kernel", kernel_path.to_str().unwrap()]) 4808 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4809 .default_disks() 4810 .default_net() 4811 .capture_output() 4812 .spawn() 4813 .unwrap(); 4814 4815 let r = std::panic::catch_unwind(|| { 4816 guest.wait_vm_boot(None).unwrap(); 4817 4818 // Check /dev/vdc is not there 4819 assert_eq!( 4820 guest 4821 .ssh_command("lsblk | grep -c vdc.*16M || true") 4822 .unwrap() 4823 .trim() 4824 .parse::<u32>() 4825 .unwrap_or(1), 4826 0 4827 ); 4828 4829 // Now let's add the extra disk. 4830 let mut blk_file_path = dirs::home_dir().unwrap(); 4831 blk_file_path.push("workloads"); 4832 blk_file_path.push("blk.img"); 4833 let (cmd_success, cmd_output) = remote_command_w_output( 4834 &api_socket, 4835 "add-disk", 4836 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4837 ); 4838 assert!(cmd_success); 4839 assert!(String::from_utf8_lossy(&cmd_output) 4840 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4841 4842 thread::sleep(std::time::Duration::new(10, 0)); 4843 4844 // Check that /dev/vdc exists and the block size is 16M. 4845 assert_eq!( 4846 guest 4847 .ssh_command("lsblk | grep vdc | grep -c 16M") 4848 .unwrap() 4849 .trim() 4850 .parse::<u32>() 4851 .unwrap_or_default(), 4852 1 4853 ); 4854 // And check the block device can be read. 4855 guest 4856 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4857 .unwrap(); 4858 4859 // Let's remove it the extra disk. 4860 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4861 thread::sleep(std::time::Duration::new(5, 0)); 4862 // And check /dev/vdc is not there 4863 assert_eq!( 4864 guest 4865 .ssh_command("lsblk | grep -c vdc.*16M || true") 4866 .unwrap() 4867 .trim() 4868 .parse::<u32>() 4869 .unwrap_or(1), 4870 0 4871 ); 4872 4873 // And add it back to validate unplug did work correctly. 4874 let (cmd_success, cmd_output) = remote_command_w_output( 4875 &api_socket, 4876 "add-disk", 4877 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4878 ); 4879 assert!(cmd_success); 4880 assert!(String::from_utf8_lossy(&cmd_output) 4881 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4882 4883 thread::sleep(std::time::Duration::new(10, 0)); 4884 4885 // Check that /dev/vdc exists and the block size is 16M. 4886 assert_eq!( 4887 guest 4888 .ssh_command("lsblk | grep vdc | grep -c 16M") 4889 .unwrap() 4890 .trim() 4891 .parse::<u32>() 4892 .unwrap_or_default(), 4893 1 4894 ); 4895 // And check the block device can be read. 4896 guest 4897 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4898 .unwrap(); 4899 4900 // Reboot the VM. 4901 guest.reboot_linux(0, None); 4902 4903 // Check still there after reboot 4904 assert_eq!( 4905 guest 4906 .ssh_command("lsblk | grep vdc | grep -c 16M") 4907 .unwrap() 4908 .trim() 4909 .parse::<u32>() 4910 .unwrap_or_default(), 4911 1 4912 ); 4913 4914 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4915 4916 thread::sleep(std::time::Duration::new(20, 0)); 4917 4918 // Check device has gone away 4919 assert_eq!( 4920 guest 4921 .ssh_command("lsblk | grep -c vdc.*16M || true") 4922 .unwrap() 4923 .trim() 4924 .parse::<u32>() 4925 .unwrap_or(1), 4926 0 4927 ); 4928 4929 guest.reboot_linux(1, None); 4930 4931 // Check device still absent 4932 assert_eq!( 4933 guest 4934 .ssh_command("lsblk | grep -c vdc.*16M || true") 4935 .unwrap() 4936 .trim() 4937 .parse::<u32>() 4938 .unwrap_or(1), 4939 0 4940 ); 4941 }); 4942 4943 let _ = child.kill(); 4944 let output = child.wait_with_output().unwrap(); 4945 4946 handle_child_output(r, &output); 4947 } 4948 4949 #[allow(clippy::useless_conversion)] 4950 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 4951 const LOOP_CONFIGURE: u64 = 0x4c0a; 4952 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 4953 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 4954 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 4955 4956 #[repr(C)] 4957 struct LoopInfo64 { 4958 lo_device: u64, 4959 lo_inode: u64, 4960 lo_rdevice: u64, 4961 lo_offset: u64, 4962 lo_sizelimit: u64, 4963 lo_number: u32, 4964 lo_encrypt_type: u32, 4965 lo_encrypt_key_size: u32, 4966 lo_flags: u32, 4967 lo_file_name: [u8; 64], 4968 lo_crypt_name: [u8; 64], 4969 lo_encrypt_key: [u8; 32], 4970 lo_init: [u64; 2], 4971 } 4972 4973 impl Default for LoopInfo64 { 4974 fn default() -> Self { 4975 LoopInfo64 { 4976 lo_device: 0, 4977 lo_inode: 0, 4978 lo_rdevice: 0, 4979 lo_offset: 0, 4980 lo_sizelimit: 0, 4981 lo_number: 0, 4982 lo_encrypt_type: 0, 4983 lo_encrypt_key_size: 0, 4984 lo_flags: 0, 4985 lo_file_name: [0; 64], 4986 lo_crypt_name: [0; 64], 4987 lo_encrypt_key: [0; 32], 4988 lo_init: [0; 2], 4989 } 4990 } 4991 } 4992 4993 #[derive(Default)] 4994 #[repr(C)] 4995 struct LoopConfig { 4996 fd: u32, 4997 block_size: u32, 4998 info: LoopInfo64, 4999 _reserved: [u64; 8], 5000 } 5001 5002 // Open loop-control device 5003 let loop_ctl_file = OpenOptions::new() 5004 .read(true) 5005 .write(true) 5006 .open(LOOP_CTL_PATH) 5007 .unwrap(); 5008 5009 // Request a free loop device 5010 let loop_device_number = unsafe { 5011 libc::ioctl( 5012 loop_ctl_file.as_raw_fd(), 5013 LOOP_CTL_GET_FREE.try_into().unwrap(), 5014 ) 5015 }; 5016 if loop_device_number < 0 { 5017 panic!("Couldn't find a free loop device"); 5018 } 5019 5020 // Create loop device path 5021 let loop_device_path = format!("{}{}", LOOP_DEVICE_PREFIX, loop_device_number); 5022 5023 // Open loop device 5024 let loop_device_file = OpenOptions::new() 5025 .read(true) 5026 .write(true) 5027 .open(&loop_device_path) 5028 .unwrap(); 5029 5030 // Open backing file 5031 let backing_file = OpenOptions::new() 5032 .read(true) 5033 .write(true) 5034 .open(backing_file_path) 5035 .unwrap(); 5036 5037 let loop_config = LoopConfig { 5038 fd: backing_file.as_raw_fd() as u32, 5039 block_size, 5040 ..Default::default() 5041 }; 5042 5043 for i in 0..num_retries { 5044 let ret = unsafe { 5045 libc::ioctl( 5046 loop_device_file.as_raw_fd(), 5047 LOOP_CONFIGURE.try_into().unwrap(), 5048 &loop_config, 5049 ) 5050 }; 5051 if ret != 0 { 5052 if i < num_retries - 1 { 5053 println!( 5054 "Iteration {}: Failed to configure the loop device {}: {}", 5055 i, 5056 loop_device_path, 5057 std::io::Error::last_os_error() 5058 ); 5059 } else { 5060 panic!( 5061 "Failed {} times trying to configure the loop device {}: {}", 5062 num_retries, 5063 loop_device_path, 5064 std::io::Error::last_os_error() 5065 ); 5066 } 5067 } else { 5068 break; 5069 } 5070 5071 // Wait for a bit before retrying 5072 thread::sleep(std::time::Duration::new(5, 0)); 5073 } 5074 5075 loop_device_path 5076 } 5077 5078 #[test] 5079 fn test_virtio_block_topology() { 5080 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5081 let guest = Guest::new(Box::new(focal)); 5082 5083 let kernel_path = direct_kernel_boot_path(); 5084 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5085 5086 let output = exec_host_command_output( 5087 format!( 5088 "qemu-img create -f raw {} 16M", 5089 test_disk_path.to_str().unwrap() 5090 ) 5091 .as_str(), 5092 ); 5093 if !output.status.success() { 5094 let stdout = String::from_utf8_lossy(&output.stdout); 5095 let stderr = String::from_utf8_lossy(&output.stderr); 5096 panic!( 5097 "qemu-img command failed\nstdout\n{}\nstderr\n{}", 5098 stdout, stderr 5099 ); 5100 } 5101 5102 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5103 5104 let mut child = GuestCommand::new(&guest) 5105 .args(["--cpus", "boot=1"]) 5106 .args(["--memory", "size=512M"]) 5107 .args(["--kernel", kernel_path.to_str().unwrap()]) 5108 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5109 .args([ 5110 "--disk", 5111 format!( 5112 "path={}", 5113 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5114 ) 5115 .as_str(), 5116 format!( 5117 "path={}", 5118 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5119 ) 5120 .as_str(), 5121 format!("path={}", &loop_dev).as_str(), 5122 ]) 5123 .default_net() 5124 .capture_output() 5125 .spawn() 5126 .unwrap(); 5127 5128 let r = std::panic::catch_unwind(|| { 5129 guest.wait_vm_boot(None).unwrap(); 5130 5131 // MIN-IO column 5132 assert_eq!( 5133 guest 5134 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5135 .unwrap() 5136 .trim() 5137 .parse::<u32>() 5138 .unwrap_or_default(), 5139 4096 5140 ); 5141 // PHY-SEC column 5142 assert_eq!( 5143 guest 5144 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5145 .unwrap() 5146 .trim() 5147 .parse::<u32>() 5148 .unwrap_or_default(), 5149 4096 5150 ); 5151 // LOG-SEC column 5152 assert_eq!( 5153 guest 5154 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5155 .unwrap() 5156 .trim() 5157 .parse::<u32>() 5158 .unwrap_or_default(), 5159 4096 5160 ); 5161 }); 5162 5163 let _ = child.kill(); 5164 let output = child.wait_with_output().unwrap(); 5165 5166 handle_child_output(r, &output); 5167 5168 Command::new("losetup") 5169 .args(["-d", &loop_dev]) 5170 .output() 5171 .expect("loop device not found"); 5172 } 5173 5174 #[test] 5175 fn test_virtio_balloon_deflate_on_oom() { 5176 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5177 let guest = Guest::new(Box::new(focal)); 5178 5179 let kernel_path = direct_kernel_boot_path(); 5180 5181 let api_socket = temp_api_path(&guest.tmp_dir); 5182 5183 //Let's start a 4G guest with balloon occupied 2G memory 5184 let mut child = GuestCommand::new(&guest) 5185 .args(["--api-socket", &api_socket]) 5186 .args(["--cpus", "boot=1"]) 5187 .args(["--memory", "size=4G"]) 5188 .args(["--kernel", kernel_path.to_str().unwrap()]) 5189 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5190 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5191 .default_disks() 5192 .default_net() 5193 .capture_output() 5194 .spawn() 5195 .unwrap(); 5196 5197 let r = std::panic::catch_unwind(|| { 5198 guest.wait_vm_boot(None).unwrap(); 5199 5200 // Wait for balloon memory's initialization and check its size. 5201 // The virtio-balloon driver might take a few seconds to report the 5202 // balloon effective size back to the VMM. 5203 thread::sleep(std::time::Duration::new(20, 0)); 5204 5205 let orig_balloon = balloon_size(&api_socket); 5206 println!("The original balloon memory size is {} bytes", orig_balloon); 5207 assert!(orig_balloon == 2147483648); 5208 5209 // Two steps to verify if the 'deflate_on_oom' parameter works. 5210 // 1st: run a command to trigger an OOM in the guest. 5211 guest 5212 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5213 .unwrap(); 5214 5215 // Give some time for the OOM to happen in the guest and be reported 5216 // back to the host. 5217 thread::sleep(std::time::Duration::new(20, 0)); 5218 5219 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5220 let deflated_balloon = balloon_size(&api_socket); 5221 println!( 5222 "After deflating, balloon memory size is {} bytes", 5223 deflated_balloon 5224 ); 5225 // Verify the balloon size deflated 5226 assert!(deflated_balloon < 2147483648); 5227 }); 5228 5229 let _ = child.kill(); 5230 let output = child.wait_with_output().unwrap(); 5231 5232 handle_child_output(r, &output); 5233 } 5234 5235 #[test] 5236 fn test_virtio_balloon_free_page_reporting() { 5237 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5238 let guest = Guest::new(Box::new(focal)); 5239 5240 //Let's start a 4G guest with balloon occupied 2G memory 5241 let mut child = GuestCommand::new(&guest) 5242 .args(["--cpus", "boot=1"]) 5243 .args(["--memory", "size=4G"]) 5244 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5245 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5246 .args(["--balloon", "size=0,free_page_reporting=on"]) 5247 .default_disks() 5248 .default_net() 5249 .capture_output() 5250 .spawn() 5251 .unwrap(); 5252 5253 let pid = child.id(); 5254 let r = std::panic::catch_unwind(|| { 5255 guest.wait_vm_boot(None).unwrap(); 5256 5257 // Check the initial RSS is less than 1GiB 5258 let rss = process_rss_kib(pid); 5259 println!("RSS {} < 1048576", rss); 5260 assert!(rss < 1048576); 5261 5262 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5263 // seconds 5264 let guest_ip = guest.network.guest_ip.clone(); 5265 thread::spawn(move || { 5266 ssh_command_ip( 5267 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5268 &guest_ip, 5269 DEFAULT_SSH_RETRIES, 5270 DEFAULT_SSH_TIMEOUT, 5271 ) 5272 .unwrap(); 5273 }); 5274 5275 // Wait for 50 seconds to make sure the stress command is consuming 5276 // the expected amount of memory. 5277 thread::sleep(std::time::Duration::new(50, 0)); 5278 let rss = process_rss_kib(pid); 5279 println!("RSS {} >= 2097152", rss); 5280 assert!(rss >= 2097152); 5281 5282 // Wait for an extra minute to make sure the stress command has 5283 // completed and that the guest reported the free pages to the VMM 5284 // through the virtio-balloon device. We expect the RSS to be under 5285 // 2GiB. 5286 thread::sleep(std::time::Duration::new(60, 0)); 5287 let rss = process_rss_kib(pid); 5288 println!("RSS {} < 2097152", rss); 5289 assert!(rss < 2097152); 5290 }); 5291 5292 let _ = child.kill(); 5293 let output = child.wait_with_output().unwrap(); 5294 5295 handle_child_output(r, &output); 5296 } 5297 5298 #[test] 5299 fn test_pmem_hotplug() { 5300 _test_pmem_hotplug(None) 5301 } 5302 5303 #[test] 5304 fn test_pmem_multi_segment_hotplug() { 5305 _test_pmem_hotplug(Some(15)) 5306 } 5307 5308 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5309 #[cfg(target_arch = "aarch64")] 5310 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5311 #[cfg(target_arch = "x86_64")] 5312 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5313 let focal = UbuntuDiskConfig::new(focal_image); 5314 let guest = Guest::new(Box::new(focal)); 5315 5316 #[cfg(target_arch = "x86_64")] 5317 let kernel_path = direct_kernel_boot_path(); 5318 #[cfg(target_arch = "aarch64")] 5319 let kernel_path = edk2_path(); 5320 5321 let api_socket = temp_api_path(&guest.tmp_dir); 5322 5323 let mut cmd = GuestCommand::new(&guest); 5324 5325 cmd.args(["--api-socket", &api_socket]) 5326 .args(["--cpus", "boot=1"]) 5327 .args(["--memory", "size=512M"]) 5328 .args(["--kernel", kernel_path.to_str().unwrap()]) 5329 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5330 .default_disks() 5331 .default_net() 5332 .capture_output(); 5333 5334 if pci_segment.is_some() { 5335 cmd.args(["--platform", "num_pci_segments=16"]); 5336 } 5337 5338 let mut child = cmd.spawn().unwrap(); 5339 5340 let r = std::panic::catch_unwind(|| { 5341 guest.wait_vm_boot(None).unwrap(); 5342 5343 // Check /dev/pmem0 is not there 5344 assert_eq!( 5345 guest 5346 .ssh_command("lsblk | grep -c pmem0 || true") 5347 .unwrap() 5348 .trim() 5349 .parse::<u32>() 5350 .unwrap_or(1), 5351 0 5352 ); 5353 5354 let pmem_temp_file = TempFile::new().unwrap(); 5355 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5356 let (cmd_success, cmd_output) = remote_command_w_output( 5357 &api_socket, 5358 "add-pmem", 5359 Some(&format!( 5360 "file={},id=test0{}", 5361 pmem_temp_file.as_path().to_str().unwrap(), 5362 if let Some(pci_segment) = pci_segment { 5363 format!(",pci_segment={}", pci_segment) 5364 } else { 5365 "".to_owned() 5366 } 5367 )), 5368 ); 5369 assert!(cmd_success); 5370 if let Some(pci_segment) = pci_segment { 5371 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5372 "{{\"id\":\"test0\",\"bdf\":\"{:04x}:00:01.0\"}}", 5373 pci_segment 5374 ))); 5375 } else { 5376 assert!(String::from_utf8_lossy(&cmd_output) 5377 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5378 } 5379 5380 // Check that /dev/pmem0 exists and the block size is 128M 5381 assert_eq!( 5382 guest 5383 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5384 .unwrap() 5385 .trim() 5386 .parse::<u32>() 5387 .unwrap_or_default(), 5388 1 5389 ); 5390 5391 guest.reboot_linux(0, None); 5392 5393 // Check still there after reboot 5394 assert_eq!( 5395 guest 5396 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5397 .unwrap() 5398 .trim() 5399 .parse::<u32>() 5400 .unwrap_or_default(), 5401 1 5402 ); 5403 5404 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5405 5406 thread::sleep(std::time::Duration::new(20, 0)); 5407 5408 // Check device has gone away 5409 assert_eq!( 5410 guest 5411 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5412 .unwrap() 5413 .trim() 5414 .parse::<u32>() 5415 .unwrap_or(1), 5416 0 5417 ); 5418 5419 guest.reboot_linux(1, None); 5420 5421 // Check still absent after reboot 5422 assert_eq!( 5423 guest 5424 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5425 .unwrap() 5426 .trim() 5427 .parse::<u32>() 5428 .unwrap_or(1), 5429 0 5430 ); 5431 }); 5432 5433 let _ = child.kill(); 5434 let output = child.wait_with_output().unwrap(); 5435 5436 handle_child_output(r, &output); 5437 } 5438 5439 #[test] 5440 fn test_net_hotplug() { 5441 _test_net_hotplug(None) 5442 } 5443 5444 #[test] 5445 fn test_net_multi_segment_hotplug() { 5446 _test_net_hotplug(Some(15)) 5447 } 5448 5449 fn _test_net_hotplug(pci_segment: Option<u16>) { 5450 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5451 let guest = Guest::new(Box::new(focal)); 5452 5453 #[cfg(target_arch = "x86_64")] 5454 let kernel_path = direct_kernel_boot_path(); 5455 #[cfg(target_arch = "aarch64")] 5456 let kernel_path = edk2_path(); 5457 5458 let api_socket = temp_api_path(&guest.tmp_dir); 5459 5460 // Boot without network 5461 let mut cmd = GuestCommand::new(&guest); 5462 5463 cmd.args(["--api-socket", &api_socket]) 5464 .args(["--cpus", "boot=1"]) 5465 .args(["--memory", "size=512M"]) 5466 .args(["--kernel", kernel_path.to_str().unwrap()]) 5467 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5468 .default_disks() 5469 .capture_output(); 5470 5471 if pci_segment.is_some() { 5472 cmd.args(["--platform", "num_pci_segments=16"]); 5473 } 5474 5475 let mut child = cmd.spawn().unwrap(); 5476 5477 thread::sleep(std::time::Duration::new(20, 0)); 5478 5479 let r = std::panic::catch_unwind(|| { 5480 // Add network 5481 let (cmd_success, cmd_output) = remote_command_w_output( 5482 &api_socket, 5483 "add-net", 5484 Some( 5485 format!( 5486 "{}{},id=test0", 5487 guest.default_net_string(), 5488 if let Some(pci_segment) = pci_segment { 5489 format!(",pci_segment={}", pci_segment) 5490 } else { 5491 "".to_owned() 5492 } 5493 ) 5494 .as_str(), 5495 ), 5496 ); 5497 assert!(cmd_success); 5498 5499 if let Some(pci_segment) = pci_segment { 5500 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5501 "{{\"id\":\"test0\",\"bdf\":\"{:04x}:00:01.0\"}}", 5502 pci_segment 5503 ))); 5504 } else { 5505 assert!(String::from_utf8_lossy(&cmd_output) 5506 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5507 } 5508 5509 thread::sleep(std::time::Duration::new(5, 0)); 5510 5511 // 1 network interfaces + default localhost ==> 2 interfaces 5512 assert_eq!( 5513 guest 5514 .ssh_command("ip -o link | wc -l") 5515 .unwrap() 5516 .trim() 5517 .parse::<u32>() 5518 .unwrap_or_default(), 5519 2 5520 ); 5521 5522 // Remove network 5523 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5524 thread::sleep(std::time::Duration::new(5, 0)); 5525 5526 let (cmd_success, cmd_output) = remote_command_w_output( 5527 &api_socket, 5528 "add-net", 5529 Some( 5530 format!( 5531 "{}{},id=test1", 5532 guest.default_net_string(), 5533 if let Some(pci_segment) = pci_segment { 5534 format!(",pci_segment={}", pci_segment) 5535 } else { 5536 "".to_owned() 5537 } 5538 ) 5539 .as_str(), 5540 ), 5541 ); 5542 assert!(cmd_success); 5543 5544 if let Some(pci_segment) = pci_segment { 5545 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5546 "{{\"id\":\"test1\",\"bdf\":\"{:04x}:00:01.0\"}}", 5547 pci_segment 5548 ))); 5549 } else { 5550 assert!(String::from_utf8_lossy(&cmd_output) 5551 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5552 } 5553 5554 thread::sleep(std::time::Duration::new(5, 0)); 5555 5556 // 1 network interfaces + default localhost ==> 2 interfaces 5557 assert_eq!( 5558 guest 5559 .ssh_command("ip -o link | wc -l") 5560 .unwrap() 5561 .trim() 5562 .parse::<u32>() 5563 .unwrap_or_default(), 5564 2 5565 ); 5566 5567 guest.reboot_linux(0, None); 5568 5569 // Check still there after reboot 5570 // 1 network interfaces + default localhost ==> 2 interfaces 5571 assert_eq!( 5572 guest 5573 .ssh_command("ip -o link | wc -l") 5574 .unwrap() 5575 .trim() 5576 .parse::<u32>() 5577 .unwrap_or_default(), 5578 2 5579 ); 5580 }); 5581 5582 let _ = child.kill(); 5583 let output = child.wait_with_output().unwrap(); 5584 5585 handle_child_output(r, &output); 5586 } 5587 5588 #[test] 5589 fn test_initramfs() { 5590 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5591 let guest = Guest::new(Box::new(focal)); 5592 let mut workload_path = dirs::home_dir().unwrap(); 5593 workload_path.push("workloads"); 5594 5595 #[cfg(target_arch = "x86_64")] 5596 let mut kernels = vec![direct_kernel_boot_path()]; 5597 #[cfg(target_arch = "aarch64")] 5598 let kernels = vec![direct_kernel_boot_path()]; 5599 5600 #[cfg(target_arch = "x86_64")] 5601 { 5602 let mut pvh_kernel_path = workload_path.clone(); 5603 pvh_kernel_path.push("vmlinux"); 5604 kernels.push(pvh_kernel_path); 5605 } 5606 5607 let mut initramfs_path = workload_path; 5608 initramfs_path.push("alpine_initramfs.img"); 5609 5610 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5611 let cmdline = format!("console=hvc0 quiet TEST_STRING={}", test_string); 5612 5613 kernels.iter().for_each(|k_path| { 5614 let mut child = GuestCommand::new(&guest) 5615 .args(["--kernel", k_path.to_str().unwrap()]) 5616 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 5617 .args(["--cmdline", &cmdline]) 5618 .capture_output() 5619 .spawn() 5620 .unwrap(); 5621 5622 thread::sleep(std::time::Duration::new(20, 0)); 5623 5624 let _ = child.kill(); 5625 let output = child.wait_with_output().unwrap(); 5626 5627 let r = std::panic::catch_unwind(|| { 5628 let s = String::from_utf8_lossy(&output.stdout); 5629 5630 assert_ne!(s.lines().position(|line| line == test_string), None); 5631 }); 5632 5633 handle_child_output(r, &output); 5634 }); 5635 } 5636 5637 // One thing to note about this test. The virtio-net device is heavily used 5638 // through each ssh command. There's no need to perform a dedicated test to 5639 // verify the migration went well for virtio-net. 5640 #[test] 5641 #[cfg(not(feature = "mshv"))] 5642 fn test_snapshot_restore_hotplug_virtiomem() { 5643 _test_snapshot_restore(true); 5644 } 5645 5646 #[test] 5647 fn test_snapshot_restore_basic() { 5648 _test_snapshot_restore(false); 5649 } 5650 5651 fn _test_snapshot_restore(use_hotplug: bool) { 5652 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5653 let guest = Guest::new(Box::new(focal)); 5654 let kernel_path = direct_kernel_boot_path(); 5655 5656 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 5657 5658 let net_id = "net123"; 5659 let net_params = format!( 5660 "id={},tap=,mac={},ip={},mask=255.255.255.0", 5661 net_id, guest.network.guest_mac, guest.network.host_ip 5662 ); 5663 let mut mem_params = "size=4G"; 5664 5665 if use_hotplug { 5666 mem_params = "size=4G,hotplug_method=virtio-mem,hotplug_size=32G" 5667 } 5668 5669 let cloudinit_params = format!( 5670 "path={},iommu=on", 5671 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5672 ); 5673 5674 let socket = temp_vsock_path(&guest.tmp_dir); 5675 let event_path = temp_event_monitor_path(&guest.tmp_dir); 5676 5677 let mut child = GuestCommand::new(&guest) 5678 .args(["--api-socket", &api_socket_source]) 5679 .args(["--event-monitor", format!("path={}", event_path).as_str()]) 5680 .args(["--cpus", "boot=4"]) 5681 .args(["--memory", mem_params]) 5682 .args(["--balloon", "size=0"]) 5683 .args(["--kernel", kernel_path.to_str().unwrap()]) 5684 .args([ 5685 "--disk", 5686 format!( 5687 "path={}", 5688 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5689 ) 5690 .as_str(), 5691 cloudinit_params.as_str(), 5692 ]) 5693 .args(["--net", net_params.as_str()]) 5694 .args(["--vsock", format!("cid=3,socket={}", socket).as_str()]) 5695 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5696 .capture_output() 5697 .spawn() 5698 .unwrap(); 5699 5700 let console_text = String::from("On a branch floating down river a cricket, singing."); 5701 // Create the snapshot directory 5702 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 5703 5704 let r = std::panic::catch_unwind(|| { 5705 guest.wait_vm_boot(None).unwrap(); 5706 5707 // Check the number of vCPUs 5708 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5709 // Check the guest RAM 5710 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5711 if use_hotplug { 5712 // Increase guest RAM with virtio-mem 5713 resize_command( 5714 &api_socket_source, 5715 None, 5716 Some(6 << 30), 5717 None, 5718 Some(&event_path), 5719 ); 5720 thread::sleep(std::time::Duration::new(5, 0)); 5721 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5722 // Use balloon to remove RAM from the VM 5723 resize_command( 5724 &api_socket_source, 5725 None, 5726 None, 5727 Some(1 << 30), 5728 Some(&event_path), 5729 ); 5730 thread::sleep(std::time::Duration::new(5, 0)); 5731 let total_memory = guest.get_total_memory().unwrap_or_default(); 5732 assert!(total_memory > 4_800_000); 5733 assert!(total_memory < 5_760_000); 5734 } 5735 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 5736 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5737 5738 // x86_64: We check that removing and adding back the virtio-net device 5739 // does not break the snapshot/restore support for virtio-pci. 5740 // This is an important thing to test as the hotplug will 5741 // trigger a PCI BAR reprogramming, which is a good way of 5742 // checking if the stored resources are correctly restored. 5743 // Unplug the virtio-net device 5744 // AArch64: Device hotplug is currently not supported, skipping here. 5745 #[cfg(target_arch = "x86_64")] 5746 { 5747 assert!(remote_command( 5748 &api_socket_source, 5749 "remove-device", 5750 Some(net_id), 5751 )); 5752 thread::sleep(std::time::Duration::new(10, 0)); 5753 let latest_events = [&MetaEvent { 5754 event: "device-removed".to_string(), 5755 device_id: Some(net_id.to_string()), 5756 }]; 5757 assert!(check_latest_events_exact(&latest_events, &event_path)); 5758 5759 // Plug the virtio-net device again 5760 assert!(remote_command( 5761 &api_socket_source, 5762 "add-net", 5763 Some(net_params.as_str()), 5764 )); 5765 thread::sleep(std::time::Duration::new(10, 0)); 5766 } 5767 5768 // Pause the VM 5769 assert!(remote_command(&api_socket_source, "pause", None)); 5770 let latest_events = [ 5771 &MetaEvent { 5772 event: "pausing".to_string(), 5773 device_id: None, 5774 }, 5775 &MetaEvent { 5776 event: "paused".to_string(), 5777 device_id: None, 5778 }, 5779 ]; 5780 assert!(check_latest_events_exact(&latest_events, &event_path)); 5781 5782 // Take a snapshot from the VM 5783 assert!(remote_command( 5784 &api_socket_source, 5785 "snapshot", 5786 Some(format!("file://{}", snapshot_dir).as_str()), 5787 )); 5788 5789 // Wait to make sure the snapshot is completed 5790 thread::sleep(std::time::Duration::new(10, 0)); 5791 5792 let latest_events = [ 5793 &MetaEvent { 5794 event: "snapshotting".to_string(), 5795 device_id: None, 5796 }, 5797 &MetaEvent { 5798 event: "snapshotted".to_string(), 5799 device_id: None, 5800 }, 5801 ]; 5802 assert!(check_latest_events_exact(&latest_events, &event_path)); 5803 }); 5804 5805 // Shutdown the source VM and check console output 5806 let _ = child.kill(); 5807 let output = child.wait_with_output().unwrap(); 5808 handle_child_output(r, &output); 5809 5810 let r = std::panic::catch_unwind(|| { 5811 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5812 }); 5813 5814 handle_child_output(r, &output); 5815 5816 // Remove the vsock socket file. 5817 Command::new("rm") 5818 .arg("-f") 5819 .arg(socket.as_str()) 5820 .output() 5821 .unwrap(); 5822 5823 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 5824 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 5825 5826 // Restore the VM from the snapshot 5827 let mut child = GuestCommand::new(&guest) 5828 .args(["--api-socket", &api_socket_restored]) 5829 .args([ 5830 "--event-monitor", 5831 format!("path={}", event_path_restored).as_str(), 5832 ]) 5833 .args([ 5834 "--restore", 5835 format!("source_url=file://{}", snapshot_dir).as_str(), 5836 ]) 5837 .capture_output() 5838 .spawn() 5839 .unwrap(); 5840 5841 // Wait for the VM to be restored 5842 thread::sleep(std::time::Duration::new(10, 0)); 5843 let expected_events = [ 5844 &MetaEvent { 5845 event: "starting".to_string(), 5846 device_id: None, 5847 }, 5848 &MetaEvent { 5849 event: "restoring".to_string(), 5850 device_id: None, 5851 }, 5852 ]; 5853 assert!(check_sequential_events_exact( 5854 &expected_events, 5855 &event_path_restored 5856 )); 5857 let latest_events = [&MetaEvent { 5858 event: "restored".to_string(), 5859 device_id: None, 5860 }]; 5861 assert!(check_latest_events_exact( 5862 &latest_events, 5863 &event_path_restored 5864 )); 5865 5866 let r = std::panic::catch_unwind(|| { 5867 // Resume the VM 5868 assert!(remote_command(&api_socket_restored, "resume", None)); 5869 let latest_events = [ 5870 &MetaEvent { 5871 event: "resuming".to_string(), 5872 device_id: None, 5873 }, 5874 &MetaEvent { 5875 event: "resumed".to_string(), 5876 device_id: None, 5877 }, 5878 ]; 5879 assert!(check_latest_events_exact( 5880 &latest_events, 5881 &event_path_restored 5882 )); 5883 5884 // Perform same checks to validate VM has been properly restored 5885 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5886 let total_memory = guest.get_total_memory().unwrap_or_default(); 5887 if !use_hotplug { 5888 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5889 } else { 5890 assert!(total_memory > 4_800_000); 5891 assert!(total_memory < 5_760_000); 5892 // Deflate balloon to restore entire RAM to the VM 5893 resize_command(&api_socket_restored, None, None, Some(0), None); 5894 thread::sleep(std::time::Duration::new(5, 0)); 5895 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5896 // Decrease guest RAM with virtio-mem 5897 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 5898 thread::sleep(std::time::Duration::new(5, 0)); 5899 let total_memory = guest.get_total_memory().unwrap_or_default(); 5900 assert!(total_memory > 4_800_000); 5901 assert!(total_memory < 5_760_000); 5902 } 5903 5904 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5905 }); 5906 // Shutdown the target VM and check console output 5907 let _ = child.kill(); 5908 let output = child.wait_with_output().unwrap(); 5909 handle_child_output(r, &output); 5910 5911 let r = std::panic::catch_unwind(|| { 5912 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5913 }); 5914 5915 handle_child_output(r, &output); 5916 } 5917 5918 #[test] 5919 fn test_counters() { 5920 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5921 let guest = Guest::new(Box::new(focal)); 5922 let api_socket = temp_api_path(&guest.tmp_dir); 5923 5924 let mut cmd = GuestCommand::new(&guest); 5925 cmd.args(["--cpus", "boot=1"]) 5926 .args(["--memory", "size=512M"]) 5927 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5928 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5929 .default_disks() 5930 .args(["--net", guest.default_net_string().as_str()]) 5931 .args(["--api-socket", &api_socket]) 5932 .capture_output(); 5933 5934 let mut child = cmd.spawn().unwrap(); 5935 5936 let r = std::panic::catch_unwind(|| { 5937 guest.wait_vm_boot(None).unwrap(); 5938 5939 let orig_counters = get_counters(&api_socket); 5940 guest 5941 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 5942 .unwrap(); 5943 5944 let new_counters = get_counters(&api_socket); 5945 5946 // Check that all the counters have increased 5947 assert!(new_counters > orig_counters); 5948 }); 5949 5950 let _ = child.kill(); 5951 let output = child.wait_with_output().unwrap(); 5952 5953 handle_child_output(r, &output); 5954 } 5955 5956 #[test] 5957 #[cfg(feature = "guest_debug")] 5958 fn test_coredump() { 5959 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5960 let guest = Guest::new(Box::new(focal)); 5961 let api_socket = temp_api_path(&guest.tmp_dir); 5962 5963 let mut cmd = GuestCommand::new(&guest); 5964 cmd.args(["--cpus", "boot=4"]) 5965 .args(["--memory", "size=4G"]) 5966 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 5967 .default_disks() 5968 .args(["--net", guest.default_net_string().as_str()]) 5969 .args(["--api-socket", &api_socket]) 5970 .capture_output(); 5971 5972 let mut child = cmd.spawn().unwrap(); 5973 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 5974 5975 let r = std::panic::catch_unwind(|| { 5976 guest.wait_vm_boot(None).unwrap(); 5977 5978 assert!(remote_command(&api_socket, "pause", None)); 5979 5980 assert!(remote_command( 5981 &api_socket, 5982 "coredump", 5983 Some(format!("file://{}", vmcore_file).as_str()), 5984 )); 5985 5986 // the num of CORE notes should equals to vcpu 5987 let readelf_core_num_cmd = format!( 5988 "readelf --all {} |grep CORE |grep -v Type |wc -l", 5989 vmcore_file 5990 ); 5991 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 5992 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 5993 5994 // the num of QEMU notes should equals to vcpu 5995 let readelf_vmm_num_cmd = format!("readelf --all {} |grep QEMU |wc -l", vmcore_file); 5996 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 5997 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 5998 }); 5999 6000 let _ = child.kill(); 6001 let output = child.wait_with_output().unwrap(); 6002 6003 handle_child_output(r, &output); 6004 } 6005 6006 #[test] 6007 fn test_watchdog() { 6008 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6009 let guest = Guest::new(Box::new(focal)); 6010 let api_socket = temp_api_path(&guest.tmp_dir); 6011 6012 let kernel_path = direct_kernel_boot_path(); 6013 6014 let mut cmd = GuestCommand::new(&guest); 6015 cmd.args(["--cpus", "boot=1"]) 6016 .args(["--memory", "size=512M"]) 6017 .args(["--kernel", kernel_path.to_str().unwrap()]) 6018 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6019 .default_disks() 6020 .args(["--net", guest.default_net_string().as_str()]) 6021 .args(["--watchdog"]) 6022 .args(["--api-socket", &api_socket]) 6023 .capture_output(); 6024 6025 let mut child = cmd.spawn().unwrap(); 6026 6027 let r = std::panic::catch_unwind(|| { 6028 guest.wait_vm_boot(None).unwrap(); 6029 6030 let mut expected_reboot_count = 1; 6031 6032 // Enable the watchdog with a 15s timeout 6033 enable_guest_watchdog(&guest, 15); 6034 6035 // Reboot and check that systemd has activated the watchdog 6036 guest.ssh_command("sudo reboot").unwrap(); 6037 guest.wait_vm_boot(None).unwrap(); 6038 expected_reboot_count += 1; 6039 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6040 assert_eq!( 6041 guest 6042 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6043 .unwrap() 6044 .trim() 6045 .parse::<u32>() 6046 .unwrap_or_default(), 6047 2 6048 ); 6049 6050 // Allow some normal time to elapse to check we don't get spurious reboots 6051 thread::sleep(std::time::Duration::new(40, 0)); 6052 // Check no reboot 6053 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6054 6055 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6056 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6057 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6058 guest.wait_vm_boot(Some(50)).unwrap(); 6059 // Check a reboot is triggerred by the watchdog 6060 expected_reboot_count += 1; 6061 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6062 6063 #[cfg(target_arch = "x86_64")] 6064 { 6065 // Now pause the VM and remain offline for 30s 6066 assert!(remote_command(&api_socket, "pause", None)); 6067 thread::sleep(std::time::Duration::new(30, 0)); 6068 assert!(remote_command(&api_socket, "resume", None)); 6069 6070 // Check no reboot 6071 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6072 } 6073 }); 6074 6075 let _ = child.kill(); 6076 let output = child.wait_with_output().unwrap(); 6077 6078 handle_child_output(r, &output); 6079 } 6080 6081 #[test] 6082 fn test_tap_from_fd() { 6083 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6084 let guest = Guest::new(Box::new(focal)); 6085 let kernel_path = direct_kernel_boot_path(); 6086 6087 // Create a TAP interface with multi-queue enabled 6088 let num_queue_pairs: usize = 2; 6089 6090 use std::str::FromStr; 6091 let taps = net_util::open_tap( 6092 Some("chtap0"), 6093 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6094 None, 6095 &mut None, 6096 None, 6097 num_queue_pairs, 6098 Some(libc::O_RDWR | libc::O_NONBLOCK), 6099 ) 6100 .unwrap(); 6101 6102 let mut child = GuestCommand::new(&guest) 6103 .args(["--cpus", &format!("boot={}", num_queue_pairs)]) 6104 .args(["--memory", "size=512M"]) 6105 .args(["--kernel", kernel_path.to_str().unwrap()]) 6106 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6107 .default_disks() 6108 .args([ 6109 "--net", 6110 &format!( 6111 "fd=[{},{}],mac={},num_queues={}", 6112 taps[0].as_raw_fd(), 6113 taps[1].as_raw_fd(), 6114 guest.network.guest_mac, 6115 num_queue_pairs * 2 6116 ), 6117 ]) 6118 .capture_output() 6119 .spawn() 6120 .unwrap(); 6121 6122 let r = std::panic::catch_unwind(|| { 6123 guest.wait_vm_boot(None).unwrap(); 6124 6125 assert_eq!( 6126 guest 6127 .ssh_command("ip -o link | wc -l") 6128 .unwrap() 6129 .trim() 6130 .parse::<u32>() 6131 .unwrap_or_default(), 6132 2 6133 ); 6134 6135 guest.reboot_linux(0, None); 6136 6137 assert_eq!( 6138 guest 6139 .ssh_command("ip -o link | wc -l") 6140 .unwrap() 6141 .trim() 6142 .parse::<u32>() 6143 .unwrap_or_default(), 6144 2 6145 ); 6146 }); 6147 6148 let _ = child.kill(); 6149 let output = child.wait_with_output().unwrap(); 6150 6151 handle_child_output(r, &output); 6152 } 6153 6154 // By design, a guest VM won't be able to connect to the host 6155 // machine when using a macvtap network interface (while it can 6156 // communicate externally). As a workaround, this integration 6157 // test creates two macvtap interfaces in 'bridge' mode on the 6158 // same physical net interface, one for the guest and one for 6159 // the host. With additional setup on the IP address and the 6160 // routing table, it enables the communications between the 6161 // guest VM and the host machine. 6162 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6163 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6164 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6165 let guest = Guest::new(Box::new(focal)); 6166 let api_socket = temp_api_path(&guest.tmp_dir); 6167 6168 #[cfg(target_arch = "x86_64")] 6169 let kernel_path = direct_kernel_boot_path(); 6170 #[cfg(target_arch = "aarch64")] 6171 let kernel_path = edk2_path(); 6172 6173 let phy_net = "eth0"; 6174 6175 // Create a macvtap interface for the guest VM to use 6176 assert!(exec_host_command_status(&format!( 6177 "sudo ip link add link {} name {} type macvtap mod bridge", 6178 phy_net, guest_macvtap_name 6179 )) 6180 .success()); 6181 assert!(exec_host_command_status(&format!( 6182 "sudo ip link set {} address {} up", 6183 guest_macvtap_name, guest.network.guest_mac 6184 )) 6185 .success()); 6186 assert!( 6187 exec_host_command_status(&format!("sudo ip link show {}", guest_macvtap_name)) 6188 .success() 6189 ); 6190 6191 let tap_index = 6192 fs::read_to_string(format!("/sys/class/net/{}/ifindex", guest_macvtap_name)).unwrap(); 6193 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6194 6195 assert!( 6196 exec_host_command_status(&format!("sudo chown $UID.$UID {}", tap_device)).success() 6197 ); 6198 6199 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6200 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6201 assert!(tap_fd1 > 0); 6202 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6203 assert!(tap_fd2 > 0); 6204 6205 // Create a macvtap on the same physical net interface for 6206 // the host machine to use 6207 assert!(exec_host_command_status(&format!( 6208 "sudo ip link add link {} name {} type macvtap mod bridge", 6209 phy_net, host_macvtap_name 6210 )) 6211 .success()); 6212 // Use default mask "255.255.255.0" 6213 assert!(exec_host_command_status(&format!( 6214 "sudo ip address add {}/24 dev {}", 6215 guest.network.host_ip, host_macvtap_name 6216 )) 6217 .success()); 6218 assert!(exec_host_command_status(&format!( 6219 "sudo ip link set dev {} up", 6220 host_macvtap_name 6221 )) 6222 .success()); 6223 6224 let mut guest_command = GuestCommand::new(&guest); 6225 guest_command 6226 .args(["--cpus", "boot=2"]) 6227 .args(["--memory", "size=512M"]) 6228 .args(["--kernel", kernel_path.to_str().unwrap()]) 6229 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6230 .default_disks() 6231 .args(["--api-socket", &api_socket]); 6232 6233 let net_params = format!( 6234 "fd=[{},{}],mac={},num_queues=4", 6235 tap_fd1, tap_fd2, guest.network.guest_mac 6236 ); 6237 6238 if !hotplug { 6239 guest_command.args(["--net", &net_params]); 6240 } 6241 6242 let mut child = guest_command.capture_output().spawn().unwrap(); 6243 6244 if hotplug { 6245 // Give some time to the VMM process to listen to the API 6246 // socket. This is the only requirement to avoid the following 6247 // call to ch-remote from failing. 6248 thread::sleep(std::time::Duration::new(10, 0)); 6249 // Hotplug the virtio-net device 6250 let (cmd_success, cmd_output) = 6251 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6252 assert!(cmd_success); 6253 #[cfg(target_arch = "x86_64")] 6254 assert!(String::from_utf8_lossy(&cmd_output) 6255 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6256 #[cfg(target_arch = "aarch64")] 6257 assert!(String::from_utf8_lossy(&cmd_output) 6258 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6259 } 6260 6261 // The functional connectivity provided by the virtio-net device 6262 // gets tested through wait_vm_boot() as it expects to receive a 6263 // HTTP request, and through the SSH command as well. 6264 let r = std::panic::catch_unwind(|| { 6265 guest.wait_vm_boot(None).unwrap(); 6266 6267 assert_eq!( 6268 guest 6269 .ssh_command("ip -o link | wc -l") 6270 .unwrap() 6271 .trim() 6272 .parse::<u32>() 6273 .unwrap_or_default(), 6274 2 6275 ); 6276 }); 6277 6278 let _ = child.kill(); 6279 6280 exec_host_command_status(&format!("sudo ip link del {}", guest_macvtap_name)); 6281 exec_host_command_status(&format!("sudo ip link del {}", host_macvtap_name)); 6282 6283 let output = child.wait_with_output().unwrap(); 6284 6285 handle_child_output(r, &output); 6286 } 6287 6288 #[test] 6289 fn test_macvtap() { 6290 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6291 } 6292 6293 #[test] 6294 fn test_macvtap_hotplug() { 6295 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6296 } 6297 6298 #[test] 6299 #[cfg(not(feature = "mshv"))] 6300 fn test_ovs_dpdk() { 6301 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6302 let guest1 = Guest::new(Box::new(focal1)); 6303 6304 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6305 let guest2 = Guest::new(Box::new(focal2)); 6306 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6307 6308 let (mut child1, mut child2) = 6309 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6310 6311 // Create the snapshot directory 6312 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6313 6314 let r = std::panic::catch_unwind(|| { 6315 // Remove one of the two ports from the OVS bridge 6316 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6317 6318 // Spawn a new netcat listener in the first VM 6319 let guest_ip = guest1.network.guest_ip.clone(); 6320 thread::spawn(move || { 6321 ssh_command_ip( 6322 "nc -l 12345", 6323 &guest_ip, 6324 DEFAULT_SSH_RETRIES, 6325 DEFAULT_SSH_TIMEOUT, 6326 ) 6327 .unwrap(); 6328 }); 6329 6330 // Wait for the server to be listening 6331 thread::sleep(std::time::Duration::new(5, 0)); 6332 6333 // Check the connection fails this time 6334 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6335 6336 // Add the OVS port back 6337 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6338 6339 // And finally check the connection is functional again 6340 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6341 6342 // Pause the VM 6343 assert!(remote_command(&api_socket_source, "pause", None)); 6344 6345 // Take a snapshot from the VM 6346 assert!(remote_command( 6347 &api_socket_source, 6348 "snapshot", 6349 Some(format!("file://{}", snapshot_dir).as_str()), 6350 )); 6351 6352 // Wait to make sure the snapshot is completed 6353 thread::sleep(std::time::Duration::new(10, 0)); 6354 }); 6355 6356 // Shutdown the source VM 6357 let _ = child2.kill(); 6358 let output = child2.wait_with_output().unwrap(); 6359 handle_child_output(r, &output); 6360 6361 // Remove the vhost-user socket file. 6362 Command::new("rm") 6363 .arg("-f") 6364 .arg("/tmp/dpdkvhostclient2") 6365 .output() 6366 .unwrap(); 6367 6368 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6369 // Restore the VM from the snapshot 6370 let mut child2 = GuestCommand::new(&guest2) 6371 .args(["--api-socket", &api_socket_restored]) 6372 .args([ 6373 "--restore", 6374 format!("source_url=file://{}", snapshot_dir).as_str(), 6375 ]) 6376 .capture_output() 6377 .spawn() 6378 .unwrap(); 6379 6380 // Wait for the VM to be restored 6381 thread::sleep(std::time::Duration::new(10, 0)); 6382 6383 let r = std::panic::catch_unwind(|| { 6384 // Resume the VM 6385 assert!(remote_command(&api_socket_restored, "resume", None)); 6386 6387 // Spawn a new netcat listener in the first VM 6388 let guest_ip = guest1.network.guest_ip.clone(); 6389 thread::spawn(move || { 6390 ssh_command_ip( 6391 "nc -l 12345", 6392 &guest_ip, 6393 DEFAULT_SSH_RETRIES, 6394 DEFAULT_SSH_TIMEOUT, 6395 ) 6396 .unwrap(); 6397 }); 6398 6399 // Wait for the server to be listening 6400 thread::sleep(std::time::Duration::new(5, 0)); 6401 6402 // And check the connection is still functional after restore 6403 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6404 }); 6405 6406 let _ = child1.kill(); 6407 let _ = child2.kill(); 6408 6409 let output = child1.wait_with_output().unwrap(); 6410 child2.wait().unwrap(); 6411 6412 cleanup_ovs_dpdk(); 6413 6414 handle_child_output(r, &output); 6415 } 6416 6417 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6418 cleanup_spdk_nvme(); 6419 6420 assert!(exec_host_command_status(&format!( 6421 "mkdir -p {}", 6422 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6423 )) 6424 .success()); 6425 assert!(exec_host_command_status(&format!( 6426 "truncate {} -s 128M", 6427 nvme_dir.join("test-disk.raw").to_str().unwrap() 6428 )) 6429 .success()); 6430 assert!(exec_host_command_status(&format!( 6431 "mkfs.ext4 {}", 6432 nvme_dir.join("test-disk.raw").to_str().unwrap() 6433 )) 6434 .success()); 6435 6436 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6437 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6438 .args(["-i", "0", "-m", "0x1"]) 6439 .spawn() 6440 .unwrap(); 6441 thread::sleep(std::time::Duration::new(2, 0)); 6442 6443 assert!(exec_host_command_status( 6444 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER" 6445 ) 6446 .success()); 6447 assert!(exec_host_command_status(&format!( 6448 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6449 nvme_dir.join("test-disk.raw").to_str().unwrap() 6450 )) 6451 .success()); 6452 assert!(exec_host_command_status( 6453 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6454 ) 6455 .success()); 6456 assert!(exec_host_command_status( 6457 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6458 ) 6459 .success()); 6460 assert!(exec_host_command_status(&format!( 6461 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6462 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6463 )) 6464 .success()); 6465 } 6466 6467 fn cleanup_spdk_nvme() { 6468 exec_host_command_status("pkill -f nvmf_tgt"); 6469 } 6470 6471 #[test] 6472 fn test_vfio_user() { 6473 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6474 let jammy = UbuntuDiskConfig::new(jammy_image); 6475 let guest = Guest::new(Box::new(jammy)); 6476 6477 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6478 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6479 6480 let api_socket = temp_api_path(&guest.tmp_dir); 6481 let mut child = GuestCommand::new(&guest) 6482 .args(["--api-socket", &api_socket]) 6483 .args(["--cpus", "boot=1"]) 6484 .args(["--memory", "size=512M,shared=on,hugepages=on"]) 6485 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6486 .args(["--serial", "tty", "--console", "off"]) 6487 .default_disks() 6488 .default_net() 6489 .capture_output() 6490 .spawn() 6491 .unwrap(); 6492 6493 let r = std::panic::catch_unwind(|| { 6494 guest.wait_vm_boot(None).unwrap(); 6495 6496 // Hotplug the SPDK-NVMe device to the VM 6497 let (cmd_success, cmd_output) = remote_command_w_output( 6498 &api_socket, 6499 "add-user-device", 6500 Some(&format!( 6501 "socket={},id=vfio_user0", 6502 spdk_nvme_dir 6503 .as_path() 6504 .join("nvme-vfio-user/cntrl") 6505 .to_str() 6506 .unwrap(), 6507 )), 6508 ); 6509 assert!(cmd_success); 6510 assert!(String::from_utf8_lossy(&cmd_output) 6511 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6512 6513 thread::sleep(std::time::Duration::new(10, 0)); 6514 6515 // Check both if /dev/nvme exists and if the block size is 128M. 6516 assert_eq!( 6517 guest 6518 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6519 .unwrap() 6520 .trim() 6521 .parse::<u32>() 6522 .unwrap_or_default(), 6523 1 6524 ); 6525 6526 // Check changes persist after reboot 6527 assert_eq!( 6528 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6529 "" 6530 ); 6531 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6532 guest 6533 .ssh_command("echo test123 | sudo tee /mnt/test") 6534 .unwrap(); 6535 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6536 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6537 6538 guest.reboot_linux(0, None); 6539 assert_eq!( 6540 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6541 "" 6542 ); 6543 assert_eq!( 6544 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6545 "test123" 6546 ); 6547 }); 6548 6549 cleanup_spdk_nvme(); 6550 6551 let _ = child.kill(); 6552 let output = child.wait_with_output().unwrap(); 6553 6554 handle_child_output(r, &output); 6555 } 6556 6557 #[test] 6558 #[cfg(target_arch = "x86_64")] 6559 fn test_vdpa_block() { 6560 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6561 if !exec_host_command_status("lsmod | grep vdpa_sim_blk").success() { 6562 return; 6563 } 6564 6565 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6566 let guest = Guest::new(Box::new(focal)); 6567 let api_socket = temp_api_path(&guest.tmp_dir); 6568 6569 let kernel_path = direct_kernel_boot_path(); 6570 6571 let mut child = GuestCommand::new(&guest) 6572 .args(["--cpus", "boot=2"]) 6573 .args(["--memory", "size=512M,hugepages=on"]) 6574 .args(["--kernel", kernel_path.to_str().unwrap()]) 6575 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6576 .default_disks() 6577 .default_net() 6578 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6579 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 6580 .args(["--api-socket", &api_socket]) 6581 .capture_output() 6582 .spawn() 6583 .unwrap(); 6584 6585 let r = std::panic::catch_unwind(|| { 6586 guest.wait_vm_boot(None).unwrap(); 6587 6588 // Check both if /dev/vdc exists and if the block size is 128M. 6589 assert_eq!( 6590 guest 6591 .ssh_command("lsblk | grep vdc | grep -c 128M") 6592 .unwrap() 6593 .trim() 6594 .parse::<u32>() 6595 .unwrap_or_default(), 6596 1 6597 ); 6598 6599 // Check the content of the block device after we wrote to it. 6600 // The vpda-sim-blk should let us read what we previously wrote. 6601 guest 6602 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6603 .unwrap(); 6604 assert_eq!( 6605 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6606 "foobar" 6607 ); 6608 6609 // Hotplug an extra vDPA block device behind the vIOMMU 6610 // Add a new vDPA device to the VM 6611 let (cmd_success, cmd_output) = remote_command_w_output( 6612 &api_socket, 6613 "add-vdpa", 6614 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6615 ); 6616 assert!(cmd_success); 6617 assert!(String::from_utf8_lossy(&cmd_output) 6618 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6619 6620 thread::sleep(std::time::Duration::new(10, 0)); 6621 6622 // Check IOMMU setup 6623 assert!(guest 6624 .does_device_vendor_pair_match("0x1057", "0x1af4") 6625 .unwrap_or_default()); 6626 assert_eq!( 6627 guest 6628 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6629 .unwrap() 6630 .trim(), 6631 "0001:00:01.0" 6632 ); 6633 6634 // Check both if /dev/vdd exists and if the block size is 128M. 6635 assert_eq!( 6636 guest 6637 .ssh_command("lsblk | grep vdd | grep -c 128M") 6638 .unwrap() 6639 .trim() 6640 .parse::<u32>() 6641 .unwrap_or_default(), 6642 1 6643 ); 6644 6645 // Write some content to the block device we've just plugged. 6646 guest 6647 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6648 .unwrap(); 6649 6650 // Check we can read the content back. 6651 assert_eq!( 6652 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6653 "foobar" 6654 ); 6655 6656 // Unplug the device 6657 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6658 assert!(cmd_success); 6659 thread::sleep(std::time::Duration::new(10, 0)); 6660 6661 // Check /dev/vdd doesn't exist anymore 6662 assert_eq!( 6663 guest 6664 .ssh_command("lsblk | grep -c vdd || true") 6665 .unwrap() 6666 .trim() 6667 .parse::<u32>() 6668 .unwrap_or(1), 6669 0 6670 ); 6671 }); 6672 6673 let _ = child.kill(); 6674 let output = child.wait_with_output().unwrap(); 6675 6676 handle_child_output(r, &output); 6677 } 6678 6679 #[test] 6680 #[cfg(target_arch = "x86_64")] 6681 fn test_vdpa_net() { 6682 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6683 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6684 return; 6685 } 6686 6687 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6688 let guest = Guest::new(Box::new(focal)); 6689 6690 let kernel_path = direct_kernel_boot_path(); 6691 6692 let mut child = GuestCommand::new(&guest) 6693 .args(["--cpus", "boot=2"]) 6694 .args(["--memory", "size=512M,hugepages=on"]) 6695 .args(["--kernel", kernel_path.to_str().unwrap()]) 6696 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6697 .default_disks() 6698 .default_net() 6699 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 6700 .capture_output() 6701 .spawn() 6702 .unwrap(); 6703 6704 let r = std::panic::catch_unwind(|| { 6705 guest.wait_vm_boot(None).unwrap(); 6706 6707 // Check we can find network interface related to vDPA device 6708 assert_eq!( 6709 guest 6710 .ssh_command("ip -o link | grep -c ens6") 6711 .unwrap() 6712 .trim() 6713 .parse::<u32>() 6714 .unwrap_or(0), 6715 1 6716 ); 6717 6718 guest 6719 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 6720 .unwrap(); 6721 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 6722 6723 // Check there is no packet yet on both TX/RX of the network interface 6724 assert_eq!( 6725 guest 6726 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 6727 .unwrap() 6728 .trim() 6729 .parse::<u32>() 6730 .unwrap_or(0), 6731 2 6732 ); 6733 6734 // Send 6 packets with ping command 6735 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 6736 6737 // Check we can find 6 packets on both TX/RX of the network interface 6738 assert_eq!( 6739 guest 6740 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 6741 .unwrap() 6742 .trim() 6743 .parse::<u32>() 6744 .unwrap_or(0), 6745 2 6746 ); 6747 6748 // No need to check for hotplug as we already tested it through 6749 // test_vdpa_block() 6750 }); 6751 6752 let _ = child.kill(); 6753 let output = child.wait_with_output().unwrap(); 6754 6755 handle_child_output(r, &output); 6756 } 6757 } 6758 6759 mod common_sequential { 6760 use crate::*; 6761 6762 #[test] 6763 fn test_memory_mergeable_on() { 6764 test_memory_mergeable(true) 6765 } 6766 } 6767 6768 mod windows { 6769 use crate::*; 6770 use once_cell::sync::Lazy; 6771 6772 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 6773 6774 struct WindowsGuest { 6775 guest: Guest, 6776 auth: PasswordAuth, 6777 } 6778 6779 trait FsType { 6780 const FS_FAT: u8; 6781 const FS_NTFS: u8; 6782 } 6783 impl FsType for WindowsGuest { 6784 const FS_FAT: u8 = 0; 6785 const FS_NTFS: u8 = 1; 6786 } 6787 6788 impl WindowsGuest { 6789 fn new() -> Self { 6790 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 6791 let guest = Guest::new(Box::new(disk)); 6792 let auth = PasswordAuth { 6793 username: String::from("administrator"), 6794 password: String::from("Admin123"), 6795 }; 6796 6797 WindowsGuest { guest, auth } 6798 } 6799 6800 fn guest(&self) -> &Guest { 6801 &self.guest 6802 } 6803 6804 fn ssh_cmd(&self, cmd: &str) -> String { 6805 ssh_command_ip_with_auth( 6806 cmd, 6807 &self.auth, 6808 &self.guest.network.guest_ip, 6809 DEFAULT_SSH_RETRIES, 6810 DEFAULT_SSH_TIMEOUT, 6811 ) 6812 .unwrap() 6813 } 6814 6815 fn cpu_count(&self) -> u8 { 6816 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 6817 .trim() 6818 .parse::<u8>() 6819 .unwrap_or(0) 6820 } 6821 6822 fn ram_size(&self) -> usize { 6823 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 6824 .trim() 6825 .parse::<usize>() 6826 .unwrap_or(0) 6827 } 6828 6829 fn netdev_count(&self) -> u8 { 6830 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6831 .trim() 6832 .parse::<u8>() 6833 .unwrap_or(0) 6834 } 6835 6836 fn disk_count(&self) -> u8 { 6837 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6838 .trim() 6839 .parse::<u8>() 6840 .unwrap_or(0) 6841 } 6842 6843 fn reboot(&self) { 6844 let _ = self.ssh_cmd("shutdown /r /t 0"); 6845 } 6846 6847 fn shutdown(&self) { 6848 let _ = self.ssh_cmd("shutdown /s /t 0"); 6849 } 6850 6851 fn run_dnsmasq(&self) -> std::process::Child { 6852 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 6853 let dhcp_host = format!( 6854 "--dhcp-host={},{}", 6855 self.guest.network.guest_mac, self.guest.network.guest_ip 6856 ); 6857 let dhcp_range = format!( 6858 "--dhcp-range=eth,{},{}", 6859 self.guest.network.guest_ip, self.guest.network.guest_ip 6860 ); 6861 6862 Command::new("dnsmasq") 6863 .arg("--no-daemon") 6864 .arg("--log-queries") 6865 .arg(listen_address.as_str()) 6866 .arg("--except-interface=lo") 6867 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 6868 .arg("--conf-file=/dev/null") 6869 .arg(dhcp_host.as_str()) 6870 .arg(dhcp_range.as_str()) 6871 .spawn() 6872 .unwrap() 6873 } 6874 6875 // TODO Cleanup image file explicitly after test, if there's some space issues. 6876 fn disk_new(&self, fs: u8, sz: usize) -> String { 6877 let mut guard = NEXT_DISK_ID.lock().unwrap(); 6878 let id = *guard; 6879 *guard = id + 1; 6880 6881 let img = PathBuf::from(format!("/tmp/test-hotplug-{}.raw", id)); 6882 let _ = fs::remove_file(&img); 6883 6884 // Create an image file 6885 let out = Command::new("qemu-img") 6886 .args([ 6887 "create", 6888 "-f", 6889 "raw", 6890 img.to_str().unwrap(), 6891 format!("{}m", sz).as_str(), 6892 ]) 6893 .output() 6894 .expect("qemu-img command failed") 6895 .stdout; 6896 println!("{:?}", out); 6897 6898 // Associate image to a loop device 6899 let out = Command::new("losetup") 6900 .args(["--show", "-f", img.to_str().unwrap()]) 6901 .output() 6902 .expect("failed to create loop device") 6903 .stdout; 6904 let _tmp = String::from_utf8_lossy(&out); 6905 let loop_dev = _tmp.trim(); 6906 println!("{:?}", out); 6907 6908 // Create a partition table 6909 // echo 'type=7' | sudo sfdisk "${LOOP}" 6910 let mut child = Command::new("sfdisk") 6911 .args([loop_dev]) 6912 .stdin(Stdio::piped()) 6913 .spawn() 6914 .unwrap(); 6915 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 6916 stdin 6917 .write_all("type=7".as_bytes()) 6918 .expect("failed to write stdin"); 6919 let out = child.wait_with_output().expect("sfdisk failed").stdout; 6920 println!("{:?}", out); 6921 6922 // Disengage the loop device 6923 let out = Command::new("losetup") 6924 .args(["-d", loop_dev]) 6925 .output() 6926 .expect("loop device not found") 6927 .stdout; 6928 println!("{:?}", out); 6929 6930 // Re-associate loop device pointing to the partition only 6931 let out = Command::new("losetup") 6932 .args([ 6933 "--show", 6934 "--offset", 6935 (512 * 2048).to_string().as_str(), 6936 "-f", 6937 img.to_str().unwrap(), 6938 ]) 6939 .output() 6940 .expect("failed to create loop device") 6941 .stdout; 6942 let _tmp = String::from_utf8_lossy(&out); 6943 let loop_dev = _tmp.trim(); 6944 println!("{:?}", out); 6945 6946 // Create filesystem. 6947 let fs_cmd = match fs { 6948 WindowsGuest::FS_FAT => "mkfs.msdos", 6949 WindowsGuest::FS_NTFS => "mkfs.ntfs", 6950 _ => panic!("Unknown filesystem type '{}'", fs), 6951 }; 6952 let out = Command::new(fs_cmd) 6953 .args([&loop_dev]) 6954 .output() 6955 .unwrap_or_else(|_| panic!("{} failed", fs_cmd)) 6956 .stdout; 6957 println!("{:?}", out); 6958 6959 // Disengage the loop device 6960 let out = Command::new("losetup") 6961 .args(["-d", loop_dev]) 6962 .output() 6963 .unwrap_or_else(|_| panic!("loop device '{}' not found", loop_dev)) 6964 .stdout; 6965 println!("{:?}", out); 6966 6967 img.to_str().unwrap().to_string() 6968 } 6969 6970 fn disks_set_rw(&self) { 6971 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 6972 } 6973 6974 fn disks_online(&self) { 6975 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 6976 } 6977 6978 fn disk_file_put(&self, fname: &str, data: &str) { 6979 let _ = self.ssh_cmd(&format!( 6980 "powershell -Command \"'{}' | Set-Content -Path {}\"", 6981 data, fname 6982 )); 6983 } 6984 6985 fn disk_file_read(&self, fname: &str) -> String { 6986 self.ssh_cmd(&format!( 6987 "powershell -Command \"Get-Content -Path {}\"", 6988 fname 6989 )) 6990 } 6991 6992 fn wait_for_boot(&self) -> bool { 6993 let cmd = "dir /b c:\\ | find \"Windows\""; 6994 let tmo_max = 180; 6995 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 6996 // interval must be small. 6997 let tmo_int = 2; 6998 let out = ssh_command_ip_with_auth( 6999 cmd, 7000 &self.auth, 7001 &self.guest.network.guest_ip, 7002 { 7003 let mut ret = 1; 7004 let mut tmo_acc = 0; 7005 loop { 7006 tmo_acc += tmo_int * ret; 7007 if tmo_acc >= tmo_max { 7008 break; 7009 } 7010 ret += 1; 7011 } 7012 ret 7013 }, 7014 tmo_int, 7015 ) 7016 .unwrap(); 7017 7018 if "Windows" == out.trim() { 7019 return true; 7020 } 7021 7022 false 7023 } 7024 } 7025 7026 fn vcpu_threads_count(pid: u32) -> u8 { 7027 // ps -T -p 12345 | grep vcpu | wc -l 7028 let out = Command::new("ps") 7029 .args(["-T", "-p", format!("{}", pid).as_str()]) 7030 .output() 7031 .expect("ps command failed") 7032 .stdout; 7033 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 7034 } 7035 7036 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 7037 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 7038 let out = Command::new("ps") 7039 .args(["-T", "-p", format!("{}", pid).as_str()]) 7040 .output() 7041 .expect("ps command failed") 7042 .stdout; 7043 let mut n = 0; 7044 String::from_utf8_lossy(&out) 7045 .split_whitespace() 7046 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 7047 n 7048 } 7049 7050 fn disk_ctrl_threads_count(pid: u32) -> u8 { 7051 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 7052 let out = Command::new("ps") 7053 .args(["-T", "-p", format!("{}", pid).as_str()]) 7054 .output() 7055 .expect("ps command failed") 7056 .stdout; 7057 let mut n = 0; 7058 String::from_utf8_lossy(&out) 7059 .split_whitespace() 7060 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 7061 n 7062 } 7063 7064 #[test] 7065 fn test_windows_guest() { 7066 let windows_guest = WindowsGuest::new(); 7067 7068 let mut child = GuestCommand::new(windows_guest.guest()) 7069 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7070 .args(["--memory", "size=4G"]) 7071 .args(["--kernel", edk2_path().to_str().unwrap()]) 7072 .args(["--serial", "tty"]) 7073 .args(["--console", "off"]) 7074 .default_disks() 7075 .default_net() 7076 .capture_output() 7077 .spawn() 7078 .unwrap(); 7079 7080 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7081 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7082 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7083 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7084 7085 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7086 7087 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7088 7089 let r = std::panic::catch_unwind(|| { 7090 // Wait to make sure Windows boots up 7091 assert!(windows_guest.wait_for_boot()); 7092 7093 windows_guest.shutdown(); 7094 }); 7095 7096 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7097 let _ = child.kill(); 7098 let output = child.wait_with_output().unwrap(); 7099 7100 let _ = child_dnsmasq.kill(); 7101 let _ = child_dnsmasq.wait(); 7102 7103 handle_child_output(r, &output); 7104 } 7105 7106 #[test] 7107 fn test_windows_guest_multiple_queues() { 7108 let windows_guest = WindowsGuest::new(); 7109 7110 let mut ovmf_path = dirs::home_dir().unwrap(); 7111 ovmf_path.push("workloads"); 7112 ovmf_path.push(OVMF_NAME); 7113 7114 let mut child = GuestCommand::new(windows_guest.guest()) 7115 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 7116 .args(["--memory", "size=4G"]) 7117 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7118 .args(["--serial", "tty"]) 7119 .args(["--console", "off"]) 7120 .args([ 7121 "--disk", 7122 format!( 7123 "path={},num_queues=4", 7124 windows_guest 7125 .guest() 7126 .disk_config 7127 .disk(DiskType::OperatingSystem) 7128 .unwrap() 7129 ) 7130 .as_str(), 7131 ]) 7132 .args([ 7133 "--net", 7134 format!( 7135 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 7136 windows_guest.guest().network.guest_mac, 7137 windows_guest.guest().network.host_ip 7138 ) 7139 .as_str(), 7140 ]) 7141 .capture_output() 7142 .spawn() 7143 .unwrap(); 7144 7145 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7146 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7147 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7148 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7149 7150 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7151 7152 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7153 7154 let r = std::panic::catch_unwind(|| { 7155 // Wait to make sure Windows boots up 7156 assert!(windows_guest.wait_for_boot()); 7157 7158 windows_guest.shutdown(); 7159 }); 7160 7161 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7162 let _ = child.kill(); 7163 let output = child.wait_with_output().unwrap(); 7164 7165 let _ = child_dnsmasq.kill(); 7166 let _ = child_dnsmasq.wait(); 7167 7168 handle_child_output(r, &output); 7169 } 7170 7171 #[test] 7172 #[cfg(not(feature = "mshv"))] 7173 #[ignore = "See #4327"] 7174 fn test_windows_guest_snapshot_restore() { 7175 let windows_guest = WindowsGuest::new(); 7176 7177 let mut ovmf_path = dirs::home_dir().unwrap(); 7178 ovmf_path.push("workloads"); 7179 ovmf_path.push(OVMF_NAME); 7180 7181 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7182 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 7183 7184 let mut child = GuestCommand::new(windows_guest.guest()) 7185 .args(["--api-socket", &api_socket_source]) 7186 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7187 .args(["--memory", "size=4G"]) 7188 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7189 .args(["--serial", "tty"]) 7190 .args(["--console", "off"]) 7191 .default_disks() 7192 .default_net() 7193 .capture_output() 7194 .spawn() 7195 .unwrap(); 7196 7197 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7198 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7199 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7200 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7201 7202 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7203 7204 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7205 7206 // Wait to make sure Windows boots up 7207 assert!(windows_guest.wait_for_boot()); 7208 7209 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 7210 7211 // Pause the VM 7212 assert!(remote_command(&api_socket_source, "pause", None)); 7213 7214 // Take a snapshot from the VM 7215 assert!(remote_command( 7216 &api_socket_source, 7217 "snapshot", 7218 Some(format!("file://{}", snapshot_dir).as_str()), 7219 )); 7220 7221 // Wait to make sure the snapshot is completed 7222 thread::sleep(std::time::Duration::new(30, 0)); 7223 7224 let _ = child.kill(); 7225 child.wait().unwrap(); 7226 7227 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 7228 7229 // Restore the VM from the snapshot 7230 let mut child = GuestCommand::new(windows_guest.guest()) 7231 .args(["--api-socket", &api_socket_restored]) 7232 .args([ 7233 "--restore", 7234 format!("source_url=file://{}", snapshot_dir).as_str(), 7235 ]) 7236 .capture_output() 7237 .spawn() 7238 .unwrap(); 7239 7240 // Wait for the VM to be restored 7241 thread::sleep(std::time::Duration::new(20, 0)); 7242 7243 let r = std::panic::catch_unwind(|| { 7244 // Resume the VM 7245 assert!(remote_command(&api_socket_restored, "resume", None)); 7246 7247 windows_guest.shutdown(); 7248 }); 7249 7250 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7251 let _ = child.kill(); 7252 let output = child.wait_with_output().unwrap(); 7253 7254 let _ = child_dnsmasq.kill(); 7255 let _ = child_dnsmasq.wait(); 7256 7257 handle_child_output(r, &output); 7258 } 7259 7260 #[test] 7261 #[cfg(not(feature = "mshv"))] 7262 #[cfg(not(target_arch = "aarch64"))] 7263 fn test_windows_guest_cpu_hotplug() { 7264 let windows_guest = WindowsGuest::new(); 7265 7266 let mut ovmf_path = dirs::home_dir().unwrap(); 7267 ovmf_path.push("workloads"); 7268 ovmf_path.push(OVMF_NAME); 7269 7270 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7271 let api_socket = temp_api_path(&tmp_dir); 7272 7273 let mut child = GuestCommand::new(windows_guest.guest()) 7274 .args(["--api-socket", &api_socket]) 7275 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 7276 .args(["--memory", "size=4G"]) 7277 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7278 .args(["--serial", "tty"]) 7279 .args(["--console", "off"]) 7280 .default_disks() 7281 .default_net() 7282 .capture_output() 7283 .spawn() 7284 .unwrap(); 7285 7286 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7287 7288 let r = std::panic::catch_unwind(|| { 7289 // Wait to make sure Windows boots up 7290 assert!(windows_guest.wait_for_boot()); 7291 7292 let vcpu_num = 2; 7293 // Check the initial number of CPUs the guest sees 7294 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7295 // Check the initial number of vcpu threads in the CH process 7296 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7297 7298 let vcpu_num = 6; 7299 // Hotplug some CPUs 7300 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7301 // Wait to make sure CPUs are added 7302 thread::sleep(std::time::Duration::new(10, 0)); 7303 // Check the guest sees the correct number 7304 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7305 // Check the CH process has the correct number of vcpu threads 7306 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7307 7308 let vcpu_num = 4; 7309 // Remove some CPUs. Note that Windows doesn't support hot-remove. 7310 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7311 // Wait to make sure CPUs are removed 7312 thread::sleep(std::time::Duration::new(10, 0)); 7313 // Reboot to let Windows catch up 7314 windows_guest.reboot(); 7315 // Wait to make sure Windows completely rebooted 7316 thread::sleep(std::time::Duration::new(60, 0)); 7317 // Check the guest sees the correct number 7318 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7319 // Check the CH process has the correct number of vcpu threads 7320 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7321 7322 windows_guest.shutdown(); 7323 }); 7324 7325 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7326 let _ = child.kill(); 7327 let output = child.wait_with_output().unwrap(); 7328 7329 let _ = child_dnsmasq.kill(); 7330 let _ = child_dnsmasq.wait(); 7331 7332 handle_child_output(r, &output); 7333 } 7334 7335 #[test] 7336 #[cfg(not(feature = "mshv"))] 7337 #[cfg(not(target_arch = "aarch64"))] 7338 fn test_windows_guest_ram_hotplug() { 7339 let windows_guest = WindowsGuest::new(); 7340 7341 let mut ovmf_path = dirs::home_dir().unwrap(); 7342 ovmf_path.push("workloads"); 7343 ovmf_path.push(OVMF_NAME); 7344 7345 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7346 let api_socket = temp_api_path(&tmp_dir); 7347 7348 let mut child = GuestCommand::new(windows_guest.guest()) 7349 .args(["--api-socket", &api_socket]) 7350 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7351 .args(["--memory", "size=2G,hotplug_size=5G"]) 7352 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7353 .args(["--serial", "tty"]) 7354 .args(["--console", "off"]) 7355 .default_disks() 7356 .default_net() 7357 .capture_output() 7358 .spawn() 7359 .unwrap(); 7360 7361 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7362 7363 let r = std::panic::catch_unwind(|| { 7364 // Wait to make sure Windows boots up 7365 assert!(windows_guest.wait_for_boot()); 7366 7367 let ram_size = 2 * 1024 * 1024 * 1024; 7368 // Check the initial number of RAM the guest sees 7369 let current_ram_size = windows_guest.ram_size(); 7370 // This size seems to be reserved by the system and thus the 7371 // reported amount differs by this constant value. 7372 let reserved_ram_size = ram_size - current_ram_size; 7373 // Verify that there's not more than 4mb constant diff wasted 7374 // by the reserved ram. 7375 assert!(reserved_ram_size < 4 * 1024 * 1024); 7376 7377 let ram_size = 4 * 1024 * 1024 * 1024; 7378 // Hotplug some RAM 7379 resize_command(&api_socket, None, Some(ram_size), None, None); 7380 // Wait to make sure RAM has been added 7381 thread::sleep(std::time::Duration::new(10, 0)); 7382 // Check the guest sees the correct number 7383 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7384 7385 let ram_size = 3 * 1024 * 1024 * 1024; 7386 // Unplug some RAM. Note that hot-remove most likely won't work. 7387 resize_command(&api_socket, None, Some(ram_size), None, None); 7388 // Wait to make sure RAM has been added 7389 thread::sleep(std::time::Duration::new(10, 0)); 7390 // Reboot to let Windows catch up 7391 windows_guest.reboot(); 7392 // Wait to make sure guest completely rebooted 7393 thread::sleep(std::time::Duration::new(60, 0)); 7394 // Check the guest sees the correct number 7395 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7396 7397 windows_guest.shutdown(); 7398 }); 7399 7400 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7401 let _ = child.kill(); 7402 let output = child.wait_with_output().unwrap(); 7403 7404 let _ = child_dnsmasq.kill(); 7405 let _ = child_dnsmasq.wait(); 7406 7407 handle_child_output(r, &output); 7408 } 7409 7410 #[test] 7411 #[cfg(not(feature = "mshv"))] 7412 fn test_windows_guest_netdev_hotplug() { 7413 let windows_guest = WindowsGuest::new(); 7414 7415 let mut ovmf_path = dirs::home_dir().unwrap(); 7416 ovmf_path.push("workloads"); 7417 ovmf_path.push(OVMF_NAME); 7418 7419 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7420 let api_socket = temp_api_path(&tmp_dir); 7421 7422 let mut child = GuestCommand::new(windows_guest.guest()) 7423 .args(["--api-socket", &api_socket]) 7424 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7425 .args(["--memory", "size=4G"]) 7426 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7427 .args(["--serial", "tty"]) 7428 .args(["--console", "off"]) 7429 .default_disks() 7430 .default_net() 7431 .capture_output() 7432 .spawn() 7433 .unwrap(); 7434 7435 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7436 7437 let r = std::panic::catch_unwind(|| { 7438 // Wait to make sure Windows boots up 7439 assert!(windows_guest.wait_for_boot()); 7440 7441 // Initially present network device 7442 let netdev_num = 1; 7443 assert_eq!(windows_guest.netdev_count(), netdev_num); 7444 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7445 7446 // Hotplug network device 7447 let (cmd_success, cmd_output) = remote_command_w_output( 7448 &api_socket, 7449 "add-net", 7450 Some(windows_guest.guest().default_net_string().as_str()), 7451 ); 7452 assert!(cmd_success); 7453 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 7454 thread::sleep(std::time::Duration::new(5, 0)); 7455 // Verify the device is on the system 7456 let netdev_num = 2; 7457 assert_eq!(windows_guest.netdev_count(), netdev_num); 7458 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7459 7460 // Remove network device 7461 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 7462 assert!(cmd_success); 7463 thread::sleep(std::time::Duration::new(5, 0)); 7464 // Verify the device has been removed 7465 let netdev_num = 1; 7466 assert_eq!(windows_guest.netdev_count(), netdev_num); 7467 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7468 7469 windows_guest.shutdown(); 7470 }); 7471 7472 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7473 let _ = child.kill(); 7474 let output = child.wait_with_output().unwrap(); 7475 7476 let _ = child_dnsmasq.kill(); 7477 let _ = child_dnsmasq.wait(); 7478 7479 handle_child_output(r, &output); 7480 } 7481 7482 #[test] 7483 #[cfg(not(feature = "mshv"))] 7484 #[cfg(not(target_arch = "aarch64"))] 7485 fn test_windows_guest_disk_hotplug() { 7486 let windows_guest = WindowsGuest::new(); 7487 7488 let mut ovmf_path = dirs::home_dir().unwrap(); 7489 ovmf_path.push("workloads"); 7490 ovmf_path.push(OVMF_NAME); 7491 7492 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7493 let api_socket = temp_api_path(&tmp_dir); 7494 7495 let mut child = GuestCommand::new(windows_guest.guest()) 7496 .args(["--api-socket", &api_socket]) 7497 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7498 .args(["--memory", "size=4G"]) 7499 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7500 .args(["--serial", "tty"]) 7501 .args(["--console", "off"]) 7502 .default_disks() 7503 .default_net() 7504 .capture_output() 7505 .spawn() 7506 .unwrap(); 7507 7508 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7509 7510 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 7511 7512 let r = std::panic::catch_unwind(|| { 7513 // Wait to make sure Windows boots up 7514 assert!(windows_guest.wait_for_boot()); 7515 7516 // Initially present disk device 7517 let disk_num = 1; 7518 assert_eq!(windows_guest.disk_count(), disk_num); 7519 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7520 7521 // Hotplug disk device 7522 let (cmd_success, cmd_output) = remote_command_w_output( 7523 &api_socket, 7524 "add-disk", 7525 Some(format!("path={},readonly=off", disk).as_str()), 7526 ); 7527 assert!(cmd_success); 7528 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 7529 thread::sleep(std::time::Duration::new(5, 0)); 7530 // Online disk device 7531 windows_guest.disks_set_rw(); 7532 windows_guest.disks_online(); 7533 // Verify the device is on the system 7534 let disk_num = 2; 7535 assert_eq!(windows_guest.disk_count(), disk_num); 7536 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7537 7538 let data = "hello"; 7539 let fname = "d:\\world"; 7540 windows_guest.disk_file_put(fname, data); 7541 7542 // Unmount disk device 7543 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 7544 assert!(cmd_success); 7545 thread::sleep(std::time::Duration::new(5, 0)); 7546 // Verify the device has been removed 7547 let disk_num = 1; 7548 assert_eq!(windows_guest.disk_count(), disk_num); 7549 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7550 7551 // Remount and check the file exists with the expected contents 7552 let (cmd_success, _cmd_output) = remote_command_w_output( 7553 &api_socket, 7554 "add-disk", 7555 Some(format!("path={},readonly=off", disk).as_str()), 7556 ); 7557 assert!(cmd_success); 7558 thread::sleep(std::time::Duration::new(5, 0)); 7559 let out = windows_guest.disk_file_read(fname); 7560 assert_eq!(data, out.trim()); 7561 7562 // Intentionally no unmount, it'll happen at shutdown. 7563 7564 windows_guest.shutdown(); 7565 }); 7566 7567 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7568 let _ = child.kill(); 7569 let output = child.wait_with_output().unwrap(); 7570 7571 let _ = child_dnsmasq.kill(); 7572 let _ = child_dnsmasq.wait(); 7573 7574 handle_child_output(r, &output); 7575 } 7576 7577 #[test] 7578 #[cfg(not(feature = "mshv"))] 7579 #[cfg(not(target_arch = "aarch64"))] 7580 fn test_windows_guest_disk_hotplug_multi() { 7581 let windows_guest = WindowsGuest::new(); 7582 7583 let mut ovmf_path = dirs::home_dir().unwrap(); 7584 ovmf_path.push("workloads"); 7585 ovmf_path.push(OVMF_NAME); 7586 7587 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7588 let api_socket = temp_api_path(&tmp_dir); 7589 7590 let mut child = GuestCommand::new(windows_guest.guest()) 7591 .args(["--api-socket", &api_socket]) 7592 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7593 .args(["--memory", "size=2G"]) 7594 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7595 .args(["--serial", "tty"]) 7596 .args(["--console", "off"]) 7597 .default_disks() 7598 .default_net() 7599 .capture_output() 7600 .spawn() 7601 .unwrap(); 7602 7603 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7604 7605 // Predefined data to used at various test stages 7606 let disk_test_data: [[String; 4]; 2] = [ 7607 [ 7608 "_disk2".to_string(), 7609 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 7610 "d:\\world".to_string(), 7611 "hello".to_string(), 7612 ], 7613 [ 7614 "_disk3".to_string(), 7615 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 7616 "e:\\hello".to_string(), 7617 "world".to_string(), 7618 ], 7619 ]; 7620 7621 let r = std::panic::catch_unwind(|| { 7622 // Wait to make sure Windows boots up 7623 assert!(windows_guest.wait_for_boot()); 7624 7625 // Initially present disk device 7626 let disk_num = 1; 7627 assert_eq!(windows_guest.disk_count(), disk_num); 7628 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7629 7630 for it in &disk_test_data { 7631 let disk_id = it[0].as_str(); 7632 let disk = it[1].as_str(); 7633 // Hotplug disk device 7634 let (cmd_success, cmd_output) = remote_command_w_output( 7635 &api_socket, 7636 "add-disk", 7637 Some(format!("path={},readonly=off", disk).as_str()), 7638 ); 7639 assert!(cmd_success); 7640 assert!(String::from_utf8_lossy(&cmd_output) 7641 .contains(format!("\"id\":\"{}\"", disk_id).as_str())); 7642 thread::sleep(std::time::Duration::new(5, 0)); 7643 // Online disk devices 7644 windows_guest.disks_set_rw(); 7645 windows_guest.disks_online(); 7646 } 7647 // Verify the devices are on the system 7648 let disk_num = (disk_test_data.len() + 1) as u8; 7649 assert_eq!(windows_guest.disk_count(), disk_num); 7650 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7651 7652 // Put test data 7653 for it in &disk_test_data { 7654 let fname = it[2].as_str(); 7655 let data = it[3].as_str(); 7656 windows_guest.disk_file_put(fname, data); 7657 } 7658 7659 // Unmount disk devices 7660 for it in &disk_test_data { 7661 let disk_id = it[0].as_str(); 7662 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 7663 assert!(cmd_success); 7664 thread::sleep(std::time::Duration::new(5, 0)); 7665 } 7666 7667 // Verify the devices have been removed 7668 let disk_num = 1; 7669 assert_eq!(windows_guest.disk_count(), disk_num); 7670 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7671 7672 // Remount 7673 for it in &disk_test_data { 7674 let disk = it[1].as_str(); 7675 let (cmd_success, _cmd_output) = remote_command_w_output( 7676 &api_socket, 7677 "add-disk", 7678 Some(format!("path={},readonly=off", disk).as_str()), 7679 ); 7680 assert!(cmd_success); 7681 thread::sleep(std::time::Duration::new(5, 0)); 7682 } 7683 7684 // Check the files exists with the expected contents 7685 for it in &disk_test_data { 7686 let fname = it[2].as_str(); 7687 let data = it[3].as_str(); 7688 let out = windows_guest.disk_file_read(fname); 7689 assert_eq!(data, out.trim()); 7690 } 7691 7692 // Intentionally no unmount, it'll happen at shutdown. 7693 7694 windows_guest.shutdown(); 7695 }); 7696 7697 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7698 let _ = child.kill(); 7699 let output = child.wait_with_output().unwrap(); 7700 7701 let _ = child_dnsmasq.kill(); 7702 let _ = child_dnsmasq.wait(); 7703 7704 handle_child_output(r, &output); 7705 } 7706 7707 #[test] 7708 #[cfg(not(feature = "mshv"))] 7709 #[cfg(not(target_arch = "aarch64"))] 7710 fn test_windows_guest_netdev_multi() { 7711 let windows_guest = WindowsGuest::new(); 7712 7713 let mut ovmf_path = dirs::home_dir().unwrap(); 7714 ovmf_path.push("workloads"); 7715 ovmf_path.push(OVMF_NAME); 7716 7717 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7718 let api_socket = temp_api_path(&tmp_dir); 7719 7720 let mut child = GuestCommand::new(windows_guest.guest()) 7721 .args(["--api-socket", &api_socket]) 7722 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7723 .args(["--memory", "size=4G"]) 7724 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7725 .args(["--serial", "tty"]) 7726 .args(["--console", "off"]) 7727 .default_disks() 7728 // The multi net dev config is borrowed from test_multiple_network_interfaces 7729 .args([ 7730 "--net", 7731 windows_guest.guest().default_net_string().as_str(), 7732 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 7733 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 7734 ]) 7735 .capture_output() 7736 .spawn() 7737 .unwrap(); 7738 7739 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7740 7741 let r = std::panic::catch_unwind(|| { 7742 // Wait to make sure Windows boots up 7743 assert!(windows_guest.wait_for_boot()); 7744 7745 let netdev_num = 3; 7746 assert_eq!(windows_guest.netdev_count(), netdev_num); 7747 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7748 7749 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 7750 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 7751 7752 windows_guest.shutdown(); 7753 }); 7754 7755 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7756 let _ = child.kill(); 7757 let output = child.wait_with_output().unwrap(); 7758 7759 let _ = child_dnsmasq.kill(); 7760 let _ = child_dnsmasq.wait(); 7761 7762 handle_child_output(r, &output); 7763 } 7764 } 7765 7766 #[cfg(target_arch = "x86_64")] 7767 mod sgx { 7768 use crate::*; 7769 7770 #[test] 7771 fn test_sgx() { 7772 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 7773 let jammy = UbuntuDiskConfig::new(jammy_image); 7774 let guest = Guest::new(Box::new(jammy)); 7775 7776 let mut child = GuestCommand::new(&guest) 7777 .args(["--cpus", "boot=1"]) 7778 .args(["--memory", "size=512M"]) 7779 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7780 .default_disks() 7781 .default_net() 7782 .args(["--sgx-epc", "id=epc0,size=64M"]) 7783 .capture_output() 7784 .spawn() 7785 .unwrap(); 7786 7787 let r = std::panic::catch_unwind(|| { 7788 guest.wait_vm_boot(None).unwrap(); 7789 7790 // Check if SGX is correctly detected in the guest. 7791 guest.check_sgx_support().unwrap(); 7792 7793 // Validate the SGX EPC section is 64MiB. 7794 assert_eq!( 7795 guest 7796 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 7797 .unwrap() 7798 .trim(), 7799 "0x0000000004000000" 7800 ); 7801 }); 7802 7803 let _ = child.kill(); 7804 let output = child.wait_with_output().unwrap(); 7805 7806 handle_child_output(r, &output); 7807 } 7808 } 7809 7810 #[cfg(target_arch = "x86_64")] 7811 mod vfio { 7812 use crate::*; 7813 7814 #[test] 7815 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 7816 // backed networking interfaces, bound through a simple bridge on the host. 7817 // So if the nested cloud-hypervisor succeeds in getting a directly 7818 // assigned interface from its cloud-hypervisor host, we should be able to 7819 // ssh into it, and verify that it's running with the right kernel command 7820 // line (We tag the command line from cloud-hypervisor for that purpose). 7821 // The third device is added to validate that hotplug works correctly since 7822 // it is being added to the L2 VM through hotplugging mechanism. 7823 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 7824 // vfio device support 7825 fn test_vfio() { 7826 setup_vfio_network_interfaces(); 7827 7828 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7829 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 7830 7831 let mut workload_path = dirs::home_dir().unwrap(); 7832 workload_path.push("workloads"); 7833 7834 let kernel_path = direct_kernel_boot_path(); 7835 7836 let mut vfio_path = workload_path.clone(); 7837 vfio_path.push("vfio"); 7838 7839 let mut cloud_init_vfio_base_path = vfio_path.clone(); 7840 cloud_init_vfio_base_path.push("cloudinit.img"); 7841 7842 // We copy our cloudinit into the vfio mount point, for the nested 7843 // cloud-hypervisor guest to use. 7844 rate_limited_copy( 7845 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 7846 &cloud_init_vfio_base_path, 7847 ) 7848 .expect("copying of cloud-init disk failed"); 7849 7850 let mut vfio_disk_path = workload_path.clone(); 7851 vfio_disk_path.push("vfio.img"); 7852 7853 // Create the vfio disk image 7854 let output = Command::new("mkfs.ext4") 7855 .arg("-d") 7856 .arg(vfio_path.to_str().unwrap()) 7857 .arg(vfio_disk_path.to_str().unwrap()) 7858 .arg("2g") 7859 .output() 7860 .unwrap(); 7861 if !output.status.success() { 7862 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 7863 panic!("mkfs.ext4 command generated an error"); 7864 } 7865 7866 let mut blk_file_path = workload_path; 7867 blk_file_path.push("blk.img"); 7868 7869 let vfio_tap0 = "vfio-tap0"; 7870 let vfio_tap1 = "vfio-tap1"; 7871 let vfio_tap2 = "vfio-tap2"; 7872 let vfio_tap3 = "vfio-tap3"; 7873 7874 let mut child = GuestCommand::new(&guest) 7875 .args(["--cpus", "boot=4"]) 7876 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 7877 .args(["--kernel", kernel_path.to_str().unwrap()]) 7878 .args([ 7879 "--disk", 7880 format!( 7881 "path={}", 7882 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7883 ) 7884 .as_str(), 7885 format!( 7886 "path={}", 7887 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7888 ) 7889 .as_str(), 7890 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 7891 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 7892 ]) 7893 .args([ 7894 "--cmdline", 7895 format!( 7896 "{} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts", 7897 DIRECT_KERNEL_BOOT_CMDLINE 7898 ) 7899 .as_str(), 7900 ]) 7901 .args([ 7902 "--net", 7903 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 7904 format!( 7905 "tap={},mac={},iommu=on", 7906 vfio_tap1, guest.network.l2_guest_mac1 7907 ) 7908 .as_str(), 7909 format!( 7910 "tap={},mac={},iommu=on", 7911 vfio_tap2, guest.network.l2_guest_mac2 7912 ) 7913 .as_str(), 7914 format!( 7915 "tap={},mac={},iommu=on", 7916 vfio_tap3, guest.network.l2_guest_mac3 7917 ) 7918 .as_str(), 7919 ]) 7920 .capture_output() 7921 .spawn() 7922 .unwrap(); 7923 7924 thread::sleep(std::time::Duration::new(30, 0)); 7925 7926 let r = std::panic::catch_unwind(|| { 7927 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 7928 thread::sleep(std::time::Duration::new(120, 0)); 7929 7930 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 7931 // added to its kernel command line. 7932 // Let's ssh into it and verify that it's there. If it is it means 7933 // we're in the right guest (The L2 one) because the QEMU L1 guest 7934 // does not have this command line tag. 7935 assert_eq!( 7936 guest 7937 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 7938 .unwrap() 7939 .trim() 7940 .parse::<u32>() 7941 .unwrap_or_default(), 7942 1 7943 ); 7944 7945 // Let's also verify from the second virtio-net device passed to 7946 // the L2 VM. 7947 assert_eq!( 7948 guest 7949 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 7950 .unwrap() 7951 .trim() 7952 .parse::<u32>() 7953 .unwrap_or_default(), 7954 1 7955 ); 7956 7957 // Check the amount of PCI devices appearing in L2 VM. 7958 assert_eq!( 7959 guest 7960 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 7961 .unwrap() 7962 .trim() 7963 .parse::<u32>() 7964 .unwrap_or_default(), 7965 8, 7966 ); 7967 7968 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 7969 assert_eq!( 7970 guest 7971 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 7972 .unwrap() 7973 .trim() 7974 .parse::<u32>() 7975 .unwrap_or_default(), 7976 1 7977 ); 7978 7979 // Hotplug an extra virtio-net device through L2 VM. 7980 guest 7981 .ssh_command_l1( 7982 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 7983 ) 7984 .unwrap(); 7985 guest 7986 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 7987 .unwrap(); 7988 let vfio_hotplug_output = guest 7989 .ssh_command_l1( 7990 "sudo /mnt/ch-remote \ 7991 --api-socket=/tmp/ch_api.sock \ 7992 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 7993 ) 7994 .unwrap(); 7995 assert!(vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}")); 7996 7997 thread::sleep(std::time::Duration::new(10, 0)); 7998 7999 // Let's also verify from the third virtio-net device passed to 8000 // the L2 VM. This third device has been hotplugged through the L2 8001 // VM, so this is our way to validate hotplug works for VFIO PCI. 8002 assert_eq!( 8003 guest 8004 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 8005 .unwrap() 8006 .trim() 8007 .parse::<u32>() 8008 .unwrap_or_default(), 8009 1 8010 ); 8011 8012 // Check the amount of PCI devices appearing in L2 VM. 8013 // There should be one more device than before, raising the count 8014 // up to 9 PCI devices. 8015 assert_eq!( 8016 guest 8017 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 8018 .unwrap() 8019 .trim() 8020 .parse::<u32>() 8021 .unwrap_or_default(), 8022 9, 8023 ); 8024 8025 // Let's now verify that we can correctly remove the virtio-net 8026 // device through the "remove-device" command responsible for 8027 // unplugging VFIO devices. 8028 guest 8029 .ssh_command_l1( 8030 "sudo /mnt/ch-remote \ 8031 --api-socket=/tmp/ch_api.sock \ 8032 remove-device vfio123", 8033 ) 8034 .unwrap(); 8035 thread::sleep(std::time::Duration::new(10, 0)); 8036 8037 // Check the amount of PCI devices appearing in L2 VM is back down 8038 // to 8 devices. 8039 assert_eq!( 8040 guest 8041 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 8042 .unwrap() 8043 .trim() 8044 .parse::<u32>() 8045 .unwrap_or_default(), 8046 8, 8047 ); 8048 8049 // Perform memory hotplug in L2 and validate the memory is showing 8050 // up as expected. In order to check, we will use the virtio-net 8051 // device already passed through L2 as a VFIO device, this will 8052 // verify that VFIO devices are functional with memory hotplug. 8053 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 8054 guest 8055 .ssh_command_l2_1( 8056 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 8057 ) 8058 .unwrap(); 8059 guest 8060 .ssh_command_l1( 8061 "sudo /mnt/ch-remote \ 8062 --api-socket=/tmp/ch_api.sock \ 8063 resize --memory=1073741824", 8064 ) 8065 .unwrap(); 8066 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 8067 }); 8068 8069 let _ = child.kill(); 8070 let output = child.wait_with_output().unwrap(); 8071 8072 cleanup_vfio_network_interfaces(); 8073 8074 handle_child_output(r, &output); 8075 } 8076 8077 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 8078 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8079 let guest = Guest::new(Box::new(jammy)); 8080 let api_socket = temp_api_path(&guest.tmp_dir); 8081 8082 let mut child = GuestCommand::new(&guest) 8083 .args(["--cpus", "boot=4"]) 8084 .args([ 8085 "--memory", 8086 format!("size=4G,hotplug_size=4G,hotplug_method={}", hotplug_method).as_str(), 8087 ]) 8088 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8089 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8090 .args(["--api-socket", &api_socket]) 8091 .default_disks() 8092 .default_net() 8093 .capture_output() 8094 .spawn() 8095 .unwrap(); 8096 8097 let r = std::panic::catch_unwind(|| { 8098 guest.wait_vm_boot(None).unwrap(); 8099 8100 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8101 8102 guest.enable_memory_hotplug(); 8103 8104 // Add RAM to the VM 8105 let desired_ram = 6 << 30; 8106 resize_command(&api_socket, None, Some(desired_ram), None, None); 8107 thread::sleep(std::time::Duration::new(30, 0)); 8108 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8109 8110 // Check the VFIO device works when RAM is increased to 6GiB 8111 guest.check_nvidia_gpu(); 8112 }); 8113 8114 let _ = child.kill(); 8115 let output = child.wait_with_output().unwrap(); 8116 8117 handle_child_output(r, &output); 8118 } 8119 8120 #[test] 8121 fn test_nvidia_card_memory_hotplug_acpi() { 8122 test_nvidia_card_memory_hotplug("acpi") 8123 } 8124 8125 #[test] 8126 fn test_nvidia_card_memory_hotplug_virtio_mem() { 8127 test_nvidia_card_memory_hotplug("virtio-mem") 8128 } 8129 8130 #[test] 8131 fn test_nvidia_card_pci_hotplug() { 8132 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8133 let guest = Guest::new(Box::new(jammy)); 8134 let api_socket = temp_api_path(&guest.tmp_dir); 8135 8136 let mut child = GuestCommand::new(&guest) 8137 .args(["--cpus", "boot=4"]) 8138 .args(["--memory", "size=4G"]) 8139 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8140 .args(["--api-socket", &api_socket]) 8141 .default_disks() 8142 .default_net() 8143 .capture_output() 8144 .spawn() 8145 .unwrap(); 8146 8147 let r = std::panic::catch_unwind(|| { 8148 guest.wait_vm_boot(None).unwrap(); 8149 8150 // Hotplug the card to the VM 8151 let (cmd_success, cmd_output) = remote_command_w_output( 8152 &api_socket, 8153 "add-device", 8154 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 8155 ); 8156 assert!(cmd_success); 8157 assert!(String::from_utf8_lossy(&cmd_output) 8158 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 8159 8160 thread::sleep(std::time::Duration::new(10, 0)); 8161 8162 // Check the VFIO device works after hotplug 8163 guest.check_nvidia_gpu(); 8164 }); 8165 8166 let _ = child.kill(); 8167 let output = child.wait_with_output().unwrap(); 8168 8169 handle_child_output(r, &output); 8170 } 8171 8172 #[test] 8173 fn test_nvidia_card_reboot() { 8174 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8175 let guest = Guest::new(Box::new(jammy)); 8176 let api_socket = temp_api_path(&guest.tmp_dir); 8177 8178 let mut child = GuestCommand::new(&guest) 8179 .args(["--cpus", "boot=4"]) 8180 .args(["--memory", "size=4G"]) 8181 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8182 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8183 .args(["--api-socket", &api_socket]) 8184 .default_disks() 8185 .default_net() 8186 .capture_output() 8187 .spawn() 8188 .unwrap(); 8189 8190 let r = std::panic::catch_unwind(|| { 8191 guest.wait_vm_boot(None).unwrap(); 8192 8193 // Check the VFIO device works after boot 8194 guest.check_nvidia_gpu(); 8195 8196 guest.reboot_linux(0, None); 8197 8198 // Check the VFIO device works after reboot 8199 guest.check_nvidia_gpu(); 8200 }); 8201 8202 let _ = child.kill(); 8203 let output = child.wait_with_output().unwrap(); 8204 8205 handle_child_output(r, &output); 8206 } 8207 } 8208 8209 mod live_migration { 8210 use crate::*; 8211 8212 fn start_live_migration( 8213 migration_socket: &str, 8214 src_api_socket: &str, 8215 dest_api_socket: &str, 8216 local: bool, 8217 ) -> bool { 8218 // Start to receive migration from the destintion VM 8219 let mut receive_migration = Command::new(clh_command("ch-remote")) 8220 .args([ 8221 &format!("--api-socket={}", dest_api_socket), 8222 "receive-migration", 8223 &format! {"unix:{}", migration_socket}, 8224 ]) 8225 .stderr(Stdio::piped()) 8226 .stdout(Stdio::piped()) 8227 .spawn() 8228 .unwrap(); 8229 // Give it '1s' to make sure the 'migration_socket' file is properly created 8230 thread::sleep(std::time::Duration::new(1, 0)); 8231 // Start to send migration from the source VM 8232 8233 let mut args = [ 8234 format!("--api-socket={}", &src_api_socket), 8235 "send-migration".to_string(), 8236 format! {"unix:{}", migration_socket}, 8237 ] 8238 .to_vec(); 8239 8240 if local { 8241 args.insert(2, "--local".to_string()); 8242 } 8243 8244 let mut send_migration = Command::new(clh_command("ch-remote")) 8245 .args(&args) 8246 .stderr(Stdio::piped()) 8247 .stdout(Stdio::piped()) 8248 .spawn() 8249 .unwrap(); 8250 8251 // The 'send-migration' command should be executed successfully within the given timeout 8252 let send_success = if let Some(status) = send_migration 8253 .wait_timeout(std::time::Duration::from_secs(30)) 8254 .unwrap() 8255 { 8256 status.success() 8257 } else { 8258 false 8259 }; 8260 8261 if !send_success { 8262 let _ = send_migration.kill(); 8263 let output = send_migration.wait_with_output().unwrap(); 8264 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 8265 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8266 } 8267 8268 // The 'receive-migration' command should be executed successfully within the given timeout 8269 let receive_success = if let Some(status) = receive_migration 8270 .wait_timeout(std::time::Duration::from_secs(30)) 8271 .unwrap() 8272 { 8273 status.success() 8274 } else { 8275 false 8276 }; 8277 8278 if !receive_success { 8279 let _ = receive_migration.kill(); 8280 let output = receive_migration.wait_with_output().unwrap(); 8281 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 8282 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8283 } 8284 8285 send_success && receive_success 8286 } 8287 8288 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 8289 let mut src_vm = src_vm; 8290 let mut dest_vm = dest_vm; 8291 8292 let _ = src_vm.kill(); 8293 let src_output = src_vm.wait_with_output().unwrap(); 8294 eprintln!( 8295 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8296 String::from_utf8_lossy(&src_output.stdout) 8297 ); 8298 eprintln!( 8299 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8300 String::from_utf8_lossy(&src_output.stderr) 8301 ); 8302 let _ = dest_vm.kill(); 8303 let dest_output = dest_vm.wait_with_output().unwrap(); 8304 eprintln!( 8305 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8306 String::from_utf8_lossy(&dest_output.stdout) 8307 ); 8308 eprintln!( 8309 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8310 String::from_utf8_lossy(&dest_output.stderr) 8311 ); 8312 8313 if let Some(ovs_vm) = ovs_vm { 8314 let mut ovs_vm = ovs_vm; 8315 let _ = ovs_vm.kill(); 8316 let ovs_output = ovs_vm.wait_with_output().unwrap(); 8317 eprintln!( 8318 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 8319 String::from_utf8_lossy(&ovs_output.stdout) 8320 ); 8321 eprintln!( 8322 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 8323 String::from_utf8_lossy(&ovs_output.stderr) 8324 ); 8325 8326 cleanup_ovs_dpdk(); 8327 } 8328 8329 panic!("Test failed: {}", message) 8330 } 8331 8332 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 8333 // same host. It ensures the following behaviors: 8334 // 1. The source VM is up and functional (including various virtio-devices are working properly); 8335 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 8336 // 3. The source VM terminated gracefully after live migration; 8337 // 4. The destination VM is functional (including various virtio-devices are working properly) after 8338 // live migration; 8339 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 8340 fn _test_live_migration(upgrade_test: bool, local: bool) { 8341 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8342 let guest = Guest::new(Box::new(focal)); 8343 let kernel_path = direct_kernel_boot_path(); 8344 let console_text = String::from("On a branch floating down river a cricket, singing."); 8345 let net_id = "net123"; 8346 let net_params = format!( 8347 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8348 net_id, guest.network.guest_mac, guest.network.host_ip 8349 ); 8350 8351 let memory_param: &[&str] = if local { 8352 &["--memory", "size=4G,shared=on"] 8353 } else { 8354 &["--memory", "size=4G"] 8355 }; 8356 8357 let boot_vcpus = 2; 8358 let max_vcpus = 4; 8359 8360 let pmem_temp_file = TempFile::new().unwrap(); 8361 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8362 std::process::Command::new("mkfs.ext4") 8363 .arg(pmem_temp_file.as_path()) 8364 .output() 8365 .expect("Expect creating disk image to succeed"); 8366 let pmem_path = String::from("/dev/pmem0"); 8367 8368 // Start the source VM 8369 let src_vm_path = if !upgrade_test { 8370 clh_command("cloud-hypervisor") 8371 } else { 8372 cloud_hypervisor_release_path() 8373 }; 8374 let src_api_socket = temp_api_path(&guest.tmp_dir); 8375 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8376 src_vm_cmd 8377 .args([ 8378 "--cpus", 8379 format!("boot={},max={}", boot_vcpus, max_vcpus).as_str(), 8380 ]) 8381 .args(memory_param) 8382 .args(["--kernel", kernel_path.to_str().unwrap()]) 8383 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8384 .default_disks() 8385 .args(["--net", net_params.as_str()]) 8386 .args(["--api-socket", &src_api_socket]) 8387 .args([ 8388 "--pmem", 8389 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8390 ]); 8391 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8392 8393 // Start the destination VM 8394 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8395 dest_api_socket.push_str(".dest"); 8396 let mut dest_child = GuestCommand::new(&guest) 8397 .args(["--api-socket", &dest_api_socket]) 8398 .capture_output() 8399 .spawn() 8400 .unwrap(); 8401 8402 let r = std::panic::catch_unwind(|| { 8403 guest.wait_vm_boot(None).unwrap(); 8404 8405 // Make sure the source VM is functaionl 8406 // Check the number of vCPUs 8407 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8408 8409 // Check the guest RAM 8410 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8411 8412 // Check the guest virtio-devices, e.g. block, rng, console, and net 8413 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8414 8415 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8416 // to make sure that removing and adding back the virtio-net device does 8417 // not break the live-migration support for virtio-pci. 8418 #[cfg(target_arch = "x86_64")] 8419 { 8420 assert!(remote_command( 8421 &src_api_socket, 8422 "remove-device", 8423 Some(net_id), 8424 )); 8425 thread::sleep(std::time::Duration::new(10, 0)); 8426 8427 // Plug the virtio-net device again 8428 assert!(remote_command( 8429 &src_api_socket, 8430 "add-net", 8431 Some(net_params.as_str()), 8432 )); 8433 thread::sleep(std::time::Duration::new(10, 0)); 8434 } 8435 8436 // Start the live-migration 8437 let migration_socket = String::from( 8438 guest 8439 .tmp_dir 8440 .as_path() 8441 .join("live-migration.sock") 8442 .to_str() 8443 .unwrap(), 8444 ); 8445 8446 assert!( 8447 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8448 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8449 ); 8450 }); 8451 8452 // Check and report any errors occured during the live-migration 8453 if r.is_err() { 8454 print_and_panic( 8455 src_child, 8456 dest_child, 8457 None, 8458 "Error occured during live-migration", 8459 ); 8460 } 8461 8462 // Check the source vm has been terminated successful (give it '3s' to settle) 8463 thread::sleep(std::time::Duration::new(3, 0)); 8464 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8465 print_and_panic( 8466 src_child, 8467 dest_child, 8468 None, 8469 "source VM was not terminated successfully.", 8470 ); 8471 }; 8472 8473 // Post live-migration check to make sure the destination VM is funcational 8474 let r = std::panic::catch_unwind(|| { 8475 // Perform same checks to validate VM has been properly migrated 8476 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8477 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8478 8479 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8480 }); 8481 8482 // Clean-up the destination VM and make sure it terminated correctly 8483 let _ = dest_child.kill(); 8484 let dest_output = dest_child.wait_with_output().unwrap(); 8485 handle_child_output(r, &dest_output); 8486 8487 // Check the destination VM has the expected 'concole_text' from its output 8488 let r = std::panic::catch_unwind(|| { 8489 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8490 }); 8491 handle_child_output(r, &dest_output); 8492 } 8493 8494 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 8495 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8496 let guest = Guest::new(Box::new(focal)); 8497 let kernel_path = direct_kernel_boot_path(); 8498 let console_text = String::from("On a branch floating down river a cricket, singing."); 8499 let net_id = "net123"; 8500 let net_params = format!( 8501 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8502 net_id, guest.network.guest_mac, guest.network.host_ip 8503 ); 8504 8505 let memory_param: &[&str] = if local { 8506 &[ 8507 "--memory", 8508 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 8509 "--balloon", 8510 "size=0", 8511 ] 8512 } else { 8513 &[ 8514 "--memory", 8515 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 8516 "--balloon", 8517 "size=0", 8518 ] 8519 }; 8520 8521 let boot_vcpus = 2; 8522 let max_vcpus = 4; 8523 8524 let pmem_temp_file = TempFile::new().unwrap(); 8525 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8526 std::process::Command::new("mkfs.ext4") 8527 .arg(pmem_temp_file.as_path()) 8528 .output() 8529 .expect("Expect creating disk image to succeed"); 8530 let pmem_path = String::from("/dev/pmem0"); 8531 8532 // Start the source VM 8533 let src_vm_path = if !upgrade_test { 8534 clh_command("cloud-hypervisor") 8535 } else { 8536 cloud_hypervisor_release_path() 8537 }; 8538 let src_api_socket = temp_api_path(&guest.tmp_dir); 8539 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8540 src_vm_cmd 8541 .args([ 8542 "--cpus", 8543 format!("boot={},max={}", boot_vcpus, max_vcpus).as_str(), 8544 ]) 8545 .args(memory_param) 8546 .args(["--kernel", kernel_path.to_str().unwrap()]) 8547 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8548 .default_disks() 8549 .args(["--net", net_params.as_str()]) 8550 .args(["--api-socket", &src_api_socket]) 8551 .args([ 8552 "--pmem", 8553 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8554 ]); 8555 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8556 8557 // Start the destination VM 8558 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8559 dest_api_socket.push_str(".dest"); 8560 let mut dest_child = GuestCommand::new(&guest) 8561 .args(["--api-socket", &dest_api_socket]) 8562 .capture_output() 8563 .spawn() 8564 .unwrap(); 8565 8566 let r = std::panic::catch_unwind(|| { 8567 guest.wait_vm_boot(None).unwrap(); 8568 8569 // Make sure the source VM is functaionl 8570 // Check the number of vCPUs 8571 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8572 8573 // Check the guest RAM 8574 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8575 // Increase the guest RAM 8576 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 8577 thread::sleep(std::time::Duration::new(5, 0)); 8578 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8579 // Use balloon to remove RAM from the VM 8580 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 8581 thread::sleep(std::time::Duration::new(5, 0)); 8582 let total_memory = guest.get_total_memory().unwrap_or_default(); 8583 assert!(total_memory > 4_800_000); 8584 assert!(total_memory < 5_760_000); 8585 8586 // Check the guest virtio-devices, e.g. block, rng, console, and net 8587 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8588 8589 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8590 // to make sure that removing and adding back the virtio-net device does 8591 // not break the live-migration support for virtio-pci. 8592 #[cfg(target_arch = "x86_64")] 8593 { 8594 assert!(remote_command( 8595 &src_api_socket, 8596 "remove-device", 8597 Some(net_id), 8598 )); 8599 thread::sleep(std::time::Duration::new(10, 0)); 8600 8601 // Plug the virtio-net device again 8602 assert!(remote_command( 8603 &src_api_socket, 8604 "add-net", 8605 Some(net_params.as_str()), 8606 )); 8607 thread::sleep(std::time::Duration::new(10, 0)); 8608 } 8609 8610 // Start the live-migration 8611 let migration_socket = String::from( 8612 guest 8613 .tmp_dir 8614 .as_path() 8615 .join("live-migration.sock") 8616 .to_str() 8617 .unwrap(), 8618 ); 8619 8620 assert!( 8621 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8622 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8623 ); 8624 }); 8625 8626 // Check and report any errors occured during the live-migration 8627 if r.is_err() { 8628 print_and_panic( 8629 src_child, 8630 dest_child, 8631 None, 8632 "Error occured during live-migration", 8633 ); 8634 } 8635 8636 // Check the source vm has been terminated successful (give it '3s' to settle) 8637 thread::sleep(std::time::Duration::new(3, 0)); 8638 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8639 print_and_panic( 8640 src_child, 8641 dest_child, 8642 None, 8643 "source VM was not terminated successfully.", 8644 ); 8645 }; 8646 8647 // Post live-migration check to make sure the destination VM is funcational 8648 let r = std::panic::catch_unwind(|| { 8649 // Perform same checks to validate VM has been properly migrated 8650 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8651 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8652 8653 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8654 8655 // Perform checks on guest RAM using balloon 8656 let total_memory = guest.get_total_memory().unwrap_or_default(); 8657 assert!(total_memory > 4_800_000); 8658 assert!(total_memory < 5_760_000); 8659 // Deflate balloon to restore entire RAM to the VM 8660 resize_command(&dest_api_socket, None, None, Some(0), None); 8661 thread::sleep(std::time::Duration::new(5, 0)); 8662 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8663 // Decrease guest RAM with virtio-mem 8664 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 8665 thread::sleep(std::time::Duration::new(5, 0)); 8666 let total_memory = guest.get_total_memory().unwrap_or_default(); 8667 assert!(total_memory > 4_800_000); 8668 assert!(total_memory < 5_760_000); 8669 }); 8670 8671 // Clean-up the destination VM and make sure it terminated correctly 8672 let _ = dest_child.kill(); 8673 let dest_output = dest_child.wait_with_output().unwrap(); 8674 handle_child_output(r, &dest_output); 8675 8676 // Check the destination VM has the expected 'concole_text' from its output 8677 let r = std::panic::catch_unwind(|| { 8678 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8679 }); 8680 handle_child_output(r, &dest_output); 8681 } 8682 8683 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 8684 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8685 let guest = Guest::new(Box::new(focal)); 8686 let kernel_path = direct_kernel_boot_path(); 8687 let console_text = String::from("On a branch floating down river a cricket, singing."); 8688 let net_id = "net123"; 8689 let net_params = format!( 8690 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8691 net_id, guest.network.guest_mac, guest.network.host_ip 8692 ); 8693 8694 let memory_param: &[&str] = if local { 8695 &[ 8696 "--memory", 8697 "size=0,hotplug_method=virtio-mem,shared=on", 8698 "--memory-zone", 8699 "id=mem0,size=1G,hotplug_size=4G,shared=on", 8700 "id=mem1,size=1G,hotplug_size=4G,shared=on", 8701 "id=mem2,size=2G,hotplug_size=4G,shared=on", 8702 "--numa", 8703 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 8704 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 8705 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 8706 ] 8707 } else { 8708 &[ 8709 "--memory", 8710 "size=0,hotplug_method=virtio-mem", 8711 "--memory-zone", 8712 "id=mem0,size=1G,hotplug_size=4G", 8713 "id=mem1,size=1G,hotplug_size=4G", 8714 "id=mem2,size=2G,hotplug_size=4G", 8715 "--numa", 8716 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 8717 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 8718 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 8719 ] 8720 }; 8721 8722 let boot_vcpus = 6; 8723 let max_vcpus = 12; 8724 8725 let pmem_temp_file = TempFile::new().unwrap(); 8726 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8727 std::process::Command::new("mkfs.ext4") 8728 .arg(pmem_temp_file.as_path()) 8729 .output() 8730 .expect("Expect creating disk image to succeed"); 8731 let pmem_path = String::from("/dev/pmem0"); 8732 8733 // Start the source VM 8734 let src_vm_path = if !upgrade_test { 8735 clh_command("cloud-hypervisor") 8736 } else { 8737 cloud_hypervisor_release_path() 8738 }; 8739 let src_api_socket = temp_api_path(&guest.tmp_dir); 8740 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8741 src_vm_cmd 8742 .args([ 8743 "--cpus", 8744 format!("boot={},max={}", boot_vcpus, max_vcpus).as_str(), 8745 ]) 8746 .args(memory_param) 8747 .args(["--kernel", kernel_path.to_str().unwrap()]) 8748 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8749 .default_disks() 8750 .args(["--net", net_params.as_str()]) 8751 .args(["--api-socket", &src_api_socket]) 8752 .args([ 8753 "--pmem", 8754 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8755 ]); 8756 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8757 8758 // Start the destination VM 8759 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8760 dest_api_socket.push_str(".dest"); 8761 let mut dest_child = GuestCommand::new(&guest) 8762 .args(["--api-socket", &dest_api_socket]) 8763 .capture_output() 8764 .spawn() 8765 .unwrap(); 8766 8767 let r = std::panic::catch_unwind(|| { 8768 guest.wait_vm_boot(None).unwrap(); 8769 8770 // Make sure the source VM is functaionl 8771 // Check the number of vCPUs 8772 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8773 8774 // Check the guest RAM 8775 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 8776 8777 // Check the guest virtio-devices, e.g. block, rng, console, and net 8778 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8779 8780 // Check the NUMA parameters are applied correctly and resize 8781 // each zone to test the case where we migrate a VM with the 8782 // virtio-mem regions being used. 8783 { 8784 guest.check_numa_common( 8785 Some(&[960_000, 960_000, 1_920_000]), 8786 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8787 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8788 ); 8789 8790 // AArch64 currently does not support hotplug, and therefore we only 8791 // test hotplug-related function on x86_64 here. 8792 #[cfg(target_arch = "x86_64")] 8793 { 8794 guest.enable_memory_hotplug(); 8795 8796 // Resize every memory zone and check each associated NUMA node 8797 // has been assigned the right amount of memory. 8798 resize_zone_command(&src_api_socket, "mem0", "2G"); 8799 resize_zone_command(&src_api_socket, "mem1", "2G"); 8800 resize_zone_command(&src_api_socket, "mem2", "3G"); 8801 thread::sleep(std::time::Duration::new(5, 0)); 8802 8803 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 8804 } 8805 } 8806 8807 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8808 // to make sure that removing and adding back the virtio-net device does 8809 // not break the live-migration support for virtio-pci. 8810 #[cfg(target_arch = "x86_64")] 8811 { 8812 assert!(remote_command( 8813 &src_api_socket, 8814 "remove-device", 8815 Some(net_id), 8816 )); 8817 thread::sleep(std::time::Duration::new(10, 0)); 8818 8819 // Plug the virtio-net device again 8820 assert!(remote_command( 8821 &src_api_socket, 8822 "add-net", 8823 Some(net_params.as_str()), 8824 )); 8825 thread::sleep(std::time::Duration::new(10, 0)); 8826 } 8827 8828 // Start the live-migration 8829 let migration_socket = String::from( 8830 guest 8831 .tmp_dir 8832 .as_path() 8833 .join("live-migration.sock") 8834 .to_str() 8835 .unwrap(), 8836 ); 8837 8838 assert!( 8839 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8840 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8841 ); 8842 }); 8843 8844 // Check and report any errors occured during the live-migration 8845 if r.is_err() { 8846 print_and_panic( 8847 src_child, 8848 dest_child, 8849 None, 8850 "Error occured during live-migration", 8851 ); 8852 } 8853 8854 // Check the source vm has been terminated successful (give it '3s' to settle) 8855 thread::sleep(std::time::Duration::new(3, 0)); 8856 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8857 print_and_panic( 8858 src_child, 8859 dest_child, 8860 None, 8861 "source VM was not terminated successfully.", 8862 ); 8863 }; 8864 8865 // Post live-migration check to make sure the destination VM is funcational 8866 let r = std::panic::catch_unwind(|| { 8867 // Perform same checks to validate VM has been properly migrated 8868 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8869 #[cfg(target_arch = "x86_64")] 8870 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 8871 #[cfg(target_arch = "aarch64")] 8872 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8873 8874 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8875 8876 // Perform NUMA related checks 8877 { 8878 #[cfg(target_arch = "aarch64")] 8879 { 8880 guest.check_numa_common( 8881 Some(&[960_000, 960_000, 1_920_000]), 8882 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8883 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8884 ); 8885 } 8886 8887 // AArch64 currently does not support hotplug, and therefore we only 8888 // test hotplug-related function on x86_64 here. 8889 #[cfg(target_arch = "x86_64")] 8890 { 8891 guest.check_numa_common( 8892 Some(&[1_920_000, 1_920_000, 2_880_000]), 8893 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8894 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8895 ); 8896 8897 guest.enable_memory_hotplug(); 8898 8899 // Resize every memory zone and check each associated NUMA node 8900 // has been assigned the right amount of memory. 8901 resize_zone_command(&dest_api_socket, "mem0", "4G"); 8902 resize_zone_command(&dest_api_socket, "mem1", "4G"); 8903 resize_zone_command(&dest_api_socket, "mem2", "4G"); 8904 // Resize to the maximum amount of CPUs and check each NUMA 8905 // node has been assigned the right CPUs set. 8906 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 8907 thread::sleep(std::time::Duration::new(5, 0)); 8908 8909 guest.check_numa_common( 8910 Some(&[3_840_000, 3_840_000, 3_840_000]), 8911 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 8912 None, 8913 ); 8914 } 8915 } 8916 }); 8917 8918 // Clean-up the destination VM and make sure it terminated correctly 8919 let _ = dest_child.kill(); 8920 let dest_output = dest_child.wait_with_output().unwrap(); 8921 handle_child_output(r, &dest_output); 8922 8923 // Check the destination VM has the expected 'concole_text' from its output 8924 let r = std::panic::catch_unwind(|| { 8925 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8926 }); 8927 handle_child_output(r, &dest_output); 8928 } 8929 8930 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 8931 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8932 let guest = Guest::new(Box::new(focal)); 8933 let kernel_path = direct_kernel_boot_path(); 8934 let console_text = String::from("On a branch floating down river a cricket, singing."); 8935 let net_id = "net123"; 8936 let net_params = format!( 8937 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8938 net_id, guest.network.guest_mac, guest.network.host_ip 8939 ); 8940 8941 let memory_param: &[&str] = if local { 8942 &["--memory", "size=4G,shared=on"] 8943 } else { 8944 &["--memory", "size=4G"] 8945 }; 8946 8947 let boot_vcpus = 2; 8948 let max_vcpus = 4; 8949 8950 let pmem_temp_file = TempFile::new().unwrap(); 8951 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8952 std::process::Command::new("mkfs.ext4") 8953 .arg(pmem_temp_file.as_path()) 8954 .output() 8955 .expect("Expect creating disk image to succeed"); 8956 let pmem_path = String::from("/dev/pmem0"); 8957 8958 // Start the source VM 8959 let src_vm_path = if !upgrade_test { 8960 clh_command("cloud-hypervisor") 8961 } else { 8962 cloud_hypervisor_release_path() 8963 }; 8964 let src_api_socket = temp_api_path(&guest.tmp_dir); 8965 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8966 src_vm_cmd 8967 .args([ 8968 "--cpus", 8969 format!("boot={},max={}", boot_vcpus, max_vcpus).as_str(), 8970 ]) 8971 .args(memory_param) 8972 .args(["--kernel", kernel_path.to_str().unwrap()]) 8973 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8974 .default_disks() 8975 .args(["--net", net_params.as_str()]) 8976 .args(["--api-socket", &src_api_socket]) 8977 .args([ 8978 "--pmem", 8979 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8980 ]) 8981 .args(["--watchdog"]); 8982 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8983 8984 // Start the destination VM 8985 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8986 dest_api_socket.push_str(".dest"); 8987 let mut dest_child = GuestCommand::new(&guest) 8988 .args(["--api-socket", &dest_api_socket]) 8989 .capture_output() 8990 .spawn() 8991 .unwrap(); 8992 8993 let r = std::panic::catch_unwind(|| { 8994 guest.wait_vm_boot(None).unwrap(); 8995 8996 // Make sure the source VM is functaionl 8997 // Check the number of vCPUs 8998 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8999 // Check the guest RAM 9000 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9001 // Check the guest virtio-devices, e.g. block, rng, console, and net 9002 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9003 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9004 // to make sure that removing and adding back the virtio-net device does 9005 // not break the live-migration support for virtio-pci. 9006 #[cfg(target_arch = "x86_64")] 9007 { 9008 assert!(remote_command( 9009 &src_api_socket, 9010 "remove-device", 9011 Some(net_id), 9012 )); 9013 thread::sleep(std::time::Duration::new(10, 0)); 9014 9015 // Plug the virtio-net device again 9016 assert!(remote_command( 9017 &src_api_socket, 9018 "add-net", 9019 Some(net_params.as_str()), 9020 )); 9021 thread::sleep(std::time::Duration::new(10, 0)); 9022 } 9023 9024 // Enable watchdog and ensure its functional 9025 let mut expected_reboot_count = 1; 9026 // Enable the watchdog with a 15s timeout 9027 enable_guest_watchdog(&guest, 15); 9028 // Reboot and check that systemd has activated the watchdog 9029 guest.ssh_command("sudo reboot").unwrap(); 9030 guest.wait_vm_boot(None).unwrap(); 9031 expected_reboot_count += 1; 9032 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9033 assert_eq!( 9034 guest 9035 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 9036 .unwrap() 9037 .trim() 9038 .parse::<u32>() 9039 .unwrap_or_default(), 9040 2 9041 ); 9042 // Allow some normal time to elapse to check we don't get spurious reboots 9043 thread::sleep(std::time::Duration::new(40, 0)); 9044 // Check no reboot 9045 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9046 9047 // Start the live-migration 9048 let migration_socket = String::from( 9049 guest 9050 .tmp_dir 9051 .as_path() 9052 .join("live-migration.sock") 9053 .to_str() 9054 .unwrap(), 9055 ); 9056 9057 assert!( 9058 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9059 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9060 ); 9061 }); 9062 9063 // Check and report any errors occured during the live-migration 9064 if r.is_err() { 9065 print_and_panic( 9066 src_child, 9067 dest_child, 9068 None, 9069 "Error occured during live-migration", 9070 ); 9071 } 9072 9073 // Check the source vm has been terminated successful (give it '3s' to settle) 9074 thread::sleep(std::time::Duration::new(3, 0)); 9075 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9076 print_and_panic( 9077 src_child, 9078 dest_child, 9079 None, 9080 "source VM was not terminated successfully.", 9081 ); 9082 }; 9083 9084 // Post live-migration check to make sure the destination VM is funcational 9085 let r = std::panic::catch_unwind(|| { 9086 // Perform same checks to validate VM has been properly migrated 9087 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9088 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9089 9090 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9091 9092 // Perform checks on watchdog 9093 let mut expected_reboot_count = 2; 9094 9095 // Allow some normal time to elapse to check we don't get spurious reboots 9096 thread::sleep(std::time::Duration::new(40, 0)); 9097 // Check no reboot 9098 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9099 9100 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 9101 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 9102 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 9103 guest.wait_vm_boot(Some(50)).unwrap(); 9104 // Check a reboot is triggerred by the watchdog 9105 expected_reboot_count += 1; 9106 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9107 9108 #[cfg(target_arch = "x86_64")] 9109 { 9110 // Now pause the VM and remain offline for 30s 9111 assert!(remote_command(&dest_api_socket, "pause", None)); 9112 thread::sleep(std::time::Duration::new(30, 0)); 9113 assert!(remote_command(&dest_api_socket, "resume", None)); 9114 9115 // Check no reboot 9116 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9117 } 9118 }); 9119 9120 // Clean-up the destination VM and make sure it terminated correctly 9121 let _ = dest_child.kill(); 9122 let dest_output = dest_child.wait_with_output().unwrap(); 9123 handle_child_output(r, &dest_output); 9124 9125 // Check the destination VM has the expected 'concole_text' from its output 9126 let r = std::panic::catch_unwind(|| { 9127 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9128 }); 9129 handle_child_output(r, &dest_output); 9130 } 9131 9132 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 9133 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9134 let ovs_guest = Guest::new(Box::new(ovs_focal)); 9135 9136 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9137 let migration_guest = Guest::new(Box::new(migration_focal)); 9138 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 9139 9140 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 9141 let (mut ovs_child, mut src_child) = 9142 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 9143 9144 // Start the destination VM 9145 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 9146 dest_api_socket.push_str(".dest"); 9147 let mut dest_child = GuestCommand::new(&migration_guest) 9148 .args(["--api-socket", &dest_api_socket]) 9149 .capture_output() 9150 .spawn() 9151 .unwrap(); 9152 9153 let r = std::panic::catch_unwind(|| { 9154 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 9155 thread::sleep(std::time::Duration::new(1, 0)); 9156 9157 // Start the live-migration 9158 let migration_socket = String::from( 9159 migration_guest 9160 .tmp_dir 9161 .as_path() 9162 .join("live-migration.sock") 9163 .to_str() 9164 .unwrap(), 9165 ); 9166 9167 assert!( 9168 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9169 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9170 ); 9171 }); 9172 9173 // Check and report any errors occured during the live-migration 9174 if r.is_err() { 9175 print_and_panic( 9176 src_child, 9177 dest_child, 9178 Some(ovs_child), 9179 "Error occured during live-migration", 9180 ); 9181 } 9182 9183 // Check the source vm has been terminated successful (give it '3s' to settle) 9184 thread::sleep(std::time::Duration::new(3, 0)); 9185 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9186 print_and_panic( 9187 src_child, 9188 dest_child, 9189 Some(ovs_child), 9190 "source VM was not terminated successfully.", 9191 ); 9192 }; 9193 9194 // Post live-migration check to make sure the destination VM is funcational 9195 let r = std::panic::catch_unwind(|| { 9196 // Perform same checks to validate VM has been properly migrated 9197 // Spawn a new netcat listener in the OVS VM 9198 let guest_ip = ovs_guest.network.guest_ip.clone(); 9199 thread::spawn(move || { 9200 ssh_command_ip( 9201 "nc -l 12345", 9202 &guest_ip, 9203 DEFAULT_SSH_RETRIES, 9204 DEFAULT_SSH_TIMEOUT, 9205 ) 9206 .unwrap(); 9207 }); 9208 9209 // Wait for the server to be listening 9210 thread::sleep(std::time::Duration::new(5, 0)); 9211 9212 // And check the connection is still functional after live-migration 9213 migration_guest 9214 .ssh_command("nc -vz 172.100.0.1 12345") 9215 .unwrap(); 9216 }); 9217 9218 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 9219 let _ = dest_child.kill(); 9220 let _ = ovs_child.kill(); 9221 let dest_output = dest_child.wait_with_output().unwrap(); 9222 let ovs_output = ovs_child.wait_with_output().unwrap(); 9223 9224 cleanup_ovs_dpdk(); 9225 9226 handle_child_output(r, &dest_output); 9227 handle_child_output(Ok(()), &ovs_output); 9228 } 9229 9230 mod live_migration_parallel { 9231 use super::*; 9232 #[test] 9233 fn test_live_migration_basic() { 9234 _test_live_migration(false, false) 9235 } 9236 9237 #[test] 9238 fn test_live_migration_local() { 9239 _test_live_migration(false, true) 9240 } 9241 9242 #[test] 9243 #[cfg(not(feature = "mshv"))] 9244 fn test_live_migration_numa() { 9245 _test_live_migration_numa(false, false) 9246 } 9247 9248 #[test] 9249 #[cfg(not(feature = "mshv"))] 9250 fn test_live_migration_numa_local() { 9251 _test_live_migration_numa(false, true) 9252 } 9253 9254 #[test] 9255 fn test_live_migration_watchdog() { 9256 _test_live_migration_watchdog(false, false) 9257 } 9258 9259 #[test] 9260 fn test_live_migration_watchdog_local() { 9261 _test_live_migration_watchdog(false, true) 9262 } 9263 9264 #[test] 9265 fn test_live_migration_balloon() { 9266 _test_live_migration_balloon(false, false) 9267 } 9268 9269 #[test] 9270 fn test_live_migration_balloon_local() { 9271 _test_live_migration_balloon(false, true) 9272 } 9273 9274 #[test] 9275 fn test_live_upgrade_basic() { 9276 _test_live_migration(true, false) 9277 } 9278 9279 #[test] 9280 fn test_live_upgrade_local() { 9281 _test_live_migration(true, true) 9282 } 9283 9284 #[test] 9285 #[cfg(not(feature = "mshv"))] 9286 fn test_live_upgrade_numa() { 9287 _test_live_migration_numa(true, false) 9288 } 9289 9290 #[test] 9291 #[cfg(not(feature = "mshv"))] 9292 fn test_live_upgrade_numa_local() { 9293 _test_live_migration_numa(true, true) 9294 } 9295 9296 #[test] 9297 fn test_live_upgrade_watchdog() { 9298 _test_live_migration_watchdog(true, false) 9299 } 9300 9301 #[test] 9302 fn test_live_upgrade_watchdog_local() { 9303 _test_live_migration_watchdog(true, true) 9304 } 9305 9306 #[test] 9307 fn test_live_upgrade_balloon() { 9308 _test_live_migration_balloon(true, false) 9309 } 9310 9311 #[test] 9312 fn test_live_upgrade_balloon_local() { 9313 _test_live_migration_balloon(true, true) 9314 } 9315 } 9316 9317 mod live_migration_sequential { 9318 #[cfg(target_arch = "x86_64")] 9319 #[cfg(not(feature = "mshv"))] 9320 use super::*; 9321 9322 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 9323 #[test] 9324 #[cfg(target_arch = "x86_64")] 9325 #[cfg(not(feature = "mshv"))] 9326 fn test_live_migration_ovs_dpdk() { 9327 _test_live_migration_ovs_dpdk(false, false); 9328 } 9329 9330 #[test] 9331 #[cfg(target_arch = "x86_64")] 9332 #[cfg(not(feature = "mshv"))] 9333 fn test_live_migration_ovs_dpdk_local() { 9334 _test_live_migration_ovs_dpdk(false, true); 9335 } 9336 9337 #[test] 9338 #[cfg(target_arch = "x86_64")] 9339 #[cfg(not(feature = "mshv"))] 9340 fn test_live_upgrade_ovs_dpdk() { 9341 _test_live_migration_ovs_dpdk(true, false); 9342 } 9343 9344 #[test] 9345 #[cfg(target_arch = "x86_64")] 9346 #[cfg(not(feature = "mshv"))] 9347 fn test_live_upgrade_ovs_dpdk_local() { 9348 _test_live_migration_ovs_dpdk(true, true); 9349 } 9350 } 9351 } 9352 9353 #[cfg(target_arch = "aarch64")] 9354 mod aarch64_acpi { 9355 use crate::*; 9356 9357 #[test] 9358 fn test_simple_launch_acpi() { 9359 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9360 9361 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 9362 let guest = Guest::new(disk_config); 9363 9364 let mut child = GuestCommand::new(&guest) 9365 .args(["--cpus", "boot=1"]) 9366 .args(["--memory", "size=512M"]) 9367 .args(["--kernel", edk2_path().to_str().unwrap()]) 9368 .default_disks() 9369 .default_net() 9370 .args(["--serial", "tty", "--console", "off"]) 9371 .capture_output() 9372 .spawn() 9373 .unwrap(); 9374 9375 let r = std::panic::catch_unwind(|| { 9376 guest.wait_vm_boot(Some(120)).unwrap(); 9377 9378 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 9379 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 9380 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 9381 }); 9382 9383 let _ = child.kill(); 9384 let output = child.wait_with_output().unwrap(); 9385 9386 handle_child_output(r, &output); 9387 }); 9388 } 9389 9390 #[test] 9391 fn test_guest_numa_nodes_acpi() { 9392 _test_guest_numa_nodes(true); 9393 } 9394 9395 #[test] 9396 fn test_cpu_topology_421_acpi() { 9397 test_cpu_topology(4, 2, 1, true); 9398 } 9399 9400 #[test] 9401 fn test_cpu_topology_142_acpi() { 9402 test_cpu_topology(1, 4, 2, true); 9403 } 9404 9405 #[test] 9406 fn test_cpu_topology_262_acpi() { 9407 test_cpu_topology(2, 6, 2, true); 9408 } 9409 9410 #[test] 9411 fn test_power_button_acpi() { 9412 _test_power_button(true); 9413 } 9414 9415 #[test] 9416 fn test_virtio_iommu() { 9417 _test_virtio_iommu(true) 9418 } 9419 } 9420 9421 mod rate_limiter { 9422 use super::*; 9423 9424 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 9425 // compared to given 'limit' rate. 9426 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 9427 let upper_limit = limit * (1_f64 + difference); 9428 let lower_limit = limit * (1_f64 - difference); 9429 9430 if measured > lower_limit && measured < upper_limit { 9431 return true; 9432 } 9433 9434 eprintln!( 9435 "\n\n==== check_rate_limit failed! ====\n\nmeasured={}, , lower_limit={}, upper_limit={}\n\n", 9436 measured, lower_limit, upper_limit 9437 ); 9438 9439 false 9440 } 9441 9442 fn _test_rate_limiter_net(rx: bool) { 9443 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9444 let guest = Guest::new(Box::new(focal)); 9445 9446 let test_timeout = 10; 9447 let num_queues = 2; 9448 let queue_size = 256; 9449 let bw_size = 10485760_u64; // bytes 9450 let bw_refill_time = 100; // ms 9451 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 9452 9453 let net_params = format!( 9454 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 9455 guest.network.guest_mac, 9456 guest.network.host_ip, 9457 num_queues, 9458 queue_size, 9459 bw_size, 9460 bw_refill_time, 9461 ); 9462 9463 let mut child = GuestCommand::new(&guest) 9464 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 9465 .args(["--memory", "size=4G"]) 9466 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9467 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9468 .default_disks() 9469 .args(["--net", net_params.as_str()]) 9470 .capture_output() 9471 .spawn() 9472 .unwrap(); 9473 9474 let r = std::panic::catch_unwind(|| { 9475 guest.wait_vm_boot(None).unwrap(); 9476 let measured_bps = 9477 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx).unwrap(); 9478 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 9479 }); 9480 9481 let _ = child.kill(); 9482 let output = child.wait_with_output().unwrap(); 9483 handle_child_output(r, &output); 9484 } 9485 9486 #[test] 9487 fn test_rate_limiter_net_rx() { 9488 _test_rate_limiter_net(true); 9489 } 9490 9491 #[test] 9492 fn test_rate_limiter_net_tx() { 9493 _test_rate_limiter_net(false); 9494 } 9495 9496 fn _test_rate_limiter_block(bandwidth: bool) { 9497 let test_timeout = 10; 9498 let num_queues = 1; 9499 let fio_ops = FioOps::RandRW; 9500 9501 let bw_size = if bandwidth { 9502 10485760_u64 // bytes 9503 } else { 9504 100_u64 // I/O 9505 }; 9506 let bw_refill_time = 100; // ms 9507 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 9508 9509 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9510 let guest = Guest::new(Box::new(focal)); 9511 let api_socket = temp_api_path(&guest.tmp_dir); 9512 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 9513 let blk_rate_limiter_test_img = 9514 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 9515 9516 // Create the test block image 9517 assert!(exec_host_command_output(&format!( 9518 "dd if=/dev/zero of={} bs=1M count=1024", 9519 blk_rate_limiter_test_img 9520 )) 9521 .status 9522 .success()); 9523 9524 let test_blk_params = if bandwidth { 9525 format!( 9526 "path={},bw_size={},bw_refill_time={}", 9527 blk_rate_limiter_test_img, bw_size, bw_refill_time 9528 ) 9529 } else { 9530 format!( 9531 "path={},ops_size={},ops_refill_time={}", 9532 blk_rate_limiter_test_img, bw_size, bw_refill_time 9533 ) 9534 }; 9535 9536 let mut child = GuestCommand::new(&guest) 9537 .args(["--cpus", &format!("boot={}", num_queues)]) 9538 .args(["--memory", "size=4G"]) 9539 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9540 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9541 .args([ 9542 "--disk", 9543 format!( 9544 "path={}", 9545 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 9546 ) 9547 .as_str(), 9548 format!( 9549 "path={}", 9550 guest.disk_config.disk(DiskType::CloudInit).unwrap() 9551 ) 9552 .as_str(), 9553 test_blk_params.as_str(), 9554 ]) 9555 .default_net() 9556 .args(["--api-socket", &api_socket]) 9557 .capture_output() 9558 .spawn() 9559 .unwrap(); 9560 9561 let r = std::panic::catch_unwind(|| { 9562 guest.wait_vm_boot(None).unwrap(); 9563 9564 let fio_command = format!( 9565 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 9566 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 9567 --rw={} --runtime={} --numjobs={}", 9568 fio_ops, test_timeout, num_queues 9569 ); 9570 let output = guest.ssh_command(&fio_command).unwrap(); 9571 9572 // Parse fio output 9573 let measured_rate = if bandwidth { 9574 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 9575 } else { 9576 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 9577 }; 9578 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 9579 }); 9580 9581 let _ = child.kill(); 9582 let output = child.wait_with_output().unwrap(); 9583 handle_child_output(r, &output); 9584 } 9585 9586 #[test] 9587 fn test_rate_limiter_block_bandwidth() { 9588 _test_rate_limiter_block(true) 9589 } 9590 9591 #[test] 9592 fn test_rate_limiter_block_iops() { 9593 _test_rate_limiter_block(false) 9594 } 9595 } 9596