1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 #[cfg(target_arch = "x86_64")] 28 use std::sync::Mutex; 29 use std::thread; 30 use test_infra::*; 31 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 32 use wait_timeout::ChildExt; 33 34 #[cfg(target_arch = "x86_64")] 35 mod x86_64 { 36 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw"; 37 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 38 pub const FOCAL_SGX_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-sgx.raw"; 39 pub const HIRSUTE_NVIDIA_IMAGE_NAME: &str = "hirsute-server-cloudimg-amd64-nvidia.raw"; 40 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 41 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 42 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 43 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20220329-0.raw"; 44 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2019.raw"; 45 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 46 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 47 } 48 49 #[cfg(target_arch = "x86_64")] 50 use x86_64::*; 51 52 #[cfg(target_arch = "aarch64")] 53 mod aarch64 { 54 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 55 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 56 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 57 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 58 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 59 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 60 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 61 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 62 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 63 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 64 } 65 66 #[cfg(target_arch = "aarch64")] 67 use aarch64::*; 68 69 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 70 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 71 72 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 73 74 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 75 let mut workload_path = dirs::home_dir().unwrap(); 76 workload_path.push("workloads"); 77 78 let mut virtiofsd_path = workload_path; 79 virtiofsd_path.push("virtiofsd"); 80 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 81 82 let virtiofsd_socket_path = 83 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 84 85 // Start the daemon 86 let child = Command::new(virtiofsd_path.as_str()) 87 .args(&["--shared-dir", shared_dir]) 88 .args(&["--socket-path", virtiofsd_socket_path.as_str()]) 89 .args(&["--cache", "never"]) 90 .spawn() 91 .unwrap(); 92 93 thread::sleep(std::time::Duration::new(10, 0)); 94 95 (child, virtiofsd_socket_path) 96 } 97 98 fn prepare_vubd( 99 tmp_dir: &TempDir, 100 blk_img: &str, 101 num_queues: usize, 102 rdonly: bool, 103 direct: bool, 104 ) -> (std::process::Child, String) { 105 let mut workload_path = dirs::home_dir().unwrap(); 106 workload_path.push("workloads"); 107 108 let mut blk_file_path = workload_path; 109 blk_file_path.push(blk_img); 110 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 111 112 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 113 114 // Start the daemon 115 let child = Command::new(clh_command("vhost_user_block")) 116 .args(&[ 117 "--block-backend", 118 format!( 119 "path={},socket={},num_queues={},readonly={},direct={}", 120 blk_file_path, vubd_socket_path, num_queues, rdonly, direct 121 ) 122 .as_str(), 123 ]) 124 .spawn() 125 .unwrap(); 126 127 thread::sleep(std::time::Duration::new(10, 0)); 128 129 (child, vubd_socket_path) 130 } 131 132 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 133 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 134 } 135 136 fn temp_api_path(tmp_dir: &TempDir) -> String { 137 String::from( 138 tmp_dir 139 .as_path() 140 .join("cloud-hypervisor.sock") 141 .to_str() 142 .unwrap(), 143 ) 144 } 145 146 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 147 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 148 } 149 150 // Creates the directory and returns the path. 151 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 152 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 153 std::fs::create_dir(&snapshot_dir).unwrap(); 154 snapshot_dir 155 } 156 157 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 158 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 159 vmcore_file 160 } 161 162 // Creates the path for direct kernel boot and return the path. 163 // For x86_64, this function returns the vmlinux kernel path. 164 // For AArch64, this function returns the PE kernel path. 165 fn direct_kernel_boot_path() -> PathBuf { 166 let mut workload_path = dirs::home_dir().unwrap(); 167 workload_path.push("workloads"); 168 169 let mut kernel_path = workload_path; 170 #[cfg(target_arch = "x86_64")] 171 kernel_path.push("vmlinux"); 172 #[cfg(target_arch = "aarch64")] 173 kernel_path.push("Image"); 174 175 kernel_path 176 } 177 178 #[cfg(target_arch = "aarch64")] 179 fn edk2_path() -> PathBuf { 180 let mut workload_path = dirs::home_dir().unwrap(); 181 workload_path.push("workloads"); 182 let mut edk2_path = workload_path; 183 edk2_path.push("CLOUDHV_EFI.fd"); 184 185 edk2_path 186 } 187 188 fn cloud_hypervisor_release_path() -> String { 189 let mut workload_path = dirs::home_dir().unwrap(); 190 workload_path.push("workloads"); 191 192 let mut ch_release_path = workload_path; 193 #[cfg(target_arch = "x86_64")] 194 ch_release_path.push("cloud-hypervisor-static"); 195 #[cfg(target_arch = "aarch64")] 196 ch_release_path.push("cloud-hypervisor-static-aarch64"); 197 198 ch_release_path.into_os_string().into_string().unwrap() 199 } 200 201 fn prepare_vhost_user_net_daemon( 202 tmp_dir: &TempDir, 203 ip: &str, 204 tap: Option<&str>, 205 num_queues: usize, 206 client_mode: bool, 207 ) -> (std::process::Command, String) { 208 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 209 210 // Start the daemon 211 let net_params = if let Some(tap_str) = tap { 212 format!( 213 "tap={},ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 214 tap_str, ip, vunet_socket_path, num_queues, client_mode 215 ) 216 } else { 217 format!( 218 "ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 219 ip, vunet_socket_path, num_queues, client_mode 220 ) 221 }; 222 223 let mut command = Command::new(clh_command("vhost_user_net")); 224 command.args(&["--net-backend", net_params.as_str()]); 225 226 (command, vunet_socket_path) 227 } 228 229 fn curl_command(api_socket: &str, method: &str, url: &str, http_body: Option<&str>) { 230 let mut curl_args: Vec<&str> = ["--unix-socket", api_socket, "-i", "-X", method, url].to_vec(); 231 232 if let Some(body) = http_body { 233 curl_args.push("-H"); 234 curl_args.push("Accept: application/json"); 235 curl_args.push("-H"); 236 curl_args.push("Content-Type: application/json"); 237 curl_args.push("-d"); 238 curl_args.push(body); 239 } 240 241 let status = Command::new("curl") 242 .args(curl_args) 243 .status() 244 .expect("Failed to launch curl command"); 245 246 assert!(status.success()); 247 } 248 249 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 250 let mut cmd = Command::new(clh_command("ch-remote")); 251 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 252 253 if let Some(arg) = arg { 254 cmd.arg(arg); 255 } 256 let output = cmd.output().unwrap(); 257 if output.status.success() { 258 true 259 } else { 260 eprintln!("Error running ch-remote command: {:?}", &cmd); 261 let stderr = String::from_utf8_lossy(&output.stderr); 262 eprintln!("stderr: {}", stderr); 263 false 264 } 265 } 266 267 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 268 let mut cmd = Command::new(clh_command("ch-remote")); 269 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 270 271 if let Some(arg) = arg { 272 cmd.arg(arg); 273 } 274 275 let output = cmd.output().expect("Failed to launch ch-remote"); 276 277 (output.status.success(), output.stdout) 278 } 279 280 fn resize_command( 281 api_socket: &str, 282 desired_vcpus: Option<u8>, 283 desired_ram: Option<usize>, 284 desired_balloon: Option<usize>, 285 event_file: Option<&str>, 286 ) -> bool { 287 let mut cmd = Command::new(clh_command("ch-remote")); 288 cmd.args(&[&format!("--api-socket={}", api_socket), "resize"]); 289 290 if let Some(desired_vcpus) = desired_vcpus { 291 cmd.arg(format!("--cpus={}", desired_vcpus)); 292 } 293 294 if let Some(desired_ram) = desired_ram { 295 cmd.arg(format!("--memory={}", desired_ram)); 296 } 297 298 if let Some(desired_balloon) = desired_balloon { 299 cmd.arg(format!("--balloon={}", desired_balloon)); 300 } 301 302 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 303 304 if let Some(event_path) = event_file { 305 let latest_events = [ 306 &MetaEvent { 307 event: "resizing".to_string(), 308 device_id: None, 309 }, 310 &MetaEvent { 311 event: "resized".to_string(), 312 device_id: None, 313 }, 314 ]; 315 assert!(check_latest_events_exact(&latest_events, event_path)); 316 } 317 318 ret 319 } 320 321 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 322 let mut cmd = Command::new(clh_command("ch-remote")); 323 cmd.args(&[ 324 &format!("--api-socket={}", api_socket), 325 "resize-zone", 326 &format!("--id={}", id), 327 &format!("--size={}", desired_size), 328 ]); 329 330 cmd.status().expect("Failed to launch ch-remote").success() 331 } 332 333 // setup OVS-DPDK bridge and ports 334 fn setup_ovs_dpdk() { 335 // setup OVS-DPDK 336 assert!(exec_host_command_status("service openvswitch-switch start").success()); 337 assert!(exec_host_command_status("ovs-vsctl init").success()); 338 assert!( 339 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 340 .success() 341 ); 342 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 343 344 // Create OVS-DPDK bridge and ports 345 assert!(exec_host_command_status( 346 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 347 ) 348 .success()); 349 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 350 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 351 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 352 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 353 } 354 fn cleanup_ovs_dpdk() { 355 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 356 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 357 } 358 // Setup two guests and ensure they are connected through ovs-dpdk 359 fn setup_ovs_dpdk_guests( 360 guest1: &Guest, 361 guest2: &Guest, 362 api_socket: &str, 363 release_binary: bool, 364 ) -> (Child, Child) { 365 setup_ovs_dpdk(); 366 367 let clh_path = if !release_binary { 368 clh_command("cloud-hypervisor") 369 } else { 370 cloud_hypervisor_release_path() 371 }; 372 373 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 374 .args(&["--cpus", "boot=2"]) 375 .args(&["--memory", "size=0,shared=on"]) 376 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 377 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 378 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 379 .default_disks() 380 .args(&["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 381 .capture_output() 382 .spawn() 383 .unwrap(); 384 385 #[cfg(target_arch = "x86_64")] 386 let guest_net_iface = "ens5"; 387 #[cfg(target_arch = "aarch64")] 388 let guest_net_iface = "enp0s5"; 389 390 let r = std::panic::catch_unwind(|| { 391 guest1.wait_vm_boot(None).unwrap(); 392 393 guest1 394 .ssh_command(&format!( 395 "sudo ip addr add 172.100.0.1/24 dev {}", 396 guest_net_iface 397 )) 398 .unwrap(); 399 guest1 400 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 401 .unwrap(); 402 403 let guest_ip = guest1.network.guest_ip.clone(); 404 thread::spawn(move || { 405 ssh_command_ip( 406 "nc -l 12345", 407 &guest_ip, 408 DEFAULT_SSH_RETRIES, 409 DEFAULT_SSH_TIMEOUT, 410 ) 411 .unwrap(); 412 }); 413 }); 414 if r.is_err() { 415 cleanup_ovs_dpdk(); 416 417 let _ = child1.kill(); 418 let output = child1.wait_with_output().unwrap(); 419 handle_child_output(r, &output); 420 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 421 } 422 423 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 424 .args(&["--api-socket", api_socket]) 425 .args(&["--cpus", "boot=2"]) 426 .args(&["--memory", "size=0,shared=on"]) 427 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 428 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 429 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 430 .default_disks() 431 .args(&["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 432 .capture_output() 433 .spawn() 434 .unwrap(); 435 436 let r = std::panic::catch_unwind(|| { 437 guest2.wait_vm_boot(None).unwrap(); 438 439 guest2 440 .ssh_command(&format!( 441 "sudo ip addr add 172.100.0.2/24 dev {}", 442 guest_net_iface 443 )) 444 .unwrap(); 445 guest2 446 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 447 .unwrap(); 448 449 // Check the connection works properly between the two VMs 450 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 451 }); 452 if r.is_err() { 453 cleanup_ovs_dpdk(); 454 455 let _ = child1.kill(); 456 let _ = child2.kill(); 457 let output = child2.wait_with_output().unwrap(); 458 handle_child_output(r, &output); 459 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 460 } 461 462 (child1, child2) 463 } 464 465 enum FwType { 466 Ovmf, 467 RustHypervisorFirmware, 468 } 469 470 fn fw_path(_fw_type: FwType) -> String { 471 let mut workload_path = dirs::home_dir().unwrap(); 472 workload_path.push("workloads"); 473 474 let mut fw_path = workload_path; 475 #[cfg(target_arch = "aarch64")] 476 fw_path.push("CLOUDHV_EFI.fd"); 477 #[cfg(target_arch = "x86_64")] 478 { 479 match _fw_type { 480 FwType::Ovmf => fw_path.push(OVMF_NAME), 481 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 482 } 483 } 484 485 fw_path.to_str().unwrap().to_string() 486 } 487 488 struct MetaEvent { 489 event: String, 490 device_id: Option<String>, 491 } 492 493 impl MetaEvent { 494 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 495 let mut matched = false; 496 if v["event"].as_str().unwrap() == self.event { 497 if let Some(device_id) = &self.device_id { 498 if v["properties"]["id"].as_str().unwrap() == device_id { 499 matched = true 500 } 501 } else { 502 matched = true; 503 } 504 } 505 matched 506 } 507 } 508 509 // Parse the event_monitor file based on the format that each event 510 // is followed by a double newline 511 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 512 let content = fs::read(event_file).unwrap(); 513 let mut ret = Vec::new(); 514 for entry in String::from_utf8_lossy(&content) 515 .trim() 516 .split("\n\n") 517 .collect::<Vec<&str>>() 518 { 519 ret.push(serde_json::from_str(entry).unwrap()); 520 } 521 522 ret 523 } 524 525 // Return true if all events from the input 'expected_events' are matched sequentially 526 // with events from the 'event_file' 527 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 528 let json_events = parse_event_file(event_file); 529 let len = expected_events.len(); 530 let mut idx = 0; 531 for e in &json_events { 532 if idx == len { 533 break; 534 } 535 if expected_events[idx].match_with_json_event(e) { 536 idx += 1; 537 } 538 } 539 540 idx == len 541 } 542 543 // Return true if all events from the input 'expected_events' are matched exactly 544 // with events from the 'event_file' 545 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 546 let json_events = parse_event_file(event_file); 547 assert!(expected_events.len() <= json_events.len()); 548 let json_events = &json_events[..expected_events.len()]; 549 550 for (idx, e) in json_events.iter().enumerate() { 551 if !expected_events[idx].match_with_json_event(e) { 552 return false; 553 } 554 } 555 556 true 557 } 558 559 // Return true if events from the input 'expected_events' are matched exactly 560 // with the most recent events from the 'event_file' 561 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 562 let json_events = parse_event_file(event_file); 563 assert!(latest_events.len() <= json_events.len()); 564 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 565 566 for (idx, e) in json_events.iter().enumerate() { 567 if !latest_events[idx].match_with_json_event(e) { 568 return false; 569 } 570 } 571 572 true 573 } 574 575 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 576 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 577 let guest = Guest::new(Box::new(focal)); 578 let total_vcpus = threads_per_core * cores_per_package * packages; 579 let direct_kernel_boot_path = direct_kernel_boot_path(); 580 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 581 let fw_path = fw_path(FwType::RustHypervisorFirmware); 582 if use_fw { 583 kernel_path = fw_path.as_str(); 584 } 585 586 let mut child = GuestCommand::new(&guest) 587 .args(&[ 588 "--cpus", 589 &format!( 590 "boot={},topology={}:{}:1:{}", 591 total_vcpus, threads_per_core, cores_per_package, packages 592 ), 593 ]) 594 .args(&["--memory", "size=512M"]) 595 .args(&["--kernel", kernel_path]) 596 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 597 .default_disks() 598 .default_net() 599 .capture_output() 600 .spawn() 601 .unwrap(); 602 603 let r = std::panic::catch_unwind(|| { 604 guest.wait_vm_boot(None).unwrap(); 605 assert_eq!( 606 guest.get_cpu_count().unwrap_or_default(), 607 u32::from(total_vcpus) 608 ); 609 assert_eq!( 610 guest 611 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 612 .unwrap() 613 .trim() 614 .parse::<u8>() 615 .unwrap_or(0), 616 threads_per_core 617 ); 618 619 assert_eq!( 620 guest 621 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 622 .unwrap() 623 .trim() 624 .parse::<u8>() 625 .unwrap_or(0), 626 cores_per_package 627 ); 628 629 assert_eq!( 630 guest 631 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 632 .unwrap() 633 .trim() 634 .parse::<u8>() 635 .unwrap_or(0), 636 packages 637 ); 638 }); 639 640 let _ = child.kill(); 641 let output = child.wait_with_output().unwrap(); 642 643 handle_child_output(r, &output); 644 } 645 646 #[allow(unused_variables)] 647 fn _test_guest_numa_nodes(acpi: bool) { 648 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 649 let guest = Guest::new(Box::new(focal)); 650 let api_socket = temp_api_path(&guest.tmp_dir); 651 #[cfg(target_arch = "x86_64")] 652 let kernel_path = direct_kernel_boot_path(); 653 #[cfg(target_arch = "aarch64")] 654 let kernel_path = if acpi { 655 edk2_path() 656 } else { 657 direct_kernel_boot_path() 658 }; 659 660 let mut child = GuestCommand::new(&guest) 661 .args(&["--cpus", "boot=6,max=12"]) 662 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 663 .args(&[ 664 "--memory-zone", 665 "id=mem0,size=1G,hotplug_size=3G", 666 "id=mem1,size=2G,hotplug_size=3G", 667 "id=mem2,size=3G,hotplug_size=3G", 668 ]) 669 .args(&[ 670 "--numa", 671 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 672 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 673 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 674 ]) 675 .args(&["--kernel", kernel_path.to_str().unwrap()]) 676 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 677 .args(&["--api-socket", &api_socket]) 678 .capture_output() 679 .default_disks() 680 .default_net() 681 .spawn() 682 .unwrap(); 683 684 let r = std::panic::catch_unwind(|| { 685 guest.wait_vm_boot(None).unwrap(); 686 687 guest.check_numa_common( 688 Some(&[960_000, 1_920_000, 2_880_000]), 689 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 690 Some(&["10 15 20", "20 10 25", "25 30 10"]), 691 ); 692 693 // AArch64 currently does not support hotplug, and therefore we only 694 // test hotplug-related function on x86_64 here. 695 #[cfg(target_arch = "x86_64")] 696 { 697 guest.enable_memory_hotplug(); 698 699 // Resize every memory zone and check each associated NUMA node 700 // has been assigned the right amount of memory. 701 resize_zone_command(&api_socket, "mem0", "4G"); 702 resize_zone_command(&api_socket, "mem1", "4G"); 703 resize_zone_command(&api_socket, "mem2", "4G"); 704 // Resize to the maximum amount of CPUs and check each NUMA 705 // node has been assigned the right CPUs set. 706 resize_command(&api_socket, Some(12), None, None, None); 707 thread::sleep(std::time::Duration::new(5, 0)); 708 709 guest.check_numa_common( 710 Some(&[3_840_000, 3_840_000, 3_840_000]), 711 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 712 None, 713 ); 714 } 715 }); 716 717 let _ = child.kill(); 718 let output = child.wait_with_output().unwrap(); 719 720 handle_child_output(r, &output); 721 } 722 723 #[allow(unused_variables)] 724 fn _test_power_button(acpi: bool) { 725 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 726 let guest = Guest::new(Box::new(focal)); 727 let mut cmd = GuestCommand::new(&guest); 728 let api_socket = temp_api_path(&guest.tmp_dir); 729 730 #[cfg(target_arch = "x86_64")] 731 let kernel_path = direct_kernel_boot_path(); 732 #[cfg(target_arch = "aarch64")] 733 let kernel_path = if acpi { 734 edk2_path() 735 } else { 736 direct_kernel_boot_path() 737 }; 738 739 cmd.args(&["--cpus", "boot=1"]) 740 .args(&["--memory", "size=512M"]) 741 .args(&["--kernel", kernel_path.to_str().unwrap()]) 742 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 743 .capture_output() 744 .default_disks() 745 .default_net() 746 .args(&["--api-socket", &api_socket]); 747 748 let child = cmd.spawn().unwrap(); 749 750 let r = std::panic::catch_unwind(|| { 751 guest.wait_vm_boot(None).unwrap(); 752 assert!(remote_command(&api_socket, "power-button", None)); 753 }); 754 755 let output = child.wait_with_output().unwrap(); 756 assert!(output.status.success()); 757 handle_child_output(r, &output); 758 } 759 760 type PrepareNetDaemon = 761 dyn Fn(&TempDir, &str, Option<&str>, usize, bool) -> (std::process::Command, String); 762 763 fn test_vhost_user_net( 764 tap: Option<&str>, 765 num_queues: usize, 766 prepare_daemon: &PrepareNetDaemon, 767 generate_host_mac: bool, 768 client_mode_daemon: bool, 769 ) { 770 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 771 let guest = Guest::new(Box::new(focal)); 772 let api_socket = temp_api_path(&guest.tmp_dir); 773 774 let kernel_path = direct_kernel_boot_path(); 775 776 let host_mac = if generate_host_mac { 777 Some(MacAddr::local_random()) 778 } else { 779 None 780 }; 781 782 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 783 &guest.tmp_dir, 784 &guest.network.host_ip, 785 tap, 786 num_queues, 787 client_mode_daemon, 788 ); 789 790 let net_params = format!( 791 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={}", 792 guest.network.guest_mac, 793 vunet_socket_path, 794 num_queues, 795 if let Some(host_mac) = host_mac { 796 format!(",host_mac={}", host_mac) 797 } else { 798 "".to_owned() 799 }, 800 if client_mode_daemon { 801 "server" 802 } else { 803 "client" 804 }, 805 ); 806 807 let mut ch_command = GuestCommand::new(&guest); 808 ch_command 809 .args(&["--cpus", format!("boot={}", num_queues / 2).as_str()]) 810 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 811 .args(&["--kernel", kernel_path.to_str().unwrap()]) 812 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 813 .default_disks() 814 .args(&["--net", net_params.as_str()]) 815 .args(&["--api-socket", &api_socket]) 816 .capture_output(); 817 818 let mut daemon_child: std::process::Child; 819 let mut child: std::process::Child; 820 821 if client_mode_daemon { 822 child = ch_command.spawn().unwrap(); 823 // Make sure the VMM is waiting for the backend to connect 824 thread::sleep(std::time::Duration::new(10, 0)); 825 daemon_child = daemon_command.spawn().unwrap(); 826 } else { 827 daemon_child = daemon_command.spawn().unwrap(); 828 // Make sure the backend is waiting for the VMM to connect 829 thread::sleep(std::time::Duration::new(10, 0)); 830 child = ch_command.spawn().unwrap(); 831 } 832 833 let r = std::panic::catch_unwind(|| { 834 guest.wait_vm_boot(None).unwrap(); 835 836 if let Some(tap_name) = tap { 837 let tap_count = exec_host_command_output(&format!("ip link | grep -c {}", tap_name)); 838 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 839 } 840 841 if let Some(host_mac) = tap { 842 let mac_count = exec_host_command_output(&format!("ip link | grep -c {}", host_mac)); 843 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 844 } 845 846 // 1 network interface + default localhost ==> 2 interfaces 847 // It's important to note that this test is fully exercising the 848 // vhost-user-net implementation and the associated backend since 849 // it does not define any --net network interface. That means all 850 // the ssh communication in that test happens through the network 851 // interface backed by vhost-user-net. 852 assert_eq!( 853 guest 854 .ssh_command("ip -o link | wc -l") 855 .unwrap() 856 .trim() 857 .parse::<u32>() 858 .unwrap_or_default(), 859 2 860 ); 861 862 // The following pci devices will appear on guest with PCI-MSI 863 // interrupt vectors assigned. 864 // 1 virtio-console with 3 vectors: config, Rx, Tx 865 // 1 virtio-blk with 2 vectors: config, Request 866 // 1 virtio-blk with 2 vectors: config, Request 867 // 1 virtio-rng with 2 vectors: config, Request 868 // Since virtio-net has 2 queue pairs, its vectors is as follows: 869 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 870 // Based on the above, the total vectors should 14. 871 #[cfg(target_arch = "x86_64")] 872 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 873 #[cfg(target_arch = "aarch64")] 874 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 875 assert_eq!( 876 guest 877 .ssh_command(grep_cmd) 878 .unwrap() 879 .trim() 880 .parse::<u32>() 881 .unwrap_or_default(), 882 10 + (num_queues as u32) 883 ); 884 885 // ACPI feature is needed. 886 #[cfg(target_arch = "x86_64")] 887 { 888 guest.enable_memory_hotplug(); 889 890 // Add RAM to the VM 891 let desired_ram = 1024 << 20; 892 resize_command(&api_socket, None, Some(desired_ram), None, None); 893 894 thread::sleep(std::time::Duration::new(10, 0)); 895 896 // Here by simply checking the size (through ssh), we validate 897 // the connection is still working, which means vhost-user-net 898 // keeps working after the resize. 899 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 900 } 901 }); 902 903 let _ = child.kill(); 904 let output = child.wait_with_output().unwrap(); 905 906 thread::sleep(std::time::Duration::new(5, 0)); 907 let _ = daemon_child.kill(); 908 let _ = daemon_child.wait(); 909 910 handle_child_output(r, &output); 911 } 912 913 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 914 915 fn test_vhost_user_blk( 916 num_queues: usize, 917 readonly: bool, 918 direct: bool, 919 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 920 ) { 921 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 922 let guest = Guest::new(Box::new(focal)); 923 let api_socket = temp_api_path(&guest.tmp_dir); 924 925 let kernel_path = direct_kernel_boot_path(); 926 927 let (blk_params, daemon_child) = { 928 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 929 // Start the daemon 930 let (daemon_child, vubd_socket_path) = 931 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 932 933 ( 934 format!( 935 "vhost_user=true,socket={},num_queues={},queue_size=128", 936 vubd_socket_path, num_queues, 937 ), 938 Some(daemon_child), 939 ) 940 }; 941 942 let mut child = GuestCommand::new(&guest) 943 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 944 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 945 .args(&["--kernel", kernel_path.to_str().unwrap()]) 946 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 947 .args(&[ 948 "--disk", 949 format!( 950 "path={}", 951 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 952 ) 953 .as_str(), 954 format!( 955 "path={}", 956 guest.disk_config.disk(DiskType::CloudInit).unwrap() 957 ) 958 .as_str(), 959 blk_params.as_str(), 960 ]) 961 .default_net() 962 .args(&["--api-socket", &api_socket]) 963 .capture_output() 964 .spawn() 965 .unwrap(); 966 967 let r = std::panic::catch_unwind(|| { 968 guest.wait_vm_boot(None).unwrap(); 969 970 // Check both if /dev/vdc exists and if the block size is 16M. 971 assert_eq!( 972 guest 973 .ssh_command("lsblk | grep vdc | grep -c 16M") 974 .unwrap() 975 .trim() 976 .parse::<u32>() 977 .unwrap_or_default(), 978 1 979 ); 980 981 // Check if this block is RO or RW. 982 assert_eq!( 983 guest 984 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 985 .unwrap() 986 .trim() 987 .parse::<u32>() 988 .unwrap_or_default(), 989 readonly as u32 990 ); 991 992 // Check if the number of queues in /sys/block/vdc/mq matches the 993 // expected num_queues. 994 assert_eq!( 995 guest 996 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 997 .unwrap() 998 .trim() 999 .parse::<u32>() 1000 .unwrap_or_default(), 1001 num_queues as u32 1002 ); 1003 1004 // Mount the device 1005 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1006 guest.ssh_command("mkdir mount_image").unwrap(); 1007 guest 1008 .ssh_command( 1009 format!( 1010 "sudo mount -o {} -t ext4 /dev/vdc mount_image/", 1011 mount_ro_rw_flag 1012 ) 1013 .as_str(), 1014 ) 1015 .unwrap(); 1016 1017 // Check the content of the block device. The file "foo" should 1018 // contain "bar". 1019 assert_eq!( 1020 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1021 "bar" 1022 ); 1023 1024 // ACPI feature is needed. 1025 #[cfg(target_arch = "x86_64")] 1026 { 1027 guest.enable_memory_hotplug(); 1028 1029 // Add RAM to the VM 1030 let desired_ram = 1024 << 20; 1031 resize_command(&api_socket, None, Some(desired_ram), None, None); 1032 1033 thread::sleep(std::time::Duration::new(10, 0)); 1034 1035 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1036 1037 // Check again the content of the block device after the resize 1038 // has been performed. 1039 assert_eq!( 1040 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1041 "bar" 1042 ); 1043 } 1044 1045 // Unmount the device 1046 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1047 guest.ssh_command("rm -r mount_image").unwrap(); 1048 }); 1049 1050 let _ = child.kill(); 1051 let output = child.wait_with_output().unwrap(); 1052 1053 if let Some(mut daemon_child) = daemon_child { 1054 thread::sleep(std::time::Duration::new(5, 0)); 1055 let _ = daemon_child.kill(); 1056 let _ = daemon_child.wait(); 1057 } 1058 1059 handle_child_output(r, &output); 1060 } 1061 1062 fn test_boot_from_vhost_user_blk( 1063 num_queues: usize, 1064 readonly: bool, 1065 direct: bool, 1066 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1067 ) { 1068 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1069 let guest = Guest::new(Box::new(focal)); 1070 1071 let kernel_path = direct_kernel_boot_path(); 1072 1073 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1074 1075 let (blk_boot_params, daemon_child) = { 1076 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1077 // Start the daemon 1078 let (daemon_child, vubd_socket_path) = prepare_daemon( 1079 &guest.tmp_dir, 1080 disk_path.as_str(), 1081 num_queues, 1082 readonly, 1083 direct, 1084 ); 1085 1086 ( 1087 format!( 1088 "vhost_user=true,socket={},num_queues={},queue_size=128", 1089 vubd_socket_path, num_queues, 1090 ), 1091 Some(daemon_child), 1092 ) 1093 }; 1094 1095 let mut child = GuestCommand::new(&guest) 1096 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 1097 .args(&["--memory", "size=512M,shared=on"]) 1098 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1099 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1100 .args(&[ 1101 "--disk", 1102 blk_boot_params.as_str(), 1103 format!( 1104 "path={}", 1105 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1106 ) 1107 .as_str(), 1108 ]) 1109 .default_net() 1110 .capture_output() 1111 .spawn() 1112 .unwrap(); 1113 1114 let r = std::panic::catch_unwind(|| { 1115 guest.wait_vm_boot(None).unwrap(); 1116 1117 // Just check the VM booted correctly. 1118 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1119 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1120 }); 1121 let _ = child.kill(); 1122 let output = child.wait_with_output().unwrap(); 1123 1124 if let Some(mut daemon_child) = daemon_child { 1125 thread::sleep(std::time::Duration::new(5, 0)); 1126 let _ = daemon_child.kill(); 1127 let _ = daemon_child.wait(); 1128 } 1129 1130 handle_child_output(r, &output); 1131 } 1132 1133 fn _test_virtio_fs( 1134 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1135 hotplug: bool, 1136 pci_segment: Option<u16>, 1137 ) { 1138 #[cfg(target_arch = "aarch64")] 1139 let focal_image = if hotplug { 1140 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1141 } else { 1142 FOCAL_IMAGE_NAME.to_string() 1143 }; 1144 #[cfg(target_arch = "x86_64")] 1145 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1146 let focal = UbuntuDiskConfig::new(focal_image); 1147 let guest = Guest::new(Box::new(focal)); 1148 let api_socket = temp_api_path(&guest.tmp_dir); 1149 1150 let mut workload_path = dirs::home_dir().unwrap(); 1151 workload_path.push("workloads"); 1152 1153 let mut shared_dir = workload_path; 1154 shared_dir.push("shared_dir"); 1155 1156 #[cfg(target_arch = "x86_64")] 1157 let kernel_path = direct_kernel_boot_path(); 1158 #[cfg(target_arch = "aarch64")] 1159 let kernel_path = if hotplug { 1160 edk2_path() 1161 } else { 1162 direct_kernel_boot_path() 1163 }; 1164 1165 let (mut daemon_child, virtiofsd_socket_path) = 1166 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1167 1168 let mut guest_command = GuestCommand::new(&guest); 1169 guest_command 1170 .args(&["--cpus", "boot=1"]) 1171 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1172 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1173 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1174 .default_disks() 1175 .default_net() 1176 .args(&["--api-socket", &api_socket]); 1177 if pci_segment.is_some() { 1178 guest_command.args(&["--platform", "num_pci_segments=16"]); 1179 } 1180 1181 let fs_params = format!( 1182 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1183 virtiofsd_socket_path, 1184 if let Some(pci_segment) = pci_segment { 1185 format!(",pci_segment={}", pci_segment) 1186 } else { 1187 "".to_owned() 1188 } 1189 ); 1190 1191 if !hotplug { 1192 guest_command.args(&["--fs", fs_params.as_str()]); 1193 } 1194 1195 let mut child = guest_command.capture_output().spawn().unwrap(); 1196 1197 let r = std::panic::catch_unwind(|| { 1198 guest.wait_vm_boot(None).unwrap(); 1199 1200 if hotplug { 1201 // Add fs to the VM 1202 let (cmd_success, cmd_output) = 1203 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1204 assert!(cmd_success); 1205 1206 if let Some(pci_segment) = pci_segment { 1207 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1208 "{{\"id\":\"myfs0\",\"bdf\":\"{:04x}:00:01.0\"}}", 1209 pci_segment 1210 ))); 1211 } else { 1212 assert!(String::from_utf8_lossy(&cmd_output) 1213 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1214 } 1215 1216 thread::sleep(std::time::Duration::new(10, 0)); 1217 } 1218 1219 // Mount shared directory through virtio_fs filesystem 1220 guest 1221 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1222 .unwrap(); 1223 1224 // Check file1 exists and its content is "foo" 1225 assert_eq!( 1226 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1227 "foo" 1228 ); 1229 // Check file2 does not exist 1230 guest 1231 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1232 .unwrap(); 1233 1234 // Check file3 exists and its content is "bar" 1235 assert_eq!( 1236 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1237 "bar" 1238 ); 1239 1240 // ACPI feature is needed. 1241 #[cfg(target_arch = "x86_64")] 1242 { 1243 guest.enable_memory_hotplug(); 1244 1245 // Add RAM to the VM 1246 let desired_ram = 1024 << 20; 1247 resize_command(&api_socket, None, Some(desired_ram), None, None); 1248 1249 thread::sleep(std::time::Duration::new(30, 0)); 1250 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1251 1252 // After the resize, check again that file1 exists and its 1253 // content is "foo". 1254 assert_eq!( 1255 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1256 "foo" 1257 ); 1258 } 1259 1260 if hotplug { 1261 // Remove from VM 1262 guest.ssh_command("sudo umount mount_dir").unwrap(); 1263 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1264 } 1265 }); 1266 1267 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1268 thread::sleep(std::time::Duration::new(10, 0)); 1269 let (daemon_child, virtiofsd_socket_path) = 1270 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1271 1272 let r = std::panic::catch_unwind(|| { 1273 thread::sleep(std::time::Duration::new(10, 0)); 1274 let fs_params = format!( 1275 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1276 virtiofsd_socket_path, 1277 if let Some(pci_segment) = pci_segment { 1278 format!(",pci_segment={}", pci_segment) 1279 } else { 1280 "".to_owned() 1281 } 1282 ); 1283 1284 // Add back and check it works 1285 let (cmd_success, cmd_output) = 1286 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1287 assert!(cmd_success); 1288 if let Some(pci_segment) = pci_segment { 1289 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1290 "{{\"id\":\"myfs0\",\"bdf\":\"{:04x}:00:01.0\"}}", 1291 pci_segment 1292 ))); 1293 } else { 1294 assert!(String::from_utf8_lossy(&cmd_output) 1295 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1296 } 1297 1298 thread::sleep(std::time::Duration::new(10, 0)); 1299 // Mount shared directory through virtio_fs filesystem 1300 guest 1301 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1302 .unwrap(); 1303 1304 // Check file1 exists and its content is "foo" 1305 assert_eq!( 1306 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1307 "foo" 1308 ); 1309 }); 1310 1311 (r, Some(daemon_child)) 1312 } else { 1313 (r, None) 1314 }; 1315 1316 let _ = child.kill(); 1317 let output = child.wait_with_output().unwrap(); 1318 1319 let _ = daemon_child.kill(); 1320 let _ = daemon_child.wait(); 1321 1322 if let Some(mut daemon_child) = hotplug_daemon_child { 1323 let _ = daemon_child.kill(); 1324 let _ = daemon_child.wait(); 1325 } 1326 1327 handle_child_output(r, &output); 1328 } 1329 1330 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1331 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1332 let guest = Guest::new(Box::new(focal)); 1333 1334 let kernel_path = direct_kernel_boot_path(); 1335 1336 let pmem_temp_file = TempFile::new().unwrap(); 1337 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1338 1339 std::process::Command::new("mkfs.ext4") 1340 .arg(pmem_temp_file.as_path()) 1341 .output() 1342 .expect("Expect creating disk image to succeed"); 1343 1344 let mut child = GuestCommand::new(&guest) 1345 .args(&["--cpus", "boot=1"]) 1346 .args(&["--memory", "size=512M"]) 1347 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1348 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1349 .default_disks() 1350 .default_net() 1351 .args(&[ 1352 "--pmem", 1353 format!( 1354 "file={}{}{}", 1355 pmem_temp_file.as_path().to_str().unwrap(), 1356 if specify_size { ",size=128M" } else { "" }, 1357 if discard_writes { 1358 ",discard_writes=on" 1359 } else { 1360 "" 1361 } 1362 ) 1363 .as_str(), 1364 ]) 1365 .capture_output() 1366 .spawn() 1367 .unwrap(); 1368 1369 let r = std::panic::catch_unwind(|| { 1370 guest.wait_vm_boot(None).unwrap(); 1371 1372 // Check for the presence of /dev/pmem0 1373 assert_eq!( 1374 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1375 "/dev/pmem0" 1376 ); 1377 1378 // Check changes persist after reboot 1379 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1380 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1381 guest 1382 .ssh_command("echo test123 | sudo tee /mnt/test") 1383 .unwrap(); 1384 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1385 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1386 1387 guest.reboot_linux(0, None); 1388 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1389 assert_eq!( 1390 guest 1391 .ssh_command("sudo cat /mnt/test || true") 1392 .unwrap() 1393 .trim(), 1394 if discard_writes { "" } else { "test123" } 1395 ); 1396 }); 1397 1398 let _ = child.kill(); 1399 let output = child.wait_with_output().unwrap(); 1400 1401 handle_child_output(r, &output); 1402 } 1403 1404 fn get_fd_count(pid: u32) -> usize { 1405 fs::read_dir(format!("/proc/{}/fd", pid)).unwrap().count() 1406 } 1407 1408 fn _test_virtio_vsock(hotplug: bool) { 1409 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1410 let guest = Guest::new(Box::new(focal)); 1411 1412 #[cfg(target_arch = "x86_64")] 1413 let kernel_path = direct_kernel_boot_path(); 1414 #[cfg(target_arch = "aarch64")] 1415 let kernel_path = if hotplug { 1416 edk2_path() 1417 } else { 1418 direct_kernel_boot_path() 1419 }; 1420 1421 let socket = temp_vsock_path(&guest.tmp_dir); 1422 let api_socket = temp_api_path(&guest.tmp_dir); 1423 1424 let mut cmd = GuestCommand::new(&guest); 1425 cmd.args(&["--api-socket", &api_socket]); 1426 cmd.args(&["--cpus", "boot=1"]); 1427 cmd.args(&["--memory", "size=512M"]); 1428 cmd.args(&["--kernel", kernel_path.to_str().unwrap()]); 1429 cmd.args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1430 cmd.default_disks(); 1431 cmd.default_net(); 1432 1433 if !hotplug { 1434 cmd.args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]); 1435 } 1436 1437 let mut child = cmd.capture_output().spawn().unwrap(); 1438 1439 let r = std::panic::catch_unwind(|| { 1440 guest.wait_vm_boot(None).unwrap(); 1441 1442 if hotplug { 1443 let (cmd_success, cmd_output) = remote_command_w_output( 1444 &api_socket, 1445 "add-vsock", 1446 Some(format!("cid=3,socket={},id=test0", socket).as_str()), 1447 ); 1448 assert!(cmd_success); 1449 assert!(String::from_utf8_lossy(&cmd_output) 1450 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1451 thread::sleep(std::time::Duration::new(10, 0)); 1452 // Check adding a second one fails 1453 assert!(!remote_command( 1454 &api_socket, 1455 "add-vsock", 1456 Some("cid=1234,socket=/tmp/fail") 1457 )); 1458 } 1459 1460 // Validate vsock works as expected. 1461 guest.check_vsock(socket.as_str()); 1462 guest.reboot_linux(0, None); 1463 // Validate vsock still works after a reboot. 1464 guest.check_vsock(socket.as_str()); 1465 1466 if hotplug { 1467 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1468 } 1469 }); 1470 1471 let _ = child.kill(); 1472 let output = child.wait_with_output().unwrap(); 1473 1474 handle_child_output(r, &output); 1475 } 1476 1477 fn get_ksm_pages_shared() -> u32 { 1478 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1479 .unwrap() 1480 .trim() 1481 .parse::<u32>() 1482 .unwrap() 1483 } 1484 1485 fn test_memory_mergeable(mergeable: bool) { 1486 let memory_param = if mergeable { 1487 "mergeable=on" 1488 } else { 1489 "mergeable=off" 1490 }; 1491 1492 // We are assuming the rest of the system in our CI is not using mergeable memeory 1493 let ksm_ps_init = get_ksm_pages_shared(); 1494 assert!(ksm_ps_init == 0); 1495 1496 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1497 let guest1 = Guest::new(Box::new(focal1)); 1498 let mut child1 = GuestCommand::new(&guest1) 1499 .args(&["--cpus", "boot=1"]) 1500 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1501 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1502 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1503 .default_disks() 1504 .args(&["--net", guest1.default_net_string().as_str()]) 1505 .args(&["--serial", "tty", "--console", "off"]) 1506 .capture_output() 1507 .spawn() 1508 .unwrap(); 1509 1510 let r = std::panic::catch_unwind(|| { 1511 guest1.wait_vm_boot(None).unwrap(); 1512 }); 1513 if r.is_err() { 1514 let _ = child1.kill(); 1515 let output = child1.wait_with_output().unwrap(); 1516 handle_child_output(r, &output); 1517 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1518 } 1519 1520 let ksm_ps_guest1 = get_ksm_pages_shared(); 1521 1522 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1523 let guest2 = Guest::new(Box::new(focal2)); 1524 let mut child2 = GuestCommand::new(&guest2) 1525 .args(&["--cpus", "boot=1"]) 1526 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1527 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1528 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1529 .default_disks() 1530 .args(&["--net", guest2.default_net_string().as_str()]) 1531 .args(&["--serial", "tty", "--console", "off"]) 1532 .capture_output() 1533 .spawn() 1534 .unwrap(); 1535 1536 let r = std::panic::catch_unwind(|| { 1537 guest2.wait_vm_boot(None).unwrap(); 1538 let ksm_ps_guest2 = get_ksm_pages_shared(); 1539 1540 if mergeable { 1541 println!( 1542 "ksm pages_shared after vm1 booted '{}', ksm pages_shared after vm2 booted '{}'", 1543 ksm_ps_guest1, ksm_ps_guest2 1544 ); 1545 // We are expecting the number of shared pages to increase as the number of VM increases 1546 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1547 } else { 1548 assert!(ksm_ps_guest1 == 0); 1549 assert!(ksm_ps_guest2 == 0); 1550 } 1551 }); 1552 1553 let _ = child1.kill(); 1554 let _ = child2.kill(); 1555 1556 let output = child1.wait_with_output().unwrap(); 1557 child2.wait().unwrap(); 1558 1559 handle_child_output(r, &output); 1560 } 1561 1562 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 1563 let smaps = fs::File::open(format!("/proc/{}/smaps", pid)).unwrap(); 1564 let reader = io::BufReader::new(smaps); 1565 1566 let mut skip_map: bool = false; 1567 let mut region_name: String = "".to_string(); 1568 let mut region_maps = HashMap::new(); 1569 for line in reader.lines() { 1570 let l = line.unwrap(); 1571 1572 if l.contains('-') { 1573 let values: Vec<&str> = l.split_whitespace().collect(); 1574 region_name = values.last().unwrap().trim().to_string(); 1575 if region_name == "0" { 1576 region_name = "anonymous".to_string() 1577 } 1578 } 1579 1580 // Each section begins with something that looks like: 1581 // Size: 2184 kB 1582 if l.starts_with("Size:") { 1583 let values: Vec<&str> = l.split_whitespace().collect(); 1584 let map_size = values[1].parse::<u32>().unwrap(); 1585 // We skip the assigned guest RAM map, its RSS is only 1586 // dependent on the guest actual memory usage. 1587 // Everything else can be added to the VMM overhead. 1588 skip_map = map_size >= guest_memory_size; 1589 continue; 1590 } 1591 1592 // If this is a map we're taking into account, then we only 1593 // count the RSS. The sum of all counted RSS is the VMM overhead. 1594 if !skip_map && l.starts_with("Rss:") { 1595 let values: Vec<&str> = l.split_whitespace().collect(); 1596 let value = values[1].trim().parse::<u32>().unwrap(); 1597 *region_maps.entry(region_name.clone()).or_insert(0) += value; 1598 } 1599 } 1600 1601 region_maps 1602 } 1603 1604 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 1605 let mut total = 0; 1606 1607 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 1608 eprintln!("{}: {}", region_name, value); 1609 total += value; 1610 } 1611 1612 total 1613 } 1614 1615 fn process_rss_kib(pid: u32) -> usize { 1616 let command = format!("ps -q {} -o rss=", pid); 1617 let rss = exec_host_command_output(&command); 1618 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 1619 } 1620 1621 // 10MB is our maximum accepted overhead. 1622 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 1623 1624 #[derive(PartialEq, Eq, PartialOrd)] 1625 struct Counters { 1626 rx_bytes: u64, 1627 rx_frames: u64, 1628 tx_bytes: u64, 1629 tx_frames: u64, 1630 read_bytes: u64, 1631 write_bytes: u64, 1632 read_ops: u64, 1633 write_ops: u64, 1634 } 1635 1636 fn get_counters(api_socket: &str) -> Counters { 1637 // Get counters 1638 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 1639 assert!(cmd_success); 1640 1641 let counters: HashMap<&str, HashMap<&str, u64>> = 1642 serde_json::from_slice(&cmd_output).unwrap_or_default(); 1643 1644 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 1645 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 1646 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 1647 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 1648 1649 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 1650 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 1651 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 1652 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 1653 1654 Counters { 1655 rx_bytes, 1656 rx_frames, 1657 tx_bytes, 1658 tx_frames, 1659 read_bytes, 1660 write_bytes, 1661 read_ops, 1662 write_ops, 1663 } 1664 } 1665 1666 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 1667 let (tx, rx) = mpsc::channel::<String>(); 1668 thread::spawn(move || loop { 1669 thread::sleep(std::time::Duration::new(1, 0)); 1670 let mut buf = [0; 512]; 1671 match pty.read(&mut buf) { 1672 Ok(_) => { 1673 let output = std::str::from_utf8(&buf).unwrap().to_string(); 1674 match tx.send(output) { 1675 Ok(_) => (), 1676 Err(_) => break, 1677 } 1678 } 1679 Err(_) => break, 1680 } 1681 }); 1682 rx 1683 } 1684 1685 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 1686 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1687 assert!(cmd_success); 1688 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1689 assert_eq!("Pty", info["config"][pty_type]["mode"]); 1690 PathBuf::from( 1691 info["config"][pty_type]["file"] 1692 .as_str() 1693 .expect("Missing pty path"), 1694 ) 1695 } 1696 1697 // VFIO test network setup. 1698 // We reserve a different IP class for it: 172.18.0.0/24. 1699 #[cfg(target_arch = "x86_64")] 1700 fn setup_vfio_network_interfaces() { 1701 // 'vfio-br0' 1702 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 1703 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 1704 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 1705 // 'vfio-tap0' 1706 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 1707 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 1708 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 1709 // 'vfio-tap1' 1710 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 1711 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 1712 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 1713 // 'vfio-tap2' 1714 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 1715 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 1716 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 1717 // 'vfio-tap3' 1718 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 1719 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 1720 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 1721 } 1722 1723 // Tear VFIO test network down 1724 #[cfg(target_arch = "x86_64")] 1725 fn cleanup_vfio_network_interfaces() { 1726 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 1727 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 1728 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 1729 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 1730 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 1731 } 1732 1733 fn balloon_size(api_socket: &str) -> u64 { 1734 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1735 assert!(cmd_success); 1736 1737 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1738 let total_mem = &info["config"]["memory"]["size"] 1739 .to_string() 1740 .parse::<u64>() 1741 .unwrap(); 1742 let actual_mem = &info["memory_actual_size"] 1743 .to_string() 1744 .parse::<u64>() 1745 .unwrap(); 1746 total_mem - actual_mem 1747 } 1748 1749 // This test validates that it can find the virtio-iommu device at first. 1750 // It also verifies that both disks and the network card are attached to 1751 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 1752 // The last interesting part of this test is that it exercises the network 1753 // interface attached to the virtual IOMMU since this is the one used to 1754 // send all commands through SSH. 1755 fn _test_virtio_iommu(acpi: bool) { 1756 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 1757 // Focal image is still old. 1758 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 1759 // the kernel binary has been updated. 1760 #[cfg(target_arch = "aarch64")] 1761 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 1762 #[cfg(target_arch = "x86_64")] 1763 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1764 let focal = UbuntuDiskConfig::new(focal_image); 1765 let guest = Guest::new(Box::new(focal)); 1766 1767 #[cfg(target_arch = "x86_64")] 1768 let kernel_path = direct_kernel_boot_path(); 1769 #[cfg(target_arch = "aarch64")] 1770 let kernel_path = if acpi { 1771 edk2_path() 1772 } else { 1773 direct_kernel_boot_path() 1774 }; 1775 1776 let mut child = GuestCommand::new(&guest) 1777 .args(&["--cpus", "boot=1"]) 1778 .args(&["--memory", "size=512M"]) 1779 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1780 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1781 .args(&[ 1782 "--disk", 1783 format!( 1784 "path={},iommu=on", 1785 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1786 ) 1787 .as_str(), 1788 format!( 1789 "path={},iommu=on", 1790 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1791 ) 1792 .as_str(), 1793 ]) 1794 .args(&["--net", guest.default_net_string_w_iommu().as_str()]) 1795 .capture_output() 1796 .spawn() 1797 .unwrap(); 1798 1799 let r = std::panic::catch_unwind(|| { 1800 guest.wait_vm_boot(None).unwrap(); 1801 1802 // Verify the virtio-iommu device is present. 1803 assert!(guest 1804 .does_device_vendor_pair_match("0x1057", "0x1af4") 1805 .unwrap_or_default()); 1806 1807 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 1808 // different with ACPI. 1809 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 1810 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 1811 // and appear under folder '/sys/kernel/iommu_groups/'. 1812 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 1813 // which is the console. The first disk "0000:00:02.0" is in group '1'. 1814 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 1815 // contains "0000:00:02.0" which is the first disk. 1816 // 1817 // Verify the iommu group of the first disk. 1818 let iommu_group = if acpi { 0 } else { 1 }; 1819 assert_eq!( 1820 guest 1821 .ssh_command( 1822 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1823 ) 1824 .unwrap() 1825 .trim(), 1826 "0000:00:02.0" 1827 ); 1828 1829 // Verify the iommu group of the second disk. 1830 let iommu_group = if acpi { 1 } else { 2 }; 1831 assert_eq!( 1832 guest 1833 .ssh_command( 1834 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1835 ) 1836 .unwrap() 1837 .trim(), 1838 "0000:00:03.0" 1839 ); 1840 1841 // Verify the iommu group of the network card. 1842 let iommu_group = if acpi { 2 } else { 3 }; 1843 assert_eq!( 1844 guest 1845 .ssh_command( 1846 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1847 ) 1848 .unwrap() 1849 .trim(), 1850 "0000:00:04.0" 1851 ); 1852 }); 1853 1854 let _ = child.kill(); 1855 let output = child.wait_with_output().unwrap(); 1856 1857 handle_child_output(r, &output); 1858 } 1859 1860 fn enable_guest_watchdog(guest: &Guest, current_reboot_count: &mut u32) { 1861 // Check for PCI device 1862 assert!(guest 1863 .does_device_vendor_pair_match("0x1063", "0x1af4") 1864 .unwrap_or_default()); 1865 1866 // Enable systemd watchdog 1867 guest 1868 .ssh_command("echo RuntimeWatchdogSec=15s | sudo tee -a /etc/systemd/system.conf") 1869 .unwrap(); 1870 1871 guest.ssh_command("sudo reboot").unwrap(); 1872 1873 guest.wait_vm_boot(None).unwrap(); 1874 1875 // Check that systemd has activated the watchdog 1876 assert_eq!( 1877 guest 1878 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 1879 .unwrap() 1880 .trim() 1881 .parse::<u32>() 1882 .unwrap_or_default(), 1883 2 1884 ); 1885 1886 // Ensure that the current boot journal is written so reboot counts are valid 1887 guest.ssh_command("sudo journalctl --sync").unwrap(); 1888 1889 *current_reboot_count += 1; 1890 1891 let boot_count = guest 1892 .ssh_command("sudo journalctl --list-boots | wc -l") 1893 .unwrap() 1894 .trim() 1895 .parse::<u32>() 1896 .unwrap_or_default(); 1897 assert_eq!(boot_count, *current_reboot_count); 1898 } 1899 1900 fn check_guest_watchdog_no_reboot(guest: &Guest, current_reboot_count: &u32) { 1901 // Allow some normal time to elapse to check we don't get spurious reboots 1902 thread::sleep(std::time::Duration::new(40, 0)); 1903 1904 // Check no reboot 1905 let boot_count = guest 1906 .ssh_command("sudo journalctl --list-boots | wc -l") 1907 .unwrap() 1908 .trim() 1909 .parse::<u32>() 1910 .unwrap_or_default(); 1911 assert_eq!(boot_count, *current_reboot_count); 1912 } 1913 1914 fn check_guest_watchdog_one_reboot( 1915 guest: &Guest, 1916 _api_socket: &str, 1917 current_reboot_count: &mut u32, 1918 ) { 1919 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 1920 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 1921 1922 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 1923 guest.wait_vm_boot(Some(50)).unwrap(); 1924 1925 // Check that watchdog triggered reboot 1926 *current_reboot_count += 1; 1927 let boot_count = guest 1928 .ssh_command("sudo journalctl --list-boots | wc -l") 1929 .unwrap() 1930 .trim() 1931 .parse::<u32>() 1932 .unwrap_or_default(); 1933 assert_eq!(boot_count, *current_reboot_count); 1934 1935 #[cfg(target_arch = "x86_64")] 1936 { 1937 // Now pause the VM and remain offline for 30s 1938 assert!(remote_command(_api_socket, "pause", None)); 1939 thread::sleep(std::time::Duration::new(30, 0)); 1940 assert!(remote_command(_api_socket, "resume", None)); 1941 1942 // Check no reboot 1943 let boot_count = guest 1944 .ssh_command("sudo journalctl --list-boots | wc -l") 1945 .unwrap() 1946 .trim() 1947 .parse::<u32>() 1948 .unwrap_or_default(); 1949 assert_eq!(boot_count, *current_reboot_count); 1950 } 1951 } 1952 1953 mod parallel { 1954 use std::io::SeekFrom; 1955 1956 use crate::*; 1957 1958 #[test] 1959 #[cfg(target_arch = "x86_64")] 1960 fn test_bionic_hypervisor_fw() { 1961 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), BIONIC_IMAGE_NAME) 1962 } 1963 1964 #[test] 1965 #[cfg(target_arch = "x86_64")] 1966 fn test_focal_hypervisor_fw() { 1967 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 1968 } 1969 1970 #[test] 1971 #[cfg(target_arch = "x86_64")] 1972 fn test_bionic_ovmf() { 1973 test_simple_launch(fw_path(FwType::Ovmf), BIONIC_IMAGE_NAME) 1974 } 1975 1976 #[test] 1977 #[cfg(target_arch = "x86_64")] 1978 fn test_focal_ovmf() { 1979 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 1980 } 1981 1982 #[cfg(target_arch = "x86_64")] 1983 fn test_simple_launch(fw_path: String, disk_path: &str) { 1984 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 1985 let guest = Guest::new(disk_config); 1986 let event_path = temp_event_monitor_path(&guest.tmp_dir); 1987 1988 let mut child = GuestCommand::new(&guest) 1989 .args(&["--cpus", "boot=1"]) 1990 .args(&["--memory", "size=512M"]) 1991 .args(&["--kernel", fw_path.as_str()]) 1992 .default_disks() 1993 .default_net() 1994 .args(&["--serial", "tty", "--console", "off"]) 1995 .args(&["--event-monitor", format!("path={}", event_path).as_str()]) 1996 .capture_output() 1997 .spawn() 1998 .unwrap(); 1999 2000 let r = std::panic::catch_unwind(|| { 2001 guest.wait_vm_boot(Some(120)).unwrap(); 2002 2003 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2004 assert_eq!(guest.get_initial_apicid().unwrap_or(1), 0); 2005 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2006 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2007 2008 let expected_sequential_events = [ 2009 &MetaEvent { 2010 event: "starting".to_string(), 2011 device_id: None, 2012 }, 2013 &MetaEvent { 2014 event: "booting".to_string(), 2015 device_id: None, 2016 }, 2017 &MetaEvent { 2018 event: "booted".to_string(), 2019 device_id: None, 2020 }, 2021 &MetaEvent { 2022 event: "activated".to_string(), 2023 device_id: Some("_disk0".to_string()), 2024 }, 2025 &MetaEvent { 2026 event: "reset".to_string(), 2027 device_id: Some("_disk0".to_string()), 2028 }, 2029 ]; 2030 assert!(check_sequential_events( 2031 &expected_sequential_events, 2032 &event_path 2033 )); 2034 2035 guest.ssh_command("sudo poweroff").unwrap(); 2036 thread::sleep(std::time::Duration::new(5, 0)); 2037 let latest_events = [ 2038 &MetaEvent { 2039 event: "shutdown".to_string(), 2040 device_id: None, 2041 }, 2042 &MetaEvent { 2043 event: "deleted".to_string(), 2044 device_id: None, 2045 }, 2046 &MetaEvent { 2047 event: "shutdown".to_string(), 2048 device_id: None, 2049 }, 2050 ]; 2051 assert!(check_latest_events_exact(&latest_events, &event_path)); 2052 }); 2053 2054 let _ = child.kill(); 2055 let output = child.wait_with_output().unwrap(); 2056 2057 handle_child_output(r, &output); 2058 } 2059 2060 #[test] 2061 fn test_multi_cpu() { 2062 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 2063 let guest = Guest::new(Box::new(bionic)); 2064 let mut cmd = GuestCommand::new(&guest); 2065 cmd.args(&["--cpus", "boot=2,max=4"]) 2066 .args(&["--memory", "size=512M"]) 2067 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2068 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2069 .capture_output() 2070 .default_disks() 2071 .default_net(); 2072 2073 let mut child = cmd.spawn().unwrap(); 2074 2075 let r = std::panic::catch_unwind(|| { 2076 guest.wait_vm_boot(Some(120)).unwrap(); 2077 2078 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2079 2080 #[cfg(target_arch = "x86_64")] 2081 assert_eq!( 2082 guest 2083 .ssh_command(r#"dmesg | grep "smpboot: Allowing" | sed "s/\[\ *[0-9.]*\] //""#) 2084 .unwrap() 2085 .trim(), 2086 "smpboot: Allowing 4 CPUs, 2 hotplug CPUs" 2087 ); 2088 #[cfg(target_arch = "aarch64")] 2089 assert_eq!( 2090 guest 2091 .ssh_command(r#"dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""#) 2092 .unwrap() 2093 .trim(), 2094 "smp: Brought up 1 node, 2 CPUs" 2095 ); 2096 }); 2097 2098 let _ = child.kill(); 2099 let output = child.wait_with_output().unwrap(); 2100 2101 handle_child_output(r, &output); 2102 } 2103 2104 #[test] 2105 #[cfg(not(feature = "mshv"))] 2106 fn test_cpu_topology_421() { 2107 test_cpu_topology(4, 2, 1, false); 2108 } 2109 2110 #[test] 2111 #[cfg(not(feature = "mshv"))] 2112 fn test_cpu_topology_142() { 2113 test_cpu_topology(1, 4, 2, false); 2114 } 2115 2116 #[test] 2117 #[cfg(not(feature = "mshv"))] 2118 fn test_cpu_topology_262() { 2119 test_cpu_topology(2, 6, 2, false); 2120 } 2121 2122 #[test] 2123 #[cfg(target_arch = "x86_64")] 2124 #[cfg(not(feature = "mshv"))] 2125 fn test_cpu_physical_bits() { 2126 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2127 let guest = Guest::new(Box::new(focal)); 2128 let max_phys_bits: u8 = 36; 2129 let mut child = GuestCommand::new(&guest) 2130 .args(&["--cpus", &format!("max_phys_bits={}", max_phys_bits)]) 2131 .args(&["--memory", "size=512M"]) 2132 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2133 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2134 .default_disks() 2135 .default_net() 2136 .capture_output() 2137 .spawn() 2138 .unwrap(); 2139 2140 let r = std::panic::catch_unwind(|| { 2141 guest.wait_vm_boot(None).unwrap(); 2142 2143 assert!( 2144 guest 2145 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2146 .unwrap() 2147 .trim() 2148 .parse::<u8>() 2149 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2150 ); 2151 }); 2152 2153 let _ = child.kill(); 2154 let output = child.wait_with_output().unwrap(); 2155 2156 handle_child_output(r, &output); 2157 } 2158 2159 #[test] 2160 fn test_cpu_affinity() { 2161 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2162 let guest = Guest::new(Box::new(focal)); 2163 2164 // We need the host to have at least 4 CPUs if we want to be able 2165 // to run this test. 2166 let host_cpus_count = exec_host_command_output("nproc"); 2167 assert!( 2168 String::from_utf8_lossy(&host_cpus_count.stdout) 2169 .trim() 2170 .parse::<u8>() 2171 .unwrap_or(0) 2172 >= 4 2173 ); 2174 2175 let mut child = GuestCommand::new(&guest) 2176 .args(&["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2177 .args(&["--memory", "size=512M"]) 2178 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2179 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2180 .default_disks() 2181 .default_net() 2182 .capture_output() 2183 .spawn() 2184 .unwrap(); 2185 2186 let r = std::panic::catch_unwind(|| { 2187 guest.wait_vm_boot(None).unwrap(); 2188 let pid = child.id(); 2189 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"", pid).as_str()); 2190 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2191 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"", pid).as_str()); 2192 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2193 }); 2194 2195 let _ = child.kill(); 2196 let output = child.wait_with_output().unwrap(); 2197 2198 handle_child_output(r, &output); 2199 } 2200 2201 #[test] 2202 #[cfg(not(feature = "mshv"))] 2203 fn test_large_vm() { 2204 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2205 let guest = Guest::new(Box::new(focal)); 2206 let mut cmd = GuestCommand::new(&guest); 2207 cmd.args(&["--cpus", "boot=48"]) 2208 .args(&["--memory", "size=5120M"]) 2209 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2210 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2211 .args(&["--serial", "tty"]) 2212 .args(&["--console", "off"]) 2213 .capture_output() 2214 .default_disks() 2215 .default_net(); 2216 2217 let mut child = cmd.spawn().unwrap(); 2218 2219 guest.wait_vm_boot(None).unwrap(); 2220 2221 let r = std::panic::catch_unwind(|| { 2222 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2223 assert_eq!( 2224 guest 2225 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2226 .unwrap() 2227 .trim(), 2228 "0-47" 2229 ); 2230 2231 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2232 }); 2233 2234 let _ = child.kill(); 2235 let output = child.wait_with_output().unwrap(); 2236 2237 handle_child_output(r, &output); 2238 } 2239 2240 #[test] 2241 #[cfg(not(feature = "mshv"))] 2242 fn test_huge_memory() { 2243 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2244 let guest = Guest::new(Box::new(focal)); 2245 let mut cmd = GuestCommand::new(&guest); 2246 cmd.args(&["--cpus", "boot=1"]) 2247 .args(&["--memory", "size=128G"]) 2248 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2249 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2250 .capture_output() 2251 .default_disks() 2252 .default_net(); 2253 2254 let mut child = cmd.spawn().unwrap(); 2255 2256 guest.wait_vm_boot(Some(120)).unwrap(); 2257 2258 let r = std::panic::catch_unwind(|| { 2259 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2260 }); 2261 2262 let _ = child.kill(); 2263 let output = child.wait_with_output().unwrap(); 2264 2265 handle_child_output(r, &output); 2266 } 2267 2268 #[test] 2269 fn test_power_button() { 2270 _test_power_button(false); 2271 } 2272 2273 #[test] 2274 #[cfg(not(feature = "mshv"))] 2275 fn test_user_defined_memory_regions() { 2276 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2277 let guest = Guest::new(Box::new(focal)); 2278 let api_socket = temp_api_path(&guest.tmp_dir); 2279 2280 let kernel_path = direct_kernel_boot_path(); 2281 2282 let mut child = GuestCommand::new(&guest) 2283 .args(&["--cpus", "boot=1"]) 2284 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 2285 .args(&[ 2286 "--memory-zone", 2287 "id=mem0,size=1G,hotplug_size=2G", 2288 "id=mem1,size=1G,file=/dev/shm", 2289 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2290 ]) 2291 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2292 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2293 .args(&["--api-socket", &api_socket]) 2294 .capture_output() 2295 .default_disks() 2296 .default_net() 2297 .spawn() 2298 .unwrap(); 2299 2300 let r = std::panic::catch_unwind(|| { 2301 guest.wait_vm_boot(None).unwrap(); 2302 2303 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2304 2305 guest.enable_memory_hotplug(); 2306 2307 resize_zone_command(&api_socket, "mem0", "3G"); 2308 thread::sleep(std::time::Duration::new(5, 0)); 2309 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2310 resize_zone_command(&api_socket, "mem2", "3G"); 2311 thread::sleep(std::time::Duration::new(5, 0)); 2312 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2313 resize_zone_command(&api_socket, "mem0", "2G"); 2314 thread::sleep(std::time::Duration::new(5, 0)); 2315 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2316 resize_zone_command(&api_socket, "mem2", "2G"); 2317 thread::sleep(std::time::Duration::new(5, 0)); 2318 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2319 2320 guest.reboot_linux(0, None); 2321 2322 // Check the amount of RAM after reboot 2323 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2324 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2325 2326 // Check if we can still resize down to the initial 'boot'size 2327 resize_zone_command(&api_socket, "mem0", "1G"); 2328 thread::sleep(std::time::Duration::new(5, 0)); 2329 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2330 resize_zone_command(&api_socket, "mem2", "1G"); 2331 thread::sleep(std::time::Duration::new(5, 0)); 2332 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2333 }); 2334 2335 let _ = child.kill(); 2336 let output = child.wait_with_output().unwrap(); 2337 2338 handle_child_output(r, &output); 2339 } 2340 2341 #[test] 2342 #[cfg(not(feature = "mshv"))] 2343 fn test_guest_numa_nodes() { 2344 _test_guest_numa_nodes(false); 2345 } 2346 2347 #[test] 2348 #[cfg(target_arch = "x86_64")] 2349 fn test_iommu_segments() { 2350 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2351 let focal = UbuntuDiskConfig::new(focal_image); 2352 let guest = Guest::new(Box::new(focal)); 2353 2354 // Prepare another disk file for the virtio-disk device 2355 let test_disk_path = String::from( 2356 guest 2357 .tmp_dir 2358 .as_path() 2359 .join("test-disk.raw") 2360 .to_str() 2361 .unwrap(), 2362 ); 2363 assert!( 2364 exec_host_command_status(format!("truncate {} -s 4M", test_disk_path).as_str()) 2365 .success() 2366 ); 2367 assert!( 2368 exec_host_command_status(format!("mkfs.ext4 {}", test_disk_path).as_str()).success() 2369 ); 2370 2371 let api_socket = temp_api_path(&guest.tmp_dir); 2372 let mut cmd = GuestCommand::new(&guest); 2373 2374 cmd.args(&["--cpus", "boot=1"]) 2375 .args(&["--api-socket", &api_socket]) 2376 .args(&["--memory", "size=512M"]) 2377 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2378 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2379 .args(&["--platform", "num_pci_segments=16,iommu_segments=[1]"]) 2380 .default_disks() 2381 .capture_output() 2382 .default_net(); 2383 2384 let mut child = cmd.spawn().unwrap(); 2385 2386 guest.wait_vm_boot(None).unwrap(); 2387 2388 let r = std::panic::catch_unwind(|| { 2389 let (cmd_success, cmd_output) = remote_command_w_output( 2390 &api_socket, 2391 "add-disk", 2392 Some( 2393 format!( 2394 "path={},id=test0,pci_segment=1,iommu=on", 2395 test_disk_path.as_str() 2396 ) 2397 .as_str(), 2398 ), 2399 ); 2400 assert!(cmd_success); 2401 assert!(String::from_utf8_lossy(&cmd_output) 2402 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2403 2404 // Check IOMMU setup 2405 assert!(guest 2406 .does_device_vendor_pair_match("0x1057", "0x1af4") 2407 .unwrap_or_default()); 2408 assert_eq!( 2409 guest 2410 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2411 .unwrap() 2412 .trim(), 2413 "0001:00:01.0" 2414 ); 2415 }); 2416 2417 let _ = child.kill(); 2418 let output = child.wait_with_output().unwrap(); 2419 2420 handle_child_output(r, &output); 2421 } 2422 2423 #[test] 2424 fn test_pci_msi() { 2425 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2426 let guest = Guest::new(Box::new(focal)); 2427 let mut cmd = GuestCommand::new(&guest); 2428 cmd.args(&["--cpus", "boot=1"]) 2429 .args(&["--memory", "size=512M"]) 2430 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2431 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2432 .capture_output() 2433 .default_disks() 2434 .default_net(); 2435 2436 let mut child = cmd.spawn().unwrap(); 2437 2438 guest.wait_vm_boot(None).unwrap(); 2439 2440 #[cfg(target_arch = "x86_64")] 2441 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2442 #[cfg(target_arch = "aarch64")] 2443 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2444 2445 let r = std::panic::catch_unwind(|| { 2446 assert_eq!( 2447 guest 2448 .ssh_command(grep_cmd) 2449 .unwrap() 2450 .trim() 2451 .parse::<u32>() 2452 .unwrap_or_default(), 2453 12 2454 ); 2455 }); 2456 2457 let _ = child.kill(); 2458 let output = child.wait_with_output().unwrap(); 2459 2460 handle_child_output(r, &output); 2461 } 2462 2463 #[test] 2464 fn test_virtio_net_ctrl_queue() { 2465 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2466 let guest = Guest::new(Box::new(focal)); 2467 let mut cmd = GuestCommand::new(&guest); 2468 cmd.args(&["--cpus", "boot=1"]) 2469 .args(&["--memory", "size=512M"]) 2470 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2471 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2472 .capture_output() 2473 .default_disks() 2474 .default_net(); 2475 2476 let mut child = cmd.spawn().unwrap(); 2477 2478 guest.wait_vm_boot(None).unwrap(); 2479 2480 #[cfg(target_arch = "aarch64")] 2481 let iface = "enp0s4"; 2482 #[cfg(target_arch = "x86_64")] 2483 let iface = "ens4"; 2484 2485 let r = std::panic::catch_unwind(|| { 2486 assert_eq!( 2487 guest 2488 .ssh_command( 2489 format!("sudo ethtool -K {} rx-gro-hw off && echo success", iface).as_str() 2490 ) 2491 .unwrap() 2492 .trim(), 2493 "success" 2494 ); 2495 }); 2496 2497 let _ = child.kill(); 2498 let output = child.wait_with_output().unwrap(); 2499 2500 handle_child_output(r, &output); 2501 } 2502 2503 #[test] 2504 #[cfg(not(feature = "mshv"))] 2505 fn test_pci_multiple_segments() { 2506 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2507 let guest = Guest::new(Box::new(focal)); 2508 2509 // Prepare another disk file for the virtio-disk device 2510 let test_disk_path = String::from( 2511 guest 2512 .tmp_dir 2513 .as_path() 2514 .join("test-disk.raw") 2515 .to_str() 2516 .unwrap(), 2517 ); 2518 assert!( 2519 exec_host_command_status(format!("truncate {} -s 4M", test_disk_path).as_str()) 2520 .success() 2521 ); 2522 assert!( 2523 exec_host_command_status(format!("mkfs.ext4 {}", test_disk_path).as_str()).success() 2524 ); 2525 2526 let mut cmd = GuestCommand::new(&guest); 2527 cmd.args(&["--cpus", "boot=1"]) 2528 .args(&["--memory", "size=512M"]) 2529 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2530 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2531 .args(&["--platform", "num_pci_segments=16"]) 2532 .args(&[ 2533 "--disk", 2534 format!( 2535 "path={}", 2536 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2537 ) 2538 .as_str(), 2539 format!( 2540 "path={}", 2541 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2542 ) 2543 .as_str(), 2544 format!("path={},pci_segment=15", test_disk_path).as_str(), 2545 ]) 2546 .capture_output() 2547 .default_net(); 2548 2549 let mut child = cmd.spawn().unwrap(); 2550 2551 guest.wait_vm_boot(None).unwrap(); 2552 2553 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2554 2555 let r = std::panic::catch_unwind(|| { 2556 // There should be 16 PCI host bridges in the guest. 2557 assert_eq!( 2558 guest 2559 .ssh_command(grep_cmd) 2560 .unwrap() 2561 .trim() 2562 .parse::<u32>() 2563 .unwrap_or_default(), 2564 16 2565 ); 2566 2567 // Check both if /dev/vdc exists and if the block size is 4M. 2568 assert_eq!( 2569 guest 2570 .ssh_command("lsblk | grep vdc | grep -c 4M") 2571 .unwrap() 2572 .trim() 2573 .parse::<u32>() 2574 .unwrap_or_default(), 2575 1 2576 ); 2577 2578 // Mount the device. 2579 guest.ssh_command("mkdir mount_image").unwrap(); 2580 guest 2581 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 2582 .unwrap(); 2583 // Grant all users with write permission. 2584 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 2585 2586 // Write something to the device. 2587 guest 2588 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 2589 .unwrap(); 2590 2591 // Check the content of the block device. The file "foo" should 2592 // contain "bar". 2593 assert_eq!( 2594 guest 2595 .ssh_command("sudo cat mount_image/foo") 2596 .unwrap() 2597 .trim(), 2598 "bar" 2599 ); 2600 }); 2601 2602 let _ = child.kill(); 2603 let output = child.wait_with_output().unwrap(); 2604 2605 handle_child_output(r, &output); 2606 } 2607 2608 #[test] 2609 fn test_direct_kernel_boot() { 2610 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2611 let guest = Guest::new(Box::new(focal)); 2612 2613 let kernel_path = direct_kernel_boot_path(); 2614 2615 let mut child = GuestCommand::new(&guest) 2616 .args(&["--cpus", "boot=1"]) 2617 .args(&["--memory", "size=512M"]) 2618 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2619 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2620 .default_disks() 2621 .default_net() 2622 .capture_output() 2623 .spawn() 2624 .unwrap(); 2625 2626 let r = std::panic::catch_unwind(|| { 2627 guest.wait_vm_boot(None).unwrap(); 2628 2629 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2630 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2631 2632 let grep_cmd = if cfg!(target_arch = "x86_64") { 2633 "grep -c PCI-MSI /proc/interrupts" 2634 } else { 2635 "grep -c ITS-MSI /proc/interrupts" 2636 }; 2637 assert_eq!( 2638 guest 2639 .ssh_command(grep_cmd) 2640 .unwrap() 2641 .trim() 2642 .parse::<u32>() 2643 .unwrap_or_default(), 2644 12 2645 ); 2646 }); 2647 2648 let _ = child.kill(); 2649 let output = child.wait_with_output().unwrap(); 2650 2651 handle_child_output(r, &output); 2652 } 2653 2654 fn _test_virtio_block(image_name: &str, disable_io_uring: bool) { 2655 let focal = UbuntuDiskConfig::new(image_name.to_string()); 2656 let guest = Guest::new(Box::new(focal)); 2657 2658 let mut workload_path = dirs::home_dir().unwrap(); 2659 workload_path.push("workloads"); 2660 2661 let mut blk_file_path = workload_path; 2662 blk_file_path.push("blk.img"); 2663 2664 let kernel_path = direct_kernel_boot_path(); 2665 2666 let mut cloud_child = GuestCommand::new(&guest) 2667 .args(&["--cpus", "boot=4"]) 2668 .args(&["--memory", "size=512M,shared=on"]) 2669 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2670 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2671 .args(&[ 2672 "--disk", 2673 format!( 2674 "path={}", 2675 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2676 ) 2677 .as_str(), 2678 format!( 2679 "path={}", 2680 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2681 ) 2682 .as_str(), 2683 format!( 2684 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={}", 2685 blk_file_path.to_str().unwrap(), 2686 disable_io_uring 2687 ) 2688 .as_str(), 2689 ]) 2690 .default_net() 2691 .capture_output() 2692 .spawn() 2693 .unwrap(); 2694 2695 let r = std::panic::catch_unwind(|| { 2696 guest.wait_vm_boot(None).unwrap(); 2697 2698 // Check both if /dev/vdc exists and if the block size is 16M. 2699 assert_eq!( 2700 guest 2701 .ssh_command("lsblk | grep vdc | grep -c 16M") 2702 .unwrap() 2703 .trim() 2704 .parse::<u32>() 2705 .unwrap_or_default(), 2706 1 2707 ); 2708 2709 // Check both if /dev/vdc exists and if this block is RO. 2710 assert_eq!( 2711 guest 2712 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 2713 .unwrap() 2714 .trim() 2715 .parse::<u32>() 2716 .unwrap_or_default(), 2717 1 2718 ); 2719 2720 // Check if the number of queues is 4. 2721 assert_eq!( 2722 guest 2723 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 2724 .unwrap() 2725 .trim() 2726 .parse::<u32>() 2727 .unwrap_or_default(), 2728 4 2729 ); 2730 }); 2731 2732 let _ = cloud_child.kill(); 2733 let output = cloud_child.wait_with_output().unwrap(); 2734 2735 handle_child_output(r, &output); 2736 } 2737 2738 #[test] 2739 fn test_virtio_block() { 2740 _test_virtio_block(FOCAL_IMAGE_NAME, false) 2741 } 2742 2743 #[test] 2744 fn test_virtio_block_disable_io_uring() { 2745 _test_virtio_block(FOCAL_IMAGE_NAME, true) 2746 } 2747 2748 #[test] 2749 fn test_virtio_block_qcow2() { 2750 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false) 2751 } 2752 2753 #[test] 2754 fn test_virtio_block_vhd() { 2755 let mut workload_path = dirs::home_dir().unwrap(); 2756 workload_path.push("workloads"); 2757 2758 let mut raw_file_path = workload_path.clone(); 2759 let mut vhd_file_path = workload_path; 2760 raw_file_path.push(FOCAL_IMAGE_NAME); 2761 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 2762 2763 // Generate VHD file from RAW file 2764 std::process::Command::new("qemu-img") 2765 .arg("convert") 2766 .arg("-p") 2767 .args(&["-f", "raw"]) 2768 .args(&["-O", "vpc"]) 2769 .args(&["-o", "subformat=fixed"]) 2770 .arg(raw_file_path.to_str().unwrap()) 2771 .arg(vhd_file_path.to_str().unwrap()) 2772 .output() 2773 .expect("Expect generating VHD image from RAW image"); 2774 2775 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false) 2776 } 2777 2778 #[test] 2779 fn test_virtio_block_vhdx() { 2780 let mut workload_path = dirs::home_dir().unwrap(); 2781 workload_path.push("workloads"); 2782 2783 let mut raw_file_path = workload_path.clone(); 2784 let mut vhdx_file_path = workload_path; 2785 raw_file_path.push(FOCAL_IMAGE_NAME); 2786 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 2787 2788 // Generate dynamic VHDX file from RAW file 2789 std::process::Command::new("qemu-img") 2790 .arg("convert") 2791 .arg("-p") 2792 .args(&["-f", "raw"]) 2793 .args(&["-O", "vhdx"]) 2794 .arg(raw_file_path.to_str().unwrap()) 2795 .arg(vhdx_file_path.to_str().unwrap()) 2796 .output() 2797 .expect("Expect generating dynamic VHDx image from RAW image"); 2798 2799 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false) 2800 } 2801 2802 #[test] 2803 fn test_virtio_block_dynamic_vhdx_expand() { 2804 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 2805 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 2806 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 2807 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 2808 2809 let mut workload_path = dirs::home_dir().unwrap(); 2810 workload_path.push("workloads"); 2811 2812 let mut vhdx_file_path = workload_path; 2813 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 2814 let vhdx_path = vhdx_file_path.to_str().unwrap(); 2815 2816 // Generate a 100 MiB dynamic VHDX file 2817 std::process::Command::new("qemu-img") 2818 .arg("create") 2819 .args(&["-f", "vhdx"]) 2820 .arg(vhdx_path) 2821 .arg(VIRTUAL_DISK_SIZE.to_string()) 2822 .output() 2823 .expect("Expect generating dynamic VHDx image from RAW image"); 2824 2825 // Check if the size matches with empty VHDx file size 2826 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 2827 2828 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2829 let guest = Guest::new(Box::new(focal)); 2830 let kernel_path = direct_kernel_boot_path(); 2831 2832 let mut cloud_child = GuestCommand::new(&guest) 2833 .args(&["--cpus", "boot=1"]) 2834 .args(&["--memory", "size=512M"]) 2835 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2836 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2837 .args(&[ 2838 "--disk", 2839 format!( 2840 "path={}", 2841 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2842 ) 2843 .as_str(), 2844 format!( 2845 "path={}", 2846 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2847 ) 2848 .as_str(), 2849 format!("path={}", vhdx_path).as_str(), 2850 ]) 2851 .default_net() 2852 .capture_output() 2853 .spawn() 2854 .unwrap(); 2855 2856 let r = std::panic::catch_unwind(|| { 2857 guest.wait_vm_boot(None).unwrap(); 2858 2859 // Check both if /dev/vdc exists and if the block size is 100 MiB. 2860 assert_eq!( 2861 guest 2862 .ssh_command("lsblk | grep vdc | grep -c 100M") 2863 .unwrap() 2864 .trim() 2865 .parse::<u32>() 2866 .unwrap_or_default(), 2867 1 2868 ); 2869 2870 // Write 100 MB of data to the VHDx disk 2871 guest 2872 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 2873 .unwrap(); 2874 }); 2875 2876 // Check if the size matches with expected expanded VHDx file size 2877 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 2878 2879 let _ = cloud_child.kill(); 2880 let output = cloud_child.wait_with_output().unwrap(); 2881 2882 handle_child_output(r, &output); 2883 } 2884 2885 fn vhdx_image_size(disk_name: &str) -> u64 { 2886 std::fs::File::open(disk_name) 2887 .unwrap() 2888 .seek(SeekFrom::End(0)) 2889 .unwrap() 2890 } 2891 2892 #[test] 2893 fn test_virtio_block_direct_and_firmware() { 2894 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2895 let guest = Guest::new(Box::new(focal)); 2896 2897 // The OS disk must be copied to a location that is not backed by 2898 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 2899 // with EINVAL because tmpfs doesn't support this flag. 2900 let mut workloads_path = dirs::home_dir().unwrap(); 2901 workloads_path.push("workloads"); 2902 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 2903 let mut os_path = os_dir.as_path().to_path_buf(); 2904 os_path.push("osdisk.img"); 2905 rate_limited_copy( 2906 &guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 2907 os_path.as_path(), 2908 ) 2909 .expect("copying of OS disk failed"); 2910 2911 let mut child = GuestCommand::new(&guest) 2912 .args(&["--cpus", "boot=1"]) 2913 .args(&["--memory", "size=512M"]) 2914 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 2915 .args(&[ 2916 "--disk", 2917 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 2918 format!( 2919 "path={}", 2920 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2921 ) 2922 .as_str(), 2923 ]) 2924 .default_net() 2925 .capture_output() 2926 .spawn() 2927 .unwrap(); 2928 2929 let r = std::panic::catch_unwind(|| { 2930 guest.wait_vm_boot(Some(120)).unwrap(); 2931 }); 2932 2933 let _ = child.kill(); 2934 let output = child.wait_with_output().unwrap(); 2935 2936 handle_child_output(r, &output); 2937 } 2938 2939 #[test] 2940 fn test_vhost_user_net_default() { 2941 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 2942 } 2943 2944 #[test] 2945 fn test_vhost_user_net_named_tap() { 2946 test_vhost_user_net( 2947 Some("mytap0"), 2948 2, 2949 &prepare_vhost_user_net_daemon, 2950 false, 2951 false, 2952 ) 2953 } 2954 2955 #[test] 2956 fn test_vhost_user_net_existing_tap() { 2957 test_vhost_user_net( 2958 Some("vunet-tap0"), 2959 2, 2960 &prepare_vhost_user_net_daemon, 2961 false, 2962 false, 2963 ) 2964 } 2965 2966 #[test] 2967 fn test_vhost_user_net_multiple_queues() { 2968 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 2969 } 2970 2971 #[test] 2972 fn test_vhost_user_net_tap_multiple_queues() { 2973 test_vhost_user_net( 2974 Some("vunet-tap1"), 2975 4, 2976 &prepare_vhost_user_net_daemon, 2977 false, 2978 false, 2979 ) 2980 } 2981 2982 #[test] 2983 fn test_vhost_user_net_host_mac() { 2984 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 2985 } 2986 2987 #[test] 2988 fn test_vhost_user_net_client_mode() { 2989 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 2990 } 2991 2992 #[test] 2993 fn test_vhost_user_blk_default() { 2994 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 2995 } 2996 2997 #[test] 2998 fn test_vhost_user_blk_readonly() { 2999 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3000 } 3001 3002 #[test] 3003 fn test_vhost_user_blk_direct() { 3004 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3005 } 3006 3007 #[test] 3008 fn test_boot_from_vhost_user_blk_default() { 3009 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3010 } 3011 3012 #[test] 3013 #[cfg(target_arch = "x86_64")] 3014 fn test_split_irqchip() { 3015 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3016 let guest = Guest::new(Box::new(focal)); 3017 3018 let mut child = GuestCommand::new(&guest) 3019 .args(&["--cpus", "boot=1"]) 3020 .args(&["--memory", "size=512M"]) 3021 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3022 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3023 .default_disks() 3024 .default_net() 3025 .capture_output() 3026 .spawn() 3027 .unwrap(); 3028 3029 let r = std::panic::catch_unwind(|| { 3030 guest.wait_vm_boot(None).unwrap(); 3031 3032 assert_eq!( 3033 guest 3034 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3035 .unwrap() 3036 .trim() 3037 .parse::<u32>() 3038 .unwrap_or(1), 3039 0 3040 ); 3041 assert_eq!( 3042 guest 3043 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3044 .unwrap() 3045 .trim() 3046 .parse::<u32>() 3047 .unwrap_or(1), 3048 0 3049 ); 3050 }); 3051 3052 let _ = child.kill(); 3053 let output = child.wait_with_output().unwrap(); 3054 3055 handle_child_output(r, &output); 3056 } 3057 3058 #[test] 3059 #[cfg(target_arch = "x86_64")] 3060 fn test_dmi_serial_number() { 3061 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3062 let guest = Guest::new(Box::new(focal)); 3063 3064 let mut child = GuestCommand::new(&guest) 3065 .args(&["--cpus", "boot=1"]) 3066 .args(&["--memory", "size=512M"]) 3067 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3068 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3069 .args(&["--platform", "serial_number=a=b;c=d"]) 3070 .default_disks() 3071 .default_net() 3072 .capture_output() 3073 .spawn() 3074 .unwrap(); 3075 3076 let r = std::panic::catch_unwind(|| { 3077 guest.wait_vm_boot(None).unwrap(); 3078 3079 assert_eq!( 3080 guest 3081 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3082 .unwrap() 3083 .trim(), 3084 "a=b;c=d" 3085 ); 3086 }); 3087 3088 let _ = child.kill(); 3089 let output = child.wait_with_output().unwrap(); 3090 3091 handle_child_output(r, &output); 3092 } 3093 3094 #[test] 3095 fn test_virtio_fs() { 3096 _test_virtio_fs(&prepare_virtiofsd, false, None) 3097 } 3098 3099 #[test] 3100 fn test_virtio_fs_hotplug() { 3101 _test_virtio_fs(&prepare_virtiofsd, true, None) 3102 } 3103 3104 #[test] 3105 #[cfg(not(feature = "mshv"))] 3106 fn test_virtio_fs_multi_segment_hotplug() { 3107 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3108 } 3109 3110 #[test] 3111 #[cfg(not(feature = "mshv"))] 3112 fn test_virtio_fs_multi_segment() { 3113 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3114 } 3115 3116 #[test] 3117 fn test_virtio_pmem_persist_writes() { 3118 test_virtio_pmem(false, false) 3119 } 3120 3121 #[test] 3122 fn test_virtio_pmem_discard_writes() { 3123 test_virtio_pmem(true, false) 3124 } 3125 3126 #[test] 3127 fn test_virtio_pmem_with_size() { 3128 test_virtio_pmem(true, true) 3129 } 3130 3131 #[test] 3132 fn test_boot_from_virtio_pmem() { 3133 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3134 let guest = Guest::new(Box::new(focal)); 3135 3136 let kernel_path = direct_kernel_boot_path(); 3137 3138 let mut child = GuestCommand::new(&guest) 3139 .args(&["--cpus", "boot=1"]) 3140 .args(&["--memory", "size=512M"]) 3141 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3142 .args(&[ 3143 "--disk", 3144 format!( 3145 "path={}", 3146 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3147 ) 3148 .as_str(), 3149 ]) 3150 .default_net() 3151 .args(&[ 3152 "--pmem", 3153 format!( 3154 "file={},size={}", 3155 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3156 fs::metadata(&guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3157 .unwrap() 3158 .len() 3159 ) 3160 .as_str(), 3161 ]) 3162 .args(&[ 3163 "--cmdline", 3164 DIRECT_KERNEL_BOOT_CMDLINE 3165 .replace("vda1", "pmem0p1") 3166 .as_str(), 3167 ]) 3168 .capture_output() 3169 .spawn() 3170 .unwrap(); 3171 3172 let r = std::panic::catch_unwind(|| { 3173 guest.wait_vm_boot(None).unwrap(); 3174 3175 // Simple checks to validate the VM booted properly 3176 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3177 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3178 }); 3179 3180 let _ = child.kill(); 3181 let output = child.wait_with_output().unwrap(); 3182 3183 handle_child_output(r, &output); 3184 } 3185 3186 #[test] 3187 fn test_multiple_network_interfaces() { 3188 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3189 let guest = Guest::new(Box::new(focal)); 3190 3191 let kernel_path = direct_kernel_boot_path(); 3192 3193 let mut child = GuestCommand::new(&guest) 3194 .args(&["--cpus", "boot=1"]) 3195 .args(&["--memory", "size=512M"]) 3196 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3197 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3198 .default_disks() 3199 .args(&[ 3200 "--net", 3201 guest.default_net_string().as_str(), 3202 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3203 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3204 ]) 3205 .capture_output() 3206 .spawn() 3207 .unwrap(); 3208 3209 let r = std::panic::catch_unwind(|| { 3210 guest.wait_vm_boot(None).unwrap(); 3211 3212 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3213 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3214 3215 // 3 network interfaces + default localhost ==> 4 interfaces 3216 assert_eq!( 3217 guest 3218 .ssh_command("ip -o link | wc -l") 3219 .unwrap() 3220 .trim() 3221 .parse::<u32>() 3222 .unwrap_or_default(), 3223 4 3224 ); 3225 }); 3226 3227 let _ = child.kill(); 3228 let output = child.wait_with_output().unwrap(); 3229 3230 handle_child_output(r, &output); 3231 } 3232 3233 #[test] 3234 #[cfg(target_arch = "aarch64")] 3235 fn test_pmu_on() { 3236 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3237 let guest = Guest::new(Box::new(focal)); 3238 let mut child = GuestCommand::new(&guest) 3239 .args(&["--cpus", "boot=1"]) 3240 .args(&["--memory", "size=512M"]) 3241 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3242 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3243 .default_disks() 3244 .default_net() 3245 .capture_output() 3246 .spawn() 3247 .unwrap(); 3248 3249 let r = std::panic::catch_unwind(|| { 3250 guest.wait_vm_boot(None).unwrap(); 3251 3252 // Test that PMU exists. 3253 assert_eq!( 3254 guest 3255 .ssh_command(GREP_PMU_IRQ_CMD) 3256 .unwrap() 3257 .trim() 3258 .parse::<u32>() 3259 .unwrap_or_default(), 3260 1 3261 ); 3262 }); 3263 3264 let _ = child.kill(); 3265 let output = child.wait_with_output().unwrap(); 3266 3267 handle_child_output(r, &output); 3268 } 3269 3270 #[test] 3271 fn test_serial_off() { 3272 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3273 let guest = Guest::new(Box::new(focal)); 3274 let mut child = GuestCommand::new(&guest) 3275 .args(&["--cpus", "boot=1"]) 3276 .args(&["--memory", "size=512M"]) 3277 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3278 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3279 .default_disks() 3280 .default_net() 3281 .args(&["--serial", "off"]) 3282 .capture_output() 3283 .spawn() 3284 .unwrap(); 3285 3286 let r = std::panic::catch_unwind(|| { 3287 guest.wait_vm_boot(None).unwrap(); 3288 3289 // Test that there is no ttyS0 3290 assert_eq!( 3291 guest 3292 .ssh_command(GREP_SERIAL_IRQ_CMD) 3293 .unwrap() 3294 .trim() 3295 .parse::<u32>() 3296 .unwrap_or(1), 3297 0 3298 ); 3299 }); 3300 3301 let _ = child.kill(); 3302 let output = child.wait_with_output().unwrap(); 3303 3304 handle_child_output(r, &output); 3305 } 3306 3307 #[test] 3308 fn test_serial_null() { 3309 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3310 let guest = Guest::new(Box::new(focal)); 3311 let mut cmd = GuestCommand::new(&guest); 3312 #[cfg(target_arch = "x86_64")] 3313 let console_str: &str = "console=ttyS0"; 3314 #[cfg(target_arch = "aarch64")] 3315 let console_str: &str = "console=ttyAMA0"; 3316 3317 cmd.args(&["--cpus", "boot=1"]) 3318 .args(&["--memory", "size=512M"]) 3319 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3320 .args(&[ 3321 "--cmdline", 3322 DIRECT_KERNEL_BOOT_CMDLINE 3323 .replace("console=hvc0 ", console_str) 3324 .as_str(), 3325 ]) 3326 .default_disks() 3327 .default_net() 3328 .args(&["--serial", "null"]) 3329 .args(&["--console", "off"]) 3330 .capture_output(); 3331 3332 let mut child = cmd.spawn().unwrap(); 3333 3334 let r = std::panic::catch_unwind(|| { 3335 guest.wait_vm_boot(None).unwrap(); 3336 3337 // Test that there is a ttyS0 3338 assert_eq!( 3339 guest 3340 .ssh_command(GREP_SERIAL_IRQ_CMD) 3341 .unwrap() 3342 .trim() 3343 .parse::<u32>() 3344 .unwrap_or_default(), 3345 1 3346 ); 3347 }); 3348 3349 let _ = child.kill(); 3350 let output = child.wait_with_output().unwrap(); 3351 handle_child_output(r, &output); 3352 3353 let r = std::panic::catch_unwind(|| { 3354 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3355 }); 3356 3357 handle_child_output(r, &output); 3358 } 3359 3360 #[test] 3361 fn test_serial_tty() { 3362 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3363 let guest = Guest::new(Box::new(focal)); 3364 3365 let kernel_path = direct_kernel_boot_path(); 3366 3367 #[cfg(target_arch = "x86_64")] 3368 let console_str: &str = "console=ttyS0"; 3369 #[cfg(target_arch = "aarch64")] 3370 let console_str: &str = "console=ttyAMA0"; 3371 3372 let mut child = GuestCommand::new(&guest) 3373 .args(&["--cpus", "boot=1"]) 3374 .args(&["--memory", "size=512M"]) 3375 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3376 .args(&[ 3377 "--cmdline", 3378 DIRECT_KERNEL_BOOT_CMDLINE 3379 .replace("console=hvc0 ", console_str) 3380 .as_str(), 3381 ]) 3382 .default_disks() 3383 .default_net() 3384 .args(&["--serial", "tty"]) 3385 .args(&["--console", "off"]) 3386 .capture_output() 3387 .spawn() 3388 .unwrap(); 3389 3390 let r = std::panic::catch_unwind(|| { 3391 guest.wait_vm_boot(None).unwrap(); 3392 3393 // Test that there is a ttyS0 3394 assert_eq!( 3395 guest 3396 .ssh_command(GREP_SERIAL_IRQ_CMD) 3397 .unwrap() 3398 .trim() 3399 .parse::<u32>() 3400 .unwrap_or_default(), 3401 1 3402 ); 3403 }); 3404 3405 // This sleep is needed to wait for the login prompt 3406 thread::sleep(std::time::Duration::new(2, 0)); 3407 3408 let _ = child.kill(); 3409 let output = child.wait_with_output().unwrap(); 3410 handle_child_output(r, &output); 3411 3412 let r = std::panic::catch_unwind(|| { 3413 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3414 }); 3415 3416 handle_child_output(r, &output); 3417 } 3418 3419 #[test] 3420 fn test_serial_file() { 3421 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3422 let guest = Guest::new(Box::new(focal)); 3423 3424 let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output"); 3425 #[cfg(target_arch = "x86_64")] 3426 let console_str: &str = "console=ttyS0"; 3427 #[cfg(target_arch = "aarch64")] 3428 let console_str: &str = "console=ttyAMA0"; 3429 3430 let mut child = GuestCommand::new(&guest) 3431 .args(&["--cpus", "boot=1"]) 3432 .args(&["--memory", "size=512M"]) 3433 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3434 .args(&[ 3435 "--cmdline", 3436 DIRECT_KERNEL_BOOT_CMDLINE 3437 .replace("console=hvc0 ", console_str) 3438 .as_str(), 3439 ]) 3440 .default_disks() 3441 .default_net() 3442 .args(&[ 3443 "--serial", 3444 format!("file={}", serial_path.to_str().unwrap()).as_str(), 3445 ]) 3446 .capture_output() 3447 .spawn() 3448 .unwrap(); 3449 3450 let r = std::panic::catch_unwind(|| { 3451 guest.wait_vm_boot(None).unwrap(); 3452 3453 // Test that there is a ttyS0 3454 assert_eq!( 3455 guest 3456 .ssh_command(GREP_SERIAL_IRQ_CMD) 3457 .unwrap() 3458 .trim() 3459 .parse::<u32>() 3460 .unwrap_or_default(), 3461 1 3462 ); 3463 3464 guest.ssh_command("sudo shutdown -h now").unwrap(); 3465 }); 3466 3467 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3468 let _ = child.kill(); 3469 let output = child.wait_with_output().unwrap(); 3470 handle_child_output(r, &output); 3471 3472 let r = std::panic::catch_unwind(|| { 3473 // Check that the cloud-hypervisor binary actually terminated 3474 assert!(output.status.success()); 3475 3476 // Do this check after shutdown of the VM as an easy way to ensure 3477 // all writes are flushed to disk 3478 let mut f = std::fs::File::open(serial_path).unwrap(); 3479 let mut buf = String::new(); 3480 f.read_to_string(&mut buf).unwrap(); 3481 assert!(buf.contains(CONSOLE_TEST_STRING)); 3482 }); 3483 3484 handle_child_output(r, &output); 3485 } 3486 3487 #[test] 3488 fn test_pty_interaction() { 3489 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3490 let guest = Guest::new(Box::new(focal)); 3491 let api_socket = temp_api_path(&guest.tmp_dir); 3492 let serial_option = if cfg!(target_arch = "x86_64") { 3493 " console=ttyS0" 3494 } else { 3495 " console=ttyAMA0" 3496 }; 3497 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 3498 3499 let mut child = GuestCommand::new(&guest) 3500 .args(&["--cpus", "boot=1"]) 3501 .args(&["--memory", "size=512M"]) 3502 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3503 .args(&["--cmdline", &cmdline]) 3504 .default_disks() 3505 .default_net() 3506 .args(&["--serial", "null"]) 3507 .args(&["--console", "pty"]) 3508 .args(&["--api-socket", &api_socket]) 3509 .spawn() 3510 .unwrap(); 3511 3512 let r = std::panic::catch_unwind(|| { 3513 guest.wait_vm_boot(None).unwrap(); 3514 // Get pty fd for console 3515 let console_path = get_pty_path(&api_socket, "console"); 3516 // TODO: Get serial pty test working 3517 let mut cf = std::fs::OpenOptions::new() 3518 .write(true) 3519 .read(true) 3520 .open(console_path) 3521 .unwrap(); 3522 3523 // Some dumb sleeps but we don't want to write 3524 // before the console is up and we don't want 3525 // to try and write the next line before the 3526 // login process is ready. 3527 thread::sleep(std::time::Duration::new(5, 0)); 3528 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 3529 thread::sleep(std::time::Duration::new(2, 0)); 3530 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 3531 thread::sleep(std::time::Duration::new(2, 0)); 3532 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 3533 thread::sleep(std::time::Duration::new(2, 0)); 3534 3535 // read pty and ensure they have a login shell 3536 // some fairly hacky workarounds to avoid looping 3537 // forever in case the channel is blocked getting output 3538 let ptyc = pty_read(cf); 3539 let mut empty = 0; 3540 let mut prev = String::new(); 3541 loop { 3542 thread::sleep(std::time::Duration::new(2, 0)); 3543 match ptyc.try_recv() { 3544 Ok(line) => { 3545 empty = 0; 3546 prev = prev + &line; 3547 if prev.contains("test_pty_console") { 3548 break; 3549 } 3550 } 3551 Err(mpsc::TryRecvError::Empty) => { 3552 empty += 1; 3553 assert!(empty <= 5, "No login on pty"); 3554 } 3555 _ => panic!("No login on pty"), 3556 } 3557 } 3558 3559 guest.ssh_command("sudo shutdown -h now").unwrap(); 3560 }); 3561 3562 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3563 let _ = child.kill(); 3564 let output = child.wait_with_output().unwrap(); 3565 handle_child_output(r, &output); 3566 3567 let r = std::panic::catch_unwind(|| { 3568 // Check that the cloud-hypervisor binary actually terminated 3569 assert!(output.status.success()) 3570 }); 3571 handle_child_output(r, &output); 3572 } 3573 3574 #[test] 3575 fn test_virtio_console() { 3576 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3577 let guest = Guest::new(Box::new(focal)); 3578 3579 let kernel_path = direct_kernel_boot_path(); 3580 3581 let mut child = GuestCommand::new(&guest) 3582 .args(&["--cpus", "boot=1"]) 3583 .args(&["--memory", "size=512M"]) 3584 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3585 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3586 .default_disks() 3587 .default_net() 3588 .args(&["--console", "tty"]) 3589 .args(&["--serial", "null"]) 3590 .capture_output() 3591 .spawn() 3592 .unwrap(); 3593 3594 let text = String::from("On a branch floating down river a cricket, singing."); 3595 let cmd = format!("echo {} | sudo tee /dev/hvc0", text); 3596 3597 let r = std::panic::catch_unwind(|| { 3598 guest.wait_vm_boot(None).unwrap(); 3599 3600 assert!(guest 3601 .does_device_vendor_pair_match("0x1043", "0x1af4") 3602 .unwrap_or_default()); 3603 3604 guest.ssh_command(&cmd).unwrap(); 3605 }); 3606 3607 let _ = child.kill(); 3608 let output = child.wait_with_output().unwrap(); 3609 handle_child_output(r, &output); 3610 3611 let r = std::panic::catch_unwind(|| { 3612 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 3613 }); 3614 3615 handle_child_output(r, &output); 3616 } 3617 3618 #[test] 3619 fn test_console_file() { 3620 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3621 let guest = Guest::new(Box::new(focal)); 3622 3623 let console_path = guest.tmp_dir.as_path().join("/tmp/console-output"); 3624 let mut child = GuestCommand::new(&guest) 3625 .args(&["--cpus", "boot=1"]) 3626 .args(&["--memory", "size=512M"]) 3627 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3628 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3629 .default_disks() 3630 .default_net() 3631 .args(&[ 3632 "--console", 3633 format!("file={}", console_path.to_str().unwrap()).as_str(), 3634 ]) 3635 .capture_output() 3636 .spawn() 3637 .unwrap(); 3638 3639 guest.wait_vm_boot(None).unwrap(); 3640 3641 guest.ssh_command("sudo shutdown -h now").unwrap(); 3642 3643 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3644 let _ = child.kill(); 3645 let output = child.wait_with_output().unwrap(); 3646 3647 let r = std::panic::catch_unwind(|| { 3648 // Check that the cloud-hypervisor binary actually terminated 3649 assert!(output.status.success()); 3650 3651 // Do this check after shutdown of the VM as an easy way to ensure 3652 // all writes are flushed to disk 3653 let mut f = std::fs::File::open(console_path).unwrap(); 3654 let mut buf = String::new(); 3655 f.read_to_string(&mut buf).unwrap(); 3656 3657 if !buf.contains(CONSOLE_TEST_STRING) { 3658 eprintln!( 3659 "\n\n==== Console file output ====\n\n{}\n\n==== End console file output ====", 3660 buf 3661 ); 3662 } 3663 assert!(buf.contains(CONSOLE_TEST_STRING)); 3664 }); 3665 3666 handle_child_output(r, &output); 3667 } 3668 3669 #[test] 3670 #[cfg(target_arch = "x86_64")] 3671 #[cfg(not(feature = "mshv"))] 3672 #[ignore = "See #4324"] 3673 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 3674 // backed networking interfaces, bound through a simple bridge on the host. 3675 // So if the nested cloud-hypervisor succeeds in getting a directly 3676 // assigned interface from its cloud-hypervisor host, we should be able to 3677 // ssh into it, and verify that it's running with the right kernel command 3678 // line (We tag the command line from cloud-hypervisor for that purpose). 3679 // The third device is added to validate that hotplug works correctly since 3680 // it is being added to the L2 VM through hotplugging mechanism. 3681 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 3682 // vfio device support 3683 fn test_vfio() { 3684 setup_vfio_network_interfaces(); 3685 3686 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3687 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 3688 3689 let mut workload_path = dirs::home_dir().unwrap(); 3690 workload_path.push("workloads"); 3691 3692 let kernel_path = direct_kernel_boot_path(); 3693 3694 let mut vfio_path = workload_path.clone(); 3695 vfio_path.push("vfio"); 3696 3697 let mut cloud_init_vfio_base_path = vfio_path.clone(); 3698 cloud_init_vfio_base_path.push("cloudinit.img"); 3699 3700 // We copy our cloudinit into the vfio mount point, for the nested 3701 // cloud-hypervisor guest to use. 3702 rate_limited_copy( 3703 &guest.disk_config.disk(DiskType::CloudInit).unwrap(), 3704 &cloud_init_vfio_base_path, 3705 ) 3706 .expect("copying of cloud-init disk failed"); 3707 3708 let mut vfio_disk_path = workload_path.clone(); 3709 vfio_disk_path.push("vfio.img"); 3710 3711 // Create the vfio disk image 3712 let output = Command::new("mkfs.ext4") 3713 .arg("-d") 3714 .arg(vfio_path.to_str().unwrap()) 3715 .arg(vfio_disk_path.to_str().unwrap()) 3716 .arg("2g") 3717 .output() 3718 .unwrap(); 3719 if !output.status.success() { 3720 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 3721 panic!("mkfs.ext4 command generated an error"); 3722 } 3723 3724 let mut blk_file_path = workload_path; 3725 blk_file_path.push("blk.img"); 3726 3727 let vfio_tap0 = "vfio-tap0"; 3728 let vfio_tap1 = "vfio-tap1"; 3729 let vfio_tap2 = "vfio-tap2"; 3730 let vfio_tap3 = "vfio-tap3"; 3731 3732 let mut child = GuestCommand::new(&guest) 3733 .args(&["--cpus", "boot=4"]) 3734 .args(&["--memory", "size=2G,hugepages=on,shared=on"]) 3735 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3736 .args(&[ 3737 "--disk", 3738 format!( 3739 "path={}", 3740 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3741 ) 3742 .as_str(), 3743 format!( 3744 "path={}", 3745 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3746 ) 3747 .as_str(), 3748 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 3749 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 3750 ]) 3751 .args(&[ 3752 "--cmdline", 3753 format!( 3754 "{} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts", 3755 DIRECT_KERNEL_BOOT_CMDLINE 3756 ) 3757 .as_str(), 3758 ]) 3759 .args(&[ 3760 "--net", 3761 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 3762 format!( 3763 "tap={},mac={},iommu=on", 3764 vfio_tap1, guest.network.l2_guest_mac1 3765 ) 3766 .as_str(), 3767 format!( 3768 "tap={},mac={},iommu=on", 3769 vfio_tap2, guest.network.l2_guest_mac2 3770 ) 3771 .as_str(), 3772 format!( 3773 "tap={},mac={},iommu=on", 3774 vfio_tap3, guest.network.l2_guest_mac3 3775 ) 3776 .as_str(), 3777 ]) 3778 .capture_output() 3779 .spawn() 3780 .unwrap(); 3781 3782 thread::sleep(std::time::Duration::new(30, 0)); 3783 3784 let r = std::panic::catch_unwind(|| { 3785 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 3786 thread::sleep(std::time::Duration::new(120, 0)); 3787 3788 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 3789 // added to its kernel command line. 3790 // Let's ssh into it and verify that it's there. If it is it means 3791 // we're in the right guest (The L2 one) because the QEMU L1 guest 3792 // does not have this command line tag. 3793 assert_eq!( 3794 guest 3795 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 3796 .unwrap() 3797 .trim() 3798 .parse::<u32>() 3799 .unwrap_or_default(), 3800 1 3801 ); 3802 3803 // Let's also verify from the second virtio-net device passed to 3804 // the L2 VM. 3805 assert_eq!( 3806 guest 3807 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 3808 .unwrap() 3809 .trim() 3810 .parse::<u32>() 3811 .unwrap_or_default(), 3812 1 3813 ); 3814 3815 // Check the amount of PCI devices appearing in L2 VM. 3816 assert_eq!( 3817 guest 3818 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3819 .unwrap() 3820 .trim() 3821 .parse::<u32>() 3822 .unwrap_or_default(), 3823 8, 3824 ); 3825 3826 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 3827 assert_eq!( 3828 guest 3829 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 3830 .unwrap() 3831 .trim() 3832 .parse::<u32>() 3833 .unwrap_or_default(), 3834 1 3835 ); 3836 3837 // Hotplug an extra virtio-net device through L2 VM. 3838 guest 3839 .ssh_command_l1( 3840 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 3841 ) 3842 .unwrap(); 3843 guest 3844 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 3845 .unwrap(); 3846 let vfio_hotplug_output = guest 3847 .ssh_command_l1( 3848 "sudo /mnt/ch-remote \ 3849 --api-socket=/tmp/ch_api.sock \ 3850 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 3851 ) 3852 .unwrap(); 3853 assert!(vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}")); 3854 3855 thread::sleep(std::time::Duration::new(10, 0)); 3856 3857 // Let's also verify from the third virtio-net device passed to 3858 // the L2 VM. This third device has been hotplugged through the L2 3859 // VM, so this is our way to validate hotplug works for VFIO PCI. 3860 assert_eq!( 3861 guest 3862 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 3863 .unwrap() 3864 .trim() 3865 .parse::<u32>() 3866 .unwrap_or_default(), 3867 1 3868 ); 3869 3870 // Check the amount of PCI devices appearing in L2 VM. 3871 // There should be one more device than before, raising the count 3872 // up to 9 PCI devices. 3873 assert_eq!( 3874 guest 3875 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3876 .unwrap() 3877 .trim() 3878 .parse::<u32>() 3879 .unwrap_or_default(), 3880 9, 3881 ); 3882 3883 // Let's now verify that we can correctly remove the virtio-net 3884 // device through the "remove-device" command responsible for 3885 // unplugging VFIO devices. 3886 guest 3887 .ssh_command_l1( 3888 "sudo /mnt/ch-remote \ 3889 --api-socket=/tmp/ch_api.sock \ 3890 remove-device vfio123", 3891 ) 3892 .unwrap(); 3893 thread::sleep(std::time::Duration::new(10, 0)); 3894 3895 // Check the amount of PCI devices appearing in L2 VM is back down 3896 // to 8 devices. 3897 assert_eq!( 3898 guest 3899 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3900 .unwrap() 3901 .trim() 3902 .parse::<u32>() 3903 .unwrap_or_default(), 3904 8, 3905 ); 3906 3907 // Perform memory hotplug in L2 and validate the memory is showing 3908 // up as expected. In order to check, we will use the virtio-net 3909 // device already passed through L2 as a VFIO device, this will 3910 // verify that VFIO devices are functional with memory hotplug. 3911 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 3912 guest 3913 .ssh_command_l2_1( 3914 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 3915 ) 3916 .unwrap(); 3917 guest 3918 .ssh_command_l1( 3919 "sudo /mnt/ch-remote \ 3920 --api-socket=/tmp/ch_api.sock \ 3921 resize --memory=1073741824", 3922 ) 3923 .unwrap(); 3924 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 3925 }); 3926 3927 let _ = child.kill(); 3928 let output = child.wait_with_output().unwrap(); 3929 3930 cleanup_vfio_network_interfaces(); 3931 3932 handle_child_output(r, &output); 3933 } 3934 3935 #[test] 3936 fn test_direct_kernel_boot_noacpi() { 3937 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3938 let guest = Guest::new(Box::new(focal)); 3939 3940 let kernel_path = direct_kernel_boot_path(); 3941 3942 let mut child = GuestCommand::new(&guest) 3943 .args(&["--cpus", "boot=1"]) 3944 .args(&["--memory", "size=512M"]) 3945 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3946 .args(&[ 3947 "--cmdline", 3948 format!("{} acpi=off", DIRECT_KERNEL_BOOT_CMDLINE).as_str(), 3949 ]) 3950 .default_disks() 3951 .default_net() 3952 .capture_output() 3953 .spawn() 3954 .unwrap(); 3955 3956 let r = std::panic::catch_unwind(|| { 3957 guest.wait_vm_boot(None).unwrap(); 3958 3959 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3960 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3961 }); 3962 3963 let _ = child.kill(); 3964 let output = child.wait_with_output().unwrap(); 3965 3966 handle_child_output(r, &output); 3967 } 3968 3969 #[test] 3970 fn test_reboot() { 3971 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 3972 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3973 3974 vec![Box::new(bionic), Box::new(focal)] 3975 .drain(..) 3976 .for_each(|disk_config| { 3977 let guest = Guest::new(disk_config); 3978 3979 let mut cmd = GuestCommand::new(&guest); 3980 cmd.args(&["--cpus", "boot=1"]) 3981 .args(&["--memory", "size=512M"]) 3982 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3983 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3984 .default_disks() 3985 .default_net() 3986 .capture_output(); 3987 3988 let mut child = cmd.spawn().unwrap(); 3989 3990 let r = std::panic::catch_unwind(|| { 3991 guest.wait_vm_boot(Some(120)).unwrap(); 3992 3993 let fd_count_1 = get_fd_count(child.id()); 3994 guest.reboot_linux(0, Some(120)); 3995 let fd_count_2 = get_fd_count(child.id()); 3996 assert_eq!(fd_count_1, fd_count_2); 3997 3998 guest.ssh_command("sudo shutdown -h now").unwrap(); 3999 }); 4000 4001 let _ = child.wait_timeout(std::time::Duration::from_secs(40)); 4002 let _ = child.kill(); 4003 let output = child.wait_with_output().unwrap(); 4004 handle_child_output(r, &output); 4005 4006 let r = std::panic::catch_unwind(|| { 4007 // Check that the cloud-hypervisor binary actually terminated 4008 assert!(output.status.success()); 4009 }); 4010 4011 handle_child_output(r, &output); 4012 }); 4013 } 4014 4015 #[test] 4016 fn test_virtio_vsock() { 4017 _test_virtio_vsock(false) 4018 } 4019 4020 #[test] 4021 fn test_virtio_vsock_hotplug() { 4022 _test_virtio_vsock(true); 4023 } 4024 4025 #[test] 4026 // Start cloud-hypervisor with no VM parameters, only the API server running. 4027 // From the API: Create a VM, boot it and check that it looks as expected. 4028 fn test_api_create_boot() { 4029 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4030 let guest = Guest::new(Box::new(focal)); 4031 4032 let api_socket = temp_api_path(&guest.tmp_dir); 4033 4034 let mut child = GuestCommand::new(&guest) 4035 .args(&["--api-socket", &api_socket]) 4036 .capture_output() 4037 .spawn() 4038 .unwrap(); 4039 4040 thread::sleep(std::time::Duration::new(1, 0)); 4041 4042 // Verify API server is running 4043 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4044 4045 // Create the VM first 4046 let cpu_count: u8 = 4; 4047 let http_body = guest.api_create_body( 4048 cpu_count, 4049 direct_kernel_boot_path().to_str().unwrap(), 4050 DIRECT_KERNEL_BOOT_CMDLINE, 4051 ); 4052 4053 let temp_config_path = guest.tmp_dir.as_path().join("config"); 4054 std::fs::write(&temp_config_path, http_body).unwrap(); 4055 4056 remote_command( 4057 &api_socket, 4058 "create", 4059 Some(temp_config_path.as_os_str().to_str().unwrap()), 4060 ); 4061 4062 // Then boot it 4063 remote_command(&api_socket, "boot", None); 4064 thread::sleep(std::time::Duration::new(20, 0)); 4065 4066 let r = std::panic::catch_unwind(|| { 4067 // Check that the VM booted as expected 4068 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4069 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4070 }); 4071 4072 let _ = child.kill(); 4073 let output = child.wait_with_output().unwrap(); 4074 4075 handle_child_output(r, &output); 4076 } 4077 4078 #[test] 4079 // Start cloud-hypervisor with no VM parameters, only the API server running. 4080 // From the API: Create a VM, boot it and check it can be shutdown and then 4081 // booted again 4082 fn test_api_shutdown() { 4083 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4084 let guest = Guest::new(Box::new(focal)); 4085 4086 let api_socket = temp_api_path(&guest.tmp_dir); 4087 4088 let mut child = GuestCommand::new(&guest) 4089 .args(&["--api-socket", &api_socket]) 4090 .capture_output() 4091 .spawn() 4092 .unwrap(); 4093 4094 thread::sleep(std::time::Duration::new(1, 0)); 4095 4096 // Verify API server is running 4097 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4098 4099 // Create the VM first 4100 let cpu_count: u8 = 4; 4101 let http_body = guest.api_create_body( 4102 cpu_count, 4103 direct_kernel_boot_path().to_str().unwrap(), 4104 DIRECT_KERNEL_BOOT_CMDLINE, 4105 ); 4106 4107 curl_command( 4108 &api_socket, 4109 "PUT", 4110 "http://localhost/api/v1/vm.create", 4111 Some(&http_body), 4112 ); 4113 4114 // Then boot it 4115 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4116 4117 let r = std::panic::catch_unwind(|| { 4118 guest.wait_vm_boot(None).unwrap(); 4119 4120 // Check that the VM booted as expected 4121 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4122 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4123 4124 // Shutdown without powering off to prevent filesystem corruption 4125 guest.ssh_command("sudo shutdown -H now").unwrap(); 4126 4127 // Then shut it down 4128 curl_command( 4129 &api_socket, 4130 "PUT", 4131 "http://localhost/api/v1/vm.shutdown", 4132 None, 4133 ); 4134 4135 // Then boot it again 4136 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4137 4138 guest.wait_vm_boot(None).unwrap(); 4139 4140 // Check that the VM booted as expected 4141 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4142 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4143 }); 4144 4145 let _ = child.kill(); 4146 let output = child.wait_with_output().unwrap(); 4147 4148 handle_child_output(r, &output); 4149 } 4150 4151 #[test] 4152 // Start cloud-hypervisor with no VM parameters, only the API server running. 4153 // From the API: Create a VM, boot it and check it can be deleted and then recreated 4154 // booted again. 4155 fn test_api_delete() { 4156 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4157 let guest = Guest::new(Box::new(focal)); 4158 4159 let api_socket = temp_api_path(&guest.tmp_dir); 4160 4161 let mut child = GuestCommand::new(&guest) 4162 .args(&["--api-socket", &api_socket]) 4163 .capture_output() 4164 .spawn() 4165 .unwrap(); 4166 4167 thread::sleep(std::time::Duration::new(1, 0)); 4168 4169 // Verify API server is running 4170 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4171 4172 // Create the VM first 4173 let cpu_count: u8 = 4; 4174 let http_body = guest.api_create_body( 4175 cpu_count, 4176 direct_kernel_boot_path().to_str().unwrap(), 4177 DIRECT_KERNEL_BOOT_CMDLINE, 4178 ); 4179 4180 let r = std::panic::catch_unwind(|| { 4181 curl_command( 4182 &api_socket, 4183 "PUT", 4184 "http://localhost/api/v1/vm.create", 4185 Some(&http_body), 4186 ); 4187 4188 // Then boot it 4189 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4190 4191 guest.wait_vm_boot(None).unwrap(); 4192 4193 // Check that the VM booted as expected 4194 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4195 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4196 4197 // Shutdown without powering off to prevent filesystem corruption 4198 guest.ssh_command("sudo shutdown -H now").unwrap(); 4199 4200 // Then delete it 4201 curl_command( 4202 &api_socket, 4203 "PUT", 4204 "http://localhost/api/v1/vm.delete", 4205 None, 4206 ); 4207 4208 curl_command( 4209 &api_socket, 4210 "PUT", 4211 "http://localhost/api/v1/vm.create", 4212 Some(&http_body), 4213 ); 4214 4215 // Then boot it again 4216 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4217 4218 guest.wait_vm_boot(None).unwrap(); 4219 4220 // Check that the VM booted as expected 4221 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4222 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4223 }); 4224 4225 let _ = child.kill(); 4226 let output = child.wait_with_output().unwrap(); 4227 4228 handle_child_output(r, &output); 4229 } 4230 4231 #[test] 4232 // Start cloud-hypervisor with no VM parameters, only the API server running. 4233 // From the API: Create a VM, boot it and check that it looks as expected. 4234 // Then we pause the VM, check that it's no longer available. 4235 // Finally we resume the VM and check that it's available. 4236 fn test_api_pause_resume() { 4237 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4238 let guest = Guest::new(Box::new(focal)); 4239 4240 let api_socket = temp_api_path(&guest.tmp_dir); 4241 4242 let mut child = GuestCommand::new(&guest) 4243 .args(&["--api-socket", &api_socket]) 4244 .capture_output() 4245 .spawn() 4246 .unwrap(); 4247 4248 thread::sleep(std::time::Duration::new(1, 0)); 4249 4250 // Verify API server is running 4251 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4252 4253 // Create the VM first 4254 let cpu_count: u8 = 4; 4255 let http_body = guest.api_create_body( 4256 cpu_count, 4257 direct_kernel_boot_path().to_str().unwrap(), 4258 DIRECT_KERNEL_BOOT_CMDLINE, 4259 ); 4260 curl_command( 4261 &api_socket, 4262 "PUT", 4263 "http://localhost/api/v1/vm.create", 4264 Some(&http_body), 4265 ); 4266 4267 // Then boot it 4268 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4269 thread::sleep(std::time::Duration::new(20, 0)); 4270 4271 let r = std::panic::catch_unwind(|| { 4272 // Check that the VM booted as expected 4273 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4274 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4275 4276 // We now pause the VM 4277 assert!(remote_command(&api_socket, "pause", None)); 4278 4279 // Check pausing again fails 4280 assert!(!remote_command(&api_socket, "pause", None)); 4281 4282 thread::sleep(std::time::Duration::new(2, 0)); 4283 4284 // SSH into the VM should fail 4285 assert!(ssh_command_ip( 4286 "grep -c processor /proc/cpuinfo", 4287 &guest.network.guest_ip, 4288 2, 4289 5 4290 ) 4291 .is_err()); 4292 4293 // Resume the VM 4294 assert!(remote_command(&api_socket, "resume", None)); 4295 4296 // Check resuming again fails 4297 assert!(!remote_command(&api_socket, "resume", None)); 4298 4299 thread::sleep(std::time::Duration::new(2, 0)); 4300 4301 // Now we should be able to SSH back in and get the right number of CPUs 4302 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4303 }); 4304 4305 let _ = child.kill(); 4306 let output = child.wait_with_output().unwrap(); 4307 4308 handle_child_output(r, &output); 4309 } 4310 4311 #[test] 4312 fn test_virtio_iommu() { 4313 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4314 } 4315 4316 #[test] 4317 // We cannot force the software running in the guest to reprogram the BAR 4318 // with some different addresses, but we have a reliable way of testing it 4319 // with a standard Linux kernel. 4320 // By removing a device from the PCI tree, and then rescanning the tree, 4321 // Linux consistently chooses to reorganize the PCI device BARs to other 4322 // locations in the guest address space. 4323 // This test creates a dedicated PCI network device to be checked as being 4324 // properly probed first, then removing it, and adding it again by doing a 4325 // rescan. 4326 fn test_pci_bar_reprogramming() { 4327 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4328 let guest = Guest::new(Box::new(focal)); 4329 4330 #[cfg(target_arch = "x86_64")] 4331 let kernel_path = direct_kernel_boot_path(); 4332 #[cfg(target_arch = "aarch64")] 4333 let kernel_path = edk2_path(); 4334 4335 let mut child = GuestCommand::new(&guest) 4336 .args(&["--cpus", "boot=1"]) 4337 .args(&["--memory", "size=512M"]) 4338 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4339 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4340 .default_disks() 4341 .args(&[ 4342 "--net", 4343 guest.default_net_string().as_str(), 4344 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4345 ]) 4346 .capture_output() 4347 .spawn() 4348 .unwrap(); 4349 4350 let r = std::panic::catch_unwind(|| { 4351 guest.wait_vm_boot(None).unwrap(); 4352 4353 // 2 network interfaces + default localhost ==> 3 interfaces 4354 assert_eq!( 4355 guest 4356 .ssh_command("ip -o link | wc -l") 4357 .unwrap() 4358 .trim() 4359 .parse::<u32>() 4360 .unwrap_or_default(), 4361 3 4362 ); 4363 4364 let init_bar_addr = guest 4365 .ssh_command( 4366 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4367 ) 4368 .unwrap(); 4369 4370 // Remove the PCI device 4371 guest 4372 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4373 .unwrap(); 4374 4375 // Only 1 network interface left + default localhost ==> 2 interfaces 4376 assert_eq!( 4377 guest 4378 .ssh_command("ip -o link | wc -l") 4379 .unwrap() 4380 .trim() 4381 .parse::<u32>() 4382 .unwrap_or_default(), 4383 2 4384 ); 4385 4386 // Remove the PCI device 4387 guest 4388 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4389 .unwrap(); 4390 4391 // Back to 2 network interface + default localhost ==> 3 interfaces 4392 assert_eq!( 4393 guest 4394 .ssh_command("ip -o link | wc -l") 4395 .unwrap() 4396 .trim() 4397 .parse::<u32>() 4398 .unwrap_or_default(), 4399 3 4400 ); 4401 4402 let new_bar_addr = guest 4403 .ssh_command( 4404 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4405 ) 4406 .unwrap(); 4407 4408 // Let's compare the BAR addresses for our virtio-net device. 4409 // They should be different as we expect the BAR reprogramming 4410 // to have happened. 4411 assert_ne!(init_bar_addr, new_bar_addr); 4412 }); 4413 4414 let _ = child.kill(); 4415 let output = child.wait_with_output().unwrap(); 4416 4417 handle_child_output(r, &output); 4418 } 4419 4420 #[test] 4421 fn test_memory_mergeable_off() { 4422 test_memory_mergeable(false) 4423 } 4424 4425 #[test] 4426 #[cfg(target_arch = "x86_64")] 4427 fn test_cpu_hotplug() { 4428 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4429 let guest = Guest::new(Box::new(focal)); 4430 let api_socket = temp_api_path(&guest.tmp_dir); 4431 4432 let kernel_path = direct_kernel_boot_path(); 4433 4434 let mut child = GuestCommand::new(&guest) 4435 .args(&["--cpus", "boot=2,max=4"]) 4436 .args(&["--memory", "size=512M"]) 4437 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4438 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4439 .default_disks() 4440 .default_net() 4441 .args(&["--api-socket", &api_socket]) 4442 .capture_output() 4443 .spawn() 4444 .unwrap(); 4445 4446 let r = std::panic::catch_unwind(|| { 4447 guest.wait_vm_boot(None).unwrap(); 4448 4449 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4450 4451 // Resize the VM 4452 let desired_vcpus = 4; 4453 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4454 4455 guest 4456 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4457 .unwrap(); 4458 guest 4459 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4460 .unwrap(); 4461 thread::sleep(std::time::Duration::new(10, 0)); 4462 assert_eq!( 4463 guest.get_cpu_count().unwrap_or_default(), 4464 u32::from(desired_vcpus) 4465 ); 4466 4467 guest.reboot_linux(0, None); 4468 4469 assert_eq!( 4470 guest.get_cpu_count().unwrap_or_default(), 4471 u32::from(desired_vcpus) 4472 ); 4473 4474 // Resize the VM 4475 let desired_vcpus = 2; 4476 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4477 4478 thread::sleep(std::time::Duration::new(10, 0)); 4479 assert_eq!( 4480 guest.get_cpu_count().unwrap_or_default(), 4481 u32::from(desired_vcpus) 4482 ); 4483 4484 // Resize the VM back up to 4 4485 let desired_vcpus = 4; 4486 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4487 4488 guest 4489 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4490 .unwrap(); 4491 guest 4492 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4493 .unwrap(); 4494 thread::sleep(std::time::Duration::new(10, 0)); 4495 assert_eq!( 4496 guest.get_cpu_count().unwrap_or_default(), 4497 u32::from(desired_vcpus) 4498 ); 4499 }); 4500 4501 let _ = child.kill(); 4502 let output = child.wait_with_output().unwrap(); 4503 4504 handle_child_output(r, &output); 4505 } 4506 4507 #[test] 4508 fn test_memory_hotplug() { 4509 #[cfg(target_arch = "aarch64")] 4510 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4511 #[cfg(target_arch = "x86_64")] 4512 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4513 let focal = UbuntuDiskConfig::new(focal_image); 4514 let guest = Guest::new(Box::new(focal)); 4515 let api_socket = temp_api_path(&guest.tmp_dir); 4516 4517 #[cfg(target_arch = "aarch64")] 4518 let kernel_path = edk2_path(); 4519 #[cfg(target_arch = "x86_64")] 4520 let kernel_path = direct_kernel_boot_path(); 4521 4522 let mut child = GuestCommand::new(&guest) 4523 .args(&["--cpus", "boot=2,max=4"]) 4524 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4525 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4526 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4527 .default_disks() 4528 .default_net() 4529 .args(&["--balloon", "size=0"]) 4530 .args(&["--api-socket", &api_socket]) 4531 .capture_output() 4532 .spawn() 4533 .unwrap(); 4534 4535 let r = std::panic::catch_unwind(|| { 4536 guest.wait_vm_boot(None).unwrap(); 4537 4538 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4539 4540 guest.enable_memory_hotplug(); 4541 4542 // Add RAM to the VM 4543 let desired_ram = 1024 << 20; 4544 resize_command(&api_socket, None, Some(desired_ram), None, None); 4545 4546 thread::sleep(std::time::Duration::new(10, 0)); 4547 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4548 4549 // Use balloon to remove RAM from the VM 4550 let desired_balloon = 512 << 20; 4551 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4552 4553 thread::sleep(std::time::Duration::new(10, 0)); 4554 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4555 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4556 4557 guest.reboot_linux(0, None); 4558 4559 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4560 4561 // Use balloon add RAM to the VM 4562 let desired_balloon = 0; 4563 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4564 4565 thread::sleep(std::time::Duration::new(10, 0)); 4566 4567 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4568 4569 guest.enable_memory_hotplug(); 4570 4571 // Add RAM to the VM 4572 let desired_ram = 2048 << 20; 4573 resize_command(&api_socket, None, Some(desired_ram), None, None); 4574 4575 thread::sleep(std::time::Duration::new(10, 0)); 4576 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4577 4578 // Remove RAM to the VM (only applies after reboot) 4579 let desired_ram = 1024 << 20; 4580 resize_command(&api_socket, None, Some(desired_ram), None, None); 4581 4582 guest.reboot_linux(1, None); 4583 4584 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4585 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4586 }); 4587 4588 let _ = child.kill(); 4589 let output = child.wait_with_output().unwrap(); 4590 4591 handle_child_output(r, &output); 4592 } 4593 4594 #[test] 4595 #[cfg(not(feature = "mshv"))] 4596 fn test_virtio_mem() { 4597 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4598 let guest = Guest::new(Box::new(focal)); 4599 let api_socket = temp_api_path(&guest.tmp_dir); 4600 4601 let kernel_path = direct_kernel_boot_path(); 4602 4603 let mut child = GuestCommand::new(&guest) 4604 .args(&["--cpus", "boot=2,max=4"]) 4605 .args(&[ 4606 "--memory", 4607 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4608 ]) 4609 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4610 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4611 .default_disks() 4612 .default_net() 4613 .args(&["--api-socket", &api_socket]) 4614 .capture_output() 4615 .spawn() 4616 .unwrap(); 4617 4618 let r = std::panic::catch_unwind(|| { 4619 guest.wait_vm_boot(None).unwrap(); 4620 4621 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4622 4623 guest.enable_memory_hotplug(); 4624 4625 // Add RAM to the VM 4626 let desired_ram = 1024 << 20; 4627 resize_command(&api_socket, None, Some(desired_ram), None, None); 4628 4629 thread::sleep(std::time::Duration::new(10, 0)); 4630 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4631 4632 // Add RAM to the VM 4633 let desired_ram = 2048 << 20; 4634 resize_command(&api_socket, None, Some(desired_ram), None, None); 4635 4636 thread::sleep(std::time::Duration::new(10, 0)); 4637 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4638 4639 // Remove RAM from the VM 4640 let desired_ram = 1024 << 20; 4641 resize_command(&api_socket, None, Some(desired_ram), None, None); 4642 4643 thread::sleep(std::time::Duration::new(10, 0)); 4644 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4645 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4646 4647 guest.reboot_linux(0, None); 4648 4649 // Check the amount of memory after reboot is 1GiB 4650 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4651 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4652 4653 // Check we can still resize to 512MiB 4654 let desired_ram = 512 << 20; 4655 resize_command(&api_socket, None, Some(desired_ram), None, None); 4656 thread::sleep(std::time::Duration::new(10, 0)); 4657 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4658 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4659 }); 4660 4661 let _ = child.kill(); 4662 let output = child.wait_with_output().unwrap(); 4663 4664 handle_child_output(r, &output); 4665 } 4666 4667 #[test] 4668 #[cfg(target_arch = "x86_64")] 4669 #[cfg(not(feature = "mshv"))] 4670 // Test both vCPU and memory resizing together 4671 fn test_resize() { 4672 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4673 let guest = Guest::new(Box::new(focal)); 4674 let api_socket = temp_api_path(&guest.tmp_dir); 4675 4676 let kernel_path = direct_kernel_boot_path(); 4677 4678 let mut child = GuestCommand::new(&guest) 4679 .args(&["--cpus", "boot=2,max=4"]) 4680 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4681 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4682 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4683 .default_disks() 4684 .default_net() 4685 .args(&["--api-socket", &api_socket]) 4686 .capture_output() 4687 .spawn() 4688 .unwrap(); 4689 4690 let r = std::panic::catch_unwind(|| { 4691 guest.wait_vm_boot(None).unwrap(); 4692 4693 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4694 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4695 4696 guest.enable_memory_hotplug(); 4697 4698 // Resize the VM 4699 let desired_vcpus = 4; 4700 let desired_ram = 1024 << 20; 4701 resize_command( 4702 &api_socket, 4703 Some(desired_vcpus), 4704 Some(desired_ram), 4705 None, 4706 None, 4707 ); 4708 4709 guest 4710 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4711 .unwrap(); 4712 guest 4713 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4714 .unwrap(); 4715 thread::sleep(std::time::Duration::new(10, 0)); 4716 assert_eq!( 4717 guest.get_cpu_count().unwrap_or_default(), 4718 u32::from(desired_vcpus) 4719 ); 4720 4721 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4722 }); 4723 4724 let _ = child.kill(); 4725 let output = child.wait_with_output().unwrap(); 4726 4727 handle_child_output(r, &output); 4728 } 4729 4730 #[test] 4731 fn test_memory_overhead() { 4732 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4733 let guest = Guest::new(Box::new(focal)); 4734 4735 let kernel_path = direct_kernel_boot_path(); 4736 4737 let guest_memory_size_kb = 512 * 1024; 4738 4739 let mut child = GuestCommand::new(&guest) 4740 .args(&["--cpus", "boot=1"]) 4741 .args(&[ 4742 "--memory", 4743 format!("size={}K", guest_memory_size_kb).as_str(), 4744 ]) 4745 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4746 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4747 .default_disks() 4748 .capture_output() 4749 .spawn() 4750 .unwrap(); 4751 4752 thread::sleep(std::time::Duration::new(20, 0)); 4753 4754 let r = std::panic::catch_unwind(|| { 4755 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 4756 eprintln!( 4757 "Guest memory overhead: {} vs {}", 4758 overhead, MAXIMUM_VMM_OVERHEAD_KB 4759 ); 4760 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 4761 }); 4762 4763 let _ = child.kill(); 4764 let output = child.wait_with_output().unwrap(); 4765 4766 handle_child_output(r, &output); 4767 } 4768 4769 #[test] 4770 fn test_disk_hotplug() { 4771 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4772 let guest = Guest::new(Box::new(focal)); 4773 4774 #[cfg(target_arch = "x86_64")] 4775 let kernel_path = direct_kernel_boot_path(); 4776 #[cfg(target_arch = "aarch64")] 4777 let kernel_path = edk2_path(); 4778 4779 let api_socket = temp_api_path(&guest.tmp_dir); 4780 4781 let mut child = GuestCommand::new(&guest) 4782 .args(&["--api-socket", &api_socket]) 4783 .args(&["--cpus", "boot=1"]) 4784 .args(&["--memory", "size=512M"]) 4785 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4786 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4787 .default_disks() 4788 .default_net() 4789 .capture_output() 4790 .spawn() 4791 .unwrap(); 4792 4793 let r = std::panic::catch_unwind(|| { 4794 guest.wait_vm_boot(None).unwrap(); 4795 4796 // Check /dev/vdc is not there 4797 assert_eq!( 4798 guest 4799 .ssh_command("lsblk | grep -c vdc.*16M || true") 4800 .unwrap() 4801 .trim() 4802 .parse::<u32>() 4803 .unwrap_or(1), 4804 0 4805 ); 4806 4807 // Now let's add the extra disk. 4808 let mut blk_file_path = dirs::home_dir().unwrap(); 4809 blk_file_path.push("workloads"); 4810 blk_file_path.push("blk.img"); 4811 let (cmd_success, cmd_output) = remote_command_w_output( 4812 &api_socket, 4813 "add-disk", 4814 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4815 ); 4816 assert!(cmd_success); 4817 assert!(String::from_utf8_lossy(&cmd_output) 4818 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4819 4820 thread::sleep(std::time::Duration::new(10, 0)); 4821 4822 // Check that /dev/vdc exists and the block size is 16M. 4823 assert_eq!( 4824 guest 4825 .ssh_command("lsblk | grep vdc | grep -c 16M") 4826 .unwrap() 4827 .trim() 4828 .parse::<u32>() 4829 .unwrap_or_default(), 4830 1 4831 ); 4832 // And check the block device can be read. 4833 guest 4834 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4835 .unwrap(); 4836 4837 // Let's remove it the extra disk. 4838 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4839 thread::sleep(std::time::Duration::new(5, 0)); 4840 // And check /dev/vdc is not there 4841 assert_eq!( 4842 guest 4843 .ssh_command("lsblk | grep -c vdc.*16M || true") 4844 .unwrap() 4845 .trim() 4846 .parse::<u32>() 4847 .unwrap_or(1), 4848 0 4849 ); 4850 4851 // And add it back to validate unplug did work correctly. 4852 let (cmd_success, cmd_output) = remote_command_w_output( 4853 &api_socket, 4854 "add-disk", 4855 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4856 ); 4857 assert!(cmd_success); 4858 assert!(String::from_utf8_lossy(&cmd_output) 4859 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4860 4861 thread::sleep(std::time::Duration::new(10, 0)); 4862 4863 // Check that /dev/vdc exists and the block size is 16M. 4864 assert_eq!( 4865 guest 4866 .ssh_command("lsblk | grep vdc | grep -c 16M") 4867 .unwrap() 4868 .trim() 4869 .parse::<u32>() 4870 .unwrap_or_default(), 4871 1 4872 ); 4873 // And check the block device can be read. 4874 guest 4875 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4876 .unwrap(); 4877 4878 // Reboot the VM. 4879 guest.reboot_linux(0, None); 4880 4881 // Check still there after reboot 4882 assert_eq!( 4883 guest 4884 .ssh_command("lsblk | grep vdc | grep -c 16M") 4885 .unwrap() 4886 .trim() 4887 .parse::<u32>() 4888 .unwrap_or_default(), 4889 1 4890 ); 4891 4892 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4893 4894 thread::sleep(std::time::Duration::new(20, 0)); 4895 4896 // Check device has gone away 4897 assert_eq!( 4898 guest 4899 .ssh_command("lsblk | grep -c vdc.*16M || true") 4900 .unwrap() 4901 .trim() 4902 .parse::<u32>() 4903 .unwrap_or(1), 4904 0 4905 ); 4906 4907 guest.reboot_linux(1, None); 4908 4909 // Check device still absent 4910 assert_eq!( 4911 guest 4912 .ssh_command("lsblk | grep -c vdc.*16M || true") 4913 .unwrap() 4914 .trim() 4915 .parse::<u32>() 4916 .unwrap_or(1), 4917 0 4918 ); 4919 }); 4920 4921 let _ = child.kill(); 4922 let output = child.wait_with_output().unwrap(); 4923 4924 handle_child_output(r, &output); 4925 } 4926 4927 #[test] 4928 #[cfg(target_arch = "x86_64")] 4929 fn test_virtio_block_topology() { 4930 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4931 let guest = Guest::new(Box::new(focal)); 4932 4933 let kernel_path = direct_kernel_boot_path(); 4934 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 4935 4936 Command::new("qemu-img") 4937 .args(&[ 4938 "create", 4939 "-f", 4940 "raw", 4941 test_disk_path.to_str().unwrap(), 4942 "16M", 4943 ]) 4944 .output() 4945 .expect("qemu-img command failed"); 4946 let out = Command::new("losetup") 4947 .args(&[ 4948 "--show", 4949 "--find", 4950 "--sector-size=4096", 4951 test_disk_path.to_str().unwrap(), 4952 ]) 4953 .output() 4954 .expect("failed to create loop device") 4955 .stdout; 4956 let _tmp = String::from_utf8_lossy(&out); 4957 let loop_dev = _tmp.trim(); 4958 4959 let mut child = GuestCommand::new(&guest) 4960 .args(&["--cpus", "boot=1"]) 4961 .args(&["--memory", "size=512M"]) 4962 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4963 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4964 .args(&[ 4965 "--disk", 4966 format!( 4967 "path={}", 4968 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4969 ) 4970 .as_str(), 4971 format!( 4972 "path={}", 4973 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4974 ) 4975 .as_str(), 4976 format!("path={}", loop_dev).as_str(), 4977 ]) 4978 .default_net() 4979 .capture_output() 4980 .spawn() 4981 .unwrap(); 4982 4983 let r = std::panic::catch_unwind(|| { 4984 guest.wait_vm_boot(None).unwrap(); 4985 4986 // MIN-IO column 4987 assert_eq!( 4988 guest 4989 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 4990 .unwrap() 4991 .trim() 4992 .parse::<u32>() 4993 .unwrap_or_default(), 4994 4096 4995 ); 4996 // PHY-SEC column 4997 assert_eq!( 4998 guest 4999 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5000 .unwrap() 5001 .trim() 5002 .parse::<u32>() 5003 .unwrap_or_default(), 5004 4096 5005 ); 5006 // LOG-SEC column 5007 assert_eq!( 5008 guest 5009 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5010 .unwrap() 5011 .trim() 5012 .parse::<u32>() 5013 .unwrap_or_default(), 5014 4096 5015 ); 5016 }); 5017 5018 let _ = child.kill(); 5019 let output = child.wait_with_output().unwrap(); 5020 5021 handle_child_output(r, &output); 5022 5023 Command::new("losetup") 5024 .args(&["-d", loop_dev]) 5025 .output() 5026 .expect("loop device not found"); 5027 } 5028 5029 #[test] 5030 fn test_virtio_balloon_deflate_on_oom() { 5031 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5032 let guest = Guest::new(Box::new(focal)); 5033 5034 let kernel_path = direct_kernel_boot_path(); 5035 5036 let api_socket = temp_api_path(&guest.tmp_dir); 5037 5038 //Let's start a 4G guest with balloon occupied 2G memory 5039 let mut child = GuestCommand::new(&guest) 5040 .args(&["--api-socket", &api_socket]) 5041 .args(&["--cpus", "boot=1"]) 5042 .args(&["--memory", "size=4G"]) 5043 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5044 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5045 .args(&["--balloon", "size=2G,deflate_on_oom=on"]) 5046 .default_disks() 5047 .default_net() 5048 .capture_output() 5049 .spawn() 5050 .unwrap(); 5051 5052 let r = std::panic::catch_unwind(|| { 5053 guest.wait_vm_boot(None).unwrap(); 5054 5055 // Wait for balloon memory's initialization and check its size. 5056 // The virtio-balloon driver might take a few seconds to report the 5057 // balloon effective size back to the VMM. 5058 thread::sleep(std::time::Duration::new(20, 0)); 5059 5060 let orig_balloon = balloon_size(&api_socket); 5061 println!("The original balloon memory size is {} bytes", orig_balloon); 5062 assert!(orig_balloon == 2147483648); 5063 5064 // Two steps to verify if the 'deflate_on_oom' parameter works. 5065 // 1st: run a command in guest to eat up memory heavily, which 5066 // will consume much more memory than $(total_mem - balloon_size) 5067 // to trigger an oom. 5068 guest 5069 .ssh_command("stress --vm 25 --vm-keep --vm-bytes 1G --timeout 20") 5070 .unwrap(); 5071 5072 // Give some time for the OOM to happen in the guest and be reported 5073 // back to the host. 5074 thread::sleep(std::time::Duration::new(20, 0)); 5075 5076 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5077 let deflated_balloon = balloon_size(&api_socket); 5078 println!( 5079 "After deflating, balloon memory size is {} bytes", 5080 deflated_balloon 5081 ); 5082 // Verify the balloon size deflated 5083 assert!(deflated_balloon < 2147483648); 5084 }); 5085 5086 let _ = child.kill(); 5087 let output = child.wait_with_output().unwrap(); 5088 5089 handle_child_output(r, &output); 5090 } 5091 5092 #[test] 5093 fn test_virtio_balloon_free_page_reporting() { 5094 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5095 let guest = Guest::new(Box::new(focal)); 5096 5097 //Let's start a 4G guest with balloon occupied 2G memory 5098 let mut child = GuestCommand::new(&guest) 5099 .args(&["--cpus", "boot=1"]) 5100 .args(&["--memory", "size=4G"]) 5101 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5102 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5103 .args(&["--balloon", "size=0,free_page_reporting=on"]) 5104 .default_disks() 5105 .default_net() 5106 .capture_output() 5107 .spawn() 5108 .unwrap(); 5109 5110 let pid = child.id(); 5111 let r = std::panic::catch_unwind(|| { 5112 guest.wait_vm_boot(None).unwrap(); 5113 5114 // Check the initial RSS is less than 1GiB 5115 let rss = process_rss_kib(pid); 5116 println!("RSS {} < 1048576", rss); 5117 assert!(rss < 1048576); 5118 5119 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5120 // seconds 5121 let guest_ip = guest.network.guest_ip.clone(); 5122 thread::spawn(move || { 5123 ssh_command_ip( 5124 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5125 &guest_ip, 5126 DEFAULT_SSH_RETRIES, 5127 DEFAULT_SSH_TIMEOUT, 5128 ) 5129 .unwrap(); 5130 }); 5131 5132 // Wait for 50 seconds to make sure the stress command is consuming 5133 // the expected amount of memory. 5134 thread::sleep(std::time::Duration::new(50, 0)); 5135 let rss = process_rss_kib(pid); 5136 println!("RSS {} >= 2097152", rss); 5137 assert!(rss >= 2097152); 5138 5139 // Wait for an extra minute to make sure the stress command has 5140 // completed and that the guest reported the free pages to the VMM 5141 // through the virtio-balloon device. We expect the RSS to be under 5142 // 2GiB. 5143 thread::sleep(std::time::Duration::new(60, 0)); 5144 let rss = process_rss_kib(pid); 5145 println!("RSS {} < 2097152", rss); 5146 assert!(rss < 2097152); 5147 }); 5148 5149 let _ = child.kill(); 5150 let output = child.wait_with_output().unwrap(); 5151 5152 handle_child_output(r, &output); 5153 } 5154 5155 #[test] 5156 fn test_pmem_hotplug() { 5157 _test_pmem_hotplug(None) 5158 } 5159 5160 #[test] 5161 fn test_pmem_multi_segment_hotplug() { 5162 _test_pmem_hotplug(Some(15)) 5163 } 5164 5165 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5166 #[cfg(target_arch = "aarch64")] 5167 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5168 #[cfg(target_arch = "x86_64")] 5169 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5170 let focal = UbuntuDiskConfig::new(focal_image); 5171 let guest = Guest::new(Box::new(focal)); 5172 5173 #[cfg(target_arch = "x86_64")] 5174 let kernel_path = direct_kernel_boot_path(); 5175 #[cfg(target_arch = "aarch64")] 5176 let kernel_path = edk2_path(); 5177 5178 let api_socket = temp_api_path(&guest.tmp_dir); 5179 5180 let mut cmd = GuestCommand::new(&guest); 5181 5182 cmd.args(&["--api-socket", &api_socket]) 5183 .args(&["--cpus", "boot=1"]) 5184 .args(&["--memory", "size=512M"]) 5185 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5186 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5187 .default_disks() 5188 .default_net() 5189 .capture_output(); 5190 5191 if pci_segment.is_some() { 5192 cmd.args(&["--platform", "num_pci_segments=16"]); 5193 } 5194 5195 let mut child = cmd.spawn().unwrap(); 5196 5197 let r = std::panic::catch_unwind(|| { 5198 guest.wait_vm_boot(None).unwrap(); 5199 5200 // Check /dev/pmem0 is not there 5201 assert_eq!( 5202 guest 5203 .ssh_command("lsblk | grep -c pmem0 || true") 5204 .unwrap() 5205 .trim() 5206 .parse::<u32>() 5207 .unwrap_or(1), 5208 0 5209 ); 5210 5211 let pmem_temp_file = TempFile::new().unwrap(); 5212 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5213 let (cmd_success, cmd_output) = remote_command_w_output( 5214 &api_socket, 5215 "add-pmem", 5216 Some(&format!( 5217 "file={},id=test0{}", 5218 pmem_temp_file.as_path().to_str().unwrap(), 5219 if let Some(pci_segment) = pci_segment { 5220 format!(",pci_segment={}", pci_segment) 5221 } else { 5222 "".to_owned() 5223 } 5224 )), 5225 ); 5226 assert!(cmd_success); 5227 if let Some(pci_segment) = pci_segment { 5228 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5229 "{{\"id\":\"test0\",\"bdf\":\"{:04x}:00:01.0\"}}", 5230 pci_segment 5231 ))); 5232 } else { 5233 assert!(String::from_utf8_lossy(&cmd_output) 5234 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5235 } 5236 5237 // Check that /dev/pmem0 exists and the block size is 128M 5238 assert_eq!( 5239 guest 5240 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5241 .unwrap() 5242 .trim() 5243 .parse::<u32>() 5244 .unwrap_or_default(), 5245 1 5246 ); 5247 5248 guest.reboot_linux(0, None); 5249 5250 // Check still there after reboot 5251 assert_eq!( 5252 guest 5253 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5254 .unwrap() 5255 .trim() 5256 .parse::<u32>() 5257 .unwrap_or_default(), 5258 1 5259 ); 5260 5261 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5262 5263 thread::sleep(std::time::Duration::new(20, 0)); 5264 5265 // Check device has gone away 5266 assert_eq!( 5267 guest 5268 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5269 .unwrap() 5270 .trim() 5271 .parse::<u32>() 5272 .unwrap_or(1), 5273 0 5274 ); 5275 5276 guest.reboot_linux(1, None); 5277 5278 // Check still absent after reboot 5279 assert_eq!( 5280 guest 5281 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5282 .unwrap() 5283 .trim() 5284 .parse::<u32>() 5285 .unwrap_or(1), 5286 0 5287 ); 5288 }); 5289 5290 let _ = child.kill(); 5291 let output = child.wait_with_output().unwrap(); 5292 5293 handle_child_output(r, &output); 5294 } 5295 5296 #[test] 5297 fn test_net_hotplug() { 5298 _test_net_hotplug(None) 5299 } 5300 5301 #[test] 5302 fn test_net_multi_segment_hotplug() { 5303 _test_net_hotplug(Some(15)) 5304 } 5305 5306 fn _test_net_hotplug(pci_segment: Option<u16>) { 5307 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5308 let guest = Guest::new(Box::new(focal)); 5309 5310 #[cfg(target_arch = "x86_64")] 5311 let kernel_path = direct_kernel_boot_path(); 5312 #[cfg(target_arch = "aarch64")] 5313 let kernel_path = edk2_path(); 5314 5315 let api_socket = temp_api_path(&guest.tmp_dir); 5316 5317 // Boot without network 5318 let mut cmd = GuestCommand::new(&guest); 5319 5320 cmd.args(&["--api-socket", &api_socket]) 5321 .args(&["--cpus", "boot=1"]) 5322 .args(&["--memory", "size=512M"]) 5323 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5324 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5325 .default_disks() 5326 .capture_output(); 5327 5328 if pci_segment.is_some() { 5329 cmd.args(&["--platform", "num_pci_segments=16"]); 5330 } 5331 5332 let mut child = cmd.spawn().unwrap(); 5333 5334 thread::sleep(std::time::Duration::new(20, 0)); 5335 5336 let r = std::panic::catch_unwind(|| { 5337 // Add network 5338 let (cmd_success, cmd_output) = remote_command_w_output( 5339 &api_socket, 5340 "add-net", 5341 Some( 5342 format!( 5343 "{}{},id=test0", 5344 guest.default_net_string(), 5345 if let Some(pci_segment) = pci_segment { 5346 format!(",pci_segment={}", pci_segment) 5347 } else { 5348 "".to_owned() 5349 } 5350 ) 5351 .as_str(), 5352 ), 5353 ); 5354 assert!(cmd_success); 5355 5356 if let Some(pci_segment) = pci_segment { 5357 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5358 "{{\"id\":\"test0\",\"bdf\":\"{:04x}:00:01.0\"}}", 5359 pci_segment 5360 ))); 5361 } else { 5362 assert!(String::from_utf8_lossy(&cmd_output) 5363 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5364 } 5365 5366 thread::sleep(std::time::Duration::new(5, 0)); 5367 5368 // 1 network interfaces + default localhost ==> 2 interfaces 5369 assert_eq!( 5370 guest 5371 .ssh_command("ip -o link | wc -l") 5372 .unwrap() 5373 .trim() 5374 .parse::<u32>() 5375 .unwrap_or_default(), 5376 2 5377 ); 5378 5379 // Remove network 5380 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5381 thread::sleep(std::time::Duration::new(5, 0)); 5382 5383 let (cmd_success, cmd_output) = remote_command_w_output( 5384 &api_socket, 5385 "add-net", 5386 Some( 5387 format!( 5388 "{}{},id=test1", 5389 guest.default_net_string(), 5390 if let Some(pci_segment) = pci_segment { 5391 format!(",pci_segment={}", pci_segment) 5392 } else { 5393 "".to_owned() 5394 } 5395 ) 5396 .as_str(), 5397 ), 5398 ); 5399 assert!(cmd_success); 5400 5401 if let Some(pci_segment) = pci_segment { 5402 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5403 "{{\"id\":\"test1\",\"bdf\":\"{:04x}:00:01.0\"}}", 5404 pci_segment 5405 ))); 5406 } else { 5407 assert!(String::from_utf8_lossy(&cmd_output) 5408 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5409 } 5410 5411 thread::sleep(std::time::Duration::new(5, 0)); 5412 5413 // 1 network interfaces + default localhost ==> 2 interfaces 5414 assert_eq!( 5415 guest 5416 .ssh_command("ip -o link | wc -l") 5417 .unwrap() 5418 .trim() 5419 .parse::<u32>() 5420 .unwrap_or_default(), 5421 2 5422 ); 5423 5424 guest.reboot_linux(0, None); 5425 5426 // Check still there after reboot 5427 // 1 network interfaces + default localhost ==> 2 interfaces 5428 assert_eq!( 5429 guest 5430 .ssh_command("ip -o link | wc -l") 5431 .unwrap() 5432 .trim() 5433 .parse::<u32>() 5434 .unwrap_or_default(), 5435 2 5436 ); 5437 }); 5438 5439 let _ = child.kill(); 5440 let output = child.wait_with_output().unwrap(); 5441 5442 handle_child_output(r, &output); 5443 } 5444 5445 #[test] 5446 fn test_initramfs() { 5447 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5448 let guest = Guest::new(Box::new(focal)); 5449 let mut workload_path = dirs::home_dir().unwrap(); 5450 workload_path.push("workloads"); 5451 5452 #[cfg(target_arch = "x86_64")] 5453 let mut kernels = vec![direct_kernel_boot_path()]; 5454 #[cfg(target_arch = "aarch64")] 5455 let kernels = vec![direct_kernel_boot_path()]; 5456 5457 #[cfg(target_arch = "x86_64")] 5458 { 5459 let mut pvh_kernel_path = workload_path.clone(); 5460 pvh_kernel_path.push("vmlinux"); 5461 kernels.push(pvh_kernel_path); 5462 } 5463 5464 let mut initramfs_path = workload_path; 5465 initramfs_path.push("alpine_initramfs.img"); 5466 5467 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5468 let cmdline = format!("console=hvc0 quiet TEST_STRING={}", test_string); 5469 5470 kernels.iter().for_each(|k_path| { 5471 let mut child = GuestCommand::new(&guest) 5472 .args(&["--kernel", k_path.to_str().unwrap()]) 5473 .args(&["--initramfs", initramfs_path.to_str().unwrap()]) 5474 .args(&["--cmdline", &cmdline]) 5475 .capture_output() 5476 .spawn() 5477 .unwrap(); 5478 5479 thread::sleep(std::time::Duration::new(20, 0)); 5480 5481 let _ = child.kill(); 5482 let output = child.wait_with_output().unwrap(); 5483 5484 let r = std::panic::catch_unwind(|| { 5485 let s = String::from_utf8_lossy(&output.stdout); 5486 5487 assert_ne!(s.lines().position(|line| line == test_string), None); 5488 }); 5489 5490 handle_child_output(r, &output); 5491 }); 5492 } 5493 5494 // One thing to note about this test. The virtio-net device is heavily used 5495 // through each ssh command. There's no need to perform a dedicated test to 5496 // verify the migration went well for virtio-net. 5497 #[test] 5498 fn test_snapshot_restore() { 5499 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5500 let guest = Guest::new(Box::new(focal)); 5501 let kernel_path = direct_kernel_boot_path(); 5502 5503 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 5504 5505 let net_id = "net123"; 5506 let net_params = format!( 5507 "id={},tap=,mac={},ip={},mask=255.255.255.0", 5508 net_id, guest.network.guest_mac, guest.network.host_ip 5509 ); 5510 5511 let cloudinit_params = format!( 5512 "path={},iommu=on", 5513 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5514 ); 5515 5516 let socket = temp_vsock_path(&guest.tmp_dir); 5517 let event_path = temp_event_monitor_path(&guest.tmp_dir); 5518 5519 let mut child = GuestCommand::new(&guest) 5520 .args(&["--api-socket", &api_socket_source]) 5521 .args(&["--event-monitor", format!("path={}", event_path).as_str()]) 5522 .args(&["--cpus", "boot=4"]) 5523 .args(&[ 5524 "--memory", 5525 "size=4G,hotplug_method=virtio-mem,hotplug_size=32G", 5526 ]) 5527 .args(&["--balloon", "size=0"]) 5528 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5529 .args(&[ 5530 "--disk", 5531 format!( 5532 "path={}", 5533 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5534 ) 5535 .as_str(), 5536 cloudinit_params.as_str(), 5537 ]) 5538 .args(&["--net", net_params.as_str()]) 5539 .args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]) 5540 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5541 .capture_output() 5542 .spawn() 5543 .unwrap(); 5544 5545 let console_text = String::from("On a branch floating down river a cricket, singing."); 5546 // Create the snapshot directory 5547 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 5548 5549 let r = std::panic::catch_unwind(|| { 5550 guest.wait_vm_boot(None).unwrap(); 5551 5552 // Check the number of vCPUs 5553 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5554 // Check the guest RAM 5555 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5556 // Increase guest RAM with virtio-mem 5557 resize_command( 5558 &api_socket_source, 5559 None, 5560 Some(6 << 30), 5561 None, 5562 Some(&event_path), 5563 ); 5564 thread::sleep(std::time::Duration::new(5, 0)); 5565 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5566 // Use balloon to remove RAM from the VM 5567 resize_command( 5568 &api_socket_source, 5569 None, 5570 None, 5571 Some(1 << 30), 5572 Some(&event_path), 5573 ); 5574 thread::sleep(std::time::Duration::new(5, 0)); 5575 let total_memory = guest.get_total_memory().unwrap_or_default(); 5576 assert!(total_memory > 4_800_000); 5577 assert!(total_memory < 5_760_000); 5578 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 5579 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5580 5581 // x86_64: We check that removing and adding back the virtio-net device 5582 // does not break the snapshot/restore support for virtio-pci. 5583 // This is an important thing to test as the hotplug will 5584 // trigger a PCI BAR reprogramming, which is a good way of 5585 // checking if the stored resources are correctly restored. 5586 // Unplug the virtio-net device 5587 // AArch64: Device hotplug is currently not supported, skipping here. 5588 #[cfg(target_arch = "x86_64")] 5589 { 5590 assert!(remote_command( 5591 &api_socket_source, 5592 "remove-device", 5593 Some(net_id), 5594 )); 5595 thread::sleep(std::time::Duration::new(10, 0)); 5596 let latest_events = [&MetaEvent { 5597 event: "device-removed".to_string(), 5598 device_id: Some(net_id.to_string()), 5599 }]; 5600 assert!(check_latest_events_exact(&latest_events, &event_path)); 5601 5602 // Plug the virtio-net device again 5603 assert!(remote_command( 5604 &api_socket_source, 5605 "add-net", 5606 Some(net_params.as_str()), 5607 )); 5608 thread::sleep(std::time::Duration::new(10, 0)); 5609 } 5610 5611 // Pause the VM 5612 assert!(remote_command(&api_socket_source, "pause", None)); 5613 let latest_events = [ 5614 &MetaEvent { 5615 event: "pausing".to_string(), 5616 device_id: None, 5617 }, 5618 &MetaEvent { 5619 event: "paused".to_string(), 5620 device_id: None, 5621 }, 5622 ]; 5623 assert!(check_latest_events_exact(&latest_events, &event_path)); 5624 5625 // Take a snapshot from the VM 5626 assert!(remote_command( 5627 &api_socket_source, 5628 "snapshot", 5629 Some(format!("file://{}", snapshot_dir).as_str()), 5630 )); 5631 5632 // Wait to make sure the snapshot is completed 5633 thread::sleep(std::time::Duration::new(10, 0)); 5634 5635 let latest_events = [ 5636 &MetaEvent { 5637 event: "snapshotting".to_string(), 5638 device_id: None, 5639 }, 5640 &MetaEvent { 5641 event: "snapshotted".to_string(), 5642 device_id: None, 5643 }, 5644 ]; 5645 assert!(check_latest_events_exact(&latest_events, &event_path)); 5646 }); 5647 5648 // Shutdown the source VM and check console output 5649 let _ = child.kill(); 5650 let output = child.wait_with_output().unwrap(); 5651 handle_child_output(r, &output); 5652 5653 let r = std::panic::catch_unwind(|| { 5654 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5655 }); 5656 5657 handle_child_output(r, &output); 5658 5659 // Remove the vsock socket file. 5660 Command::new("rm") 5661 .arg("-f") 5662 .arg(socket.as_str()) 5663 .output() 5664 .unwrap(); 5665 5666 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 5667 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 5668 5669 // Restore the VM from the snapshot 5670 let mut child = GuestCommand::new(&guest) 5671 .args(&["--api-socket", &api_socket_restored]) 5672 .args(&[ 5673 "--event-monitor", 5674 format!("path={}", event_path_restored).as_str(), 5675 ]) 5676 .args(&[ 5677 "--restore", 5678 format!("source_url=file://{}", snapshot_dir).as_str(), 5679 ]) 5680 .capture_output() 5681 .spawn() 5682 .unwrap(); 5683 5684 // Wait for the VM to be restored 5685 thread::sleep(std::time::Duration::new(10, 0)); 5686 let expected_events = [ 5687 &MetaEvent { 5688 event: "starting".to_string(), 5689 device_id: None, 5690 }, 5691 &MetaEvent { 5692 event: "restoring".to_string(), 5693 device_id: None, 5694 }, 5695 ]; 5696 assert!(check_sequential_events_exact( 5697 &expected_events, 5698 &event_path_restored 5699 )); 5700 let latest_events = [&MetaEvent { 5701 event: "restored".to_string(), 5702 device_id: None, 5703 }]; 5704 assert!(check_latest_events_exact( 5705 &latest_events, 5706 &event_path_restored 5707 )); 5708 5709 let r = std::panic::catch_unwind(|| { 5710 // Resume the VM 5711 assert!(remote_command(&api_socket_restored, "resume", None)); 5712 let latest_events = [ 5713 &MetaEvent { 5714 event: "resuming".to_string(), 5715 device_id: None, 5716 }, 5717 &MetaEvent { 5718 event: "resumed".to_string(), 5719 device_id: None, 5720 }, 5721 ]; 5722 assert!(check_latest_events_exact( 5723 &latest_events, 5724 &event_path_restored 5725 )); 5726 5727 // Perform same checks to validate VM has been properly restored 5728 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5729 let total_memory = guest.get_total_memory().unwrap_or_default(); 5730 assert!(total_memory > 4_800_000); 5731 assert!(total_memory < 5_760_000); 5732 // Deflate balloon to restore entire RAM to the VM 5733 resize_command(&api_socket_restored, None, None, Some(0), None); 5734 thread::sleep(std::time::Duration::new(5, 0)); 5735 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5736 // Decrease guest RAM with virtio-mem 5737 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 5738 thread::sleep(std::time::Duration::new(5, 0)); 5739 let total_memory = guest.get_total_memory().unwrap_or_default(); 5740 assert!(total_memory > 4_800_000); 5741 assert!(total_memory < 5_760_000); 5742 5743 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5744 }); 5745 // Shutdown the target VM and check console output 5746 let _ = child.kill(); 5747 let output = child.wait_with_output().unwrap(); 5748 handle_child_output(r, &output); 5749 5750 let r = std::panic::catch_unwind(|| { 5751 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5752 }); 5753 5754 handle_child_output(r, &output); 5755 } 5756 5757 #[test] 5758 fn test_counters() { 5759 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5760 let guest = Guest::new(Box::new(focal)); 5761 let api_socket = temp_api_path(&guest.tmp_dir); 5762 5763 let mut cmd = GuestCommand::new(&guest); 5764 cmd.args(&["--cpus", "boot=1"]) 5765 .args(&["--memory", "size=512M"]) 5766 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5767 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5768 .default_disks() 5769 .args(&["--net", guest.default_net_string().as_str()]) 5770 .args(&["--api-socket", &api_socket]) 5771 .capture_output(); 5772 5773 let mut child = cmd.spawn().unwrap(); 5774 5775 let r = std::panic::catch_unwind(|| { 5776 guest.wait_vm_boot(None).unwrap(); 5777 5778 let orig_counters = get_counters(&api_socket); 5779 guest 5780 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 5781 .unwrap(); 5782 5783 let new_counters = get_counters(&api_socket); 5784 5785 // Check that all the counters have increased 5786 assert!(new_counters > orig_counters); 5787 }); 5788 5789 let _ = child.kill(); 5790 let output = child.wait_with_output().unwrap(); 5791 5792 handle_child_output(r, &output); 5793 } 5794 5795 #[test] 5796 #[cfg(feature = "guest_debug")] 5797 fn test_coredump() { 5798 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5799 let guest = Guest::new(Box::new(focal)); 5800 let api_socket = temp_api_path(&guest.tmp_dir); 5801 5802 let mut cmd = GuestCommand::new(&guest); 5803 cmd.args(&["--cpus", "boot=4"]) 5804 .args(&["--memory", "size=4G"]) 5805 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 5806 .default_disks() 5807 .args(&["--net", guest.default_net_string().as_str()]) 5808 .args(&["--api-socket", &api_socket]) 5809 .capture_output(); 5810 5811 let mut child = cmd.spawn().unwrap(); 5812 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 5813 5814 let r = std::panic::catch_unwind(|| { 5815 guest.wait_vm_boot(None).unwrap(); 5816 5817 assert!(remote_command(&api_socket, "pause", None)); 5818 5819 assert!(remote_command( 5820 &api_socket, 5821 "coredump", 5822 Some(format!("file://{}", vmcore_file).as_str()), 5823 )); 5824 5825 // the num of CORE notes should equals to vcpu 5826 let readelf_core_num_cmd = format!( 5827 "readelf --all {} |grep CORE |grep -v Type |wc -l", 5828 vmcore_file 5829 ); 5830 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 5831 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 5832 5833 // the num of QEMU notes should equals to vcpu 5834 let readelf_vmm_num_cmd = format!("readelf --all {} |grep QEMU |wc -l", vmcore_file); 5835 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 5836 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 5837 }); 5838 5839 let _ = child.kill(); 5840 let output = child.wait_with_output().unwrap(); 5841 5842 handle_child_output(r, &output); 5843 } 5844 5845 #[test] 5846 fn test_watchdog() { 5847 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5848 let guest = Guest::new(Box::new(focal)); 5849 let api_socket = temp_api_path(&guest.tmp_dir); 5850 5851 let kernel_path = direct_kernel_boot_path(); 5852 5853 let mut cmd = GuestCommand::new(&guest); 5854 cmd.args(&["--cpus", "boot=1"]) 5855 .args(&["--memory", "size=512M"]) 5856 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5857 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5858 .default_disks() 5859 .args(&["--net", guest.default_net_string().as_str()]) 5860 .args(&["--watchdog"]) 5861 .args(&["--api-socket", &api_socket]) 5862 .capture_output(); 5863 5864 let mut child = cmd.spawn().unwrap(); 5865 5866 let r = std::panic::catch_unwind(|| { 5867 guest.wait_vm_boot(None).unwrap(); 5868 let mut current_reboot_count = 1; 5869 enable_guest_watchdog(&guest, &mut current_reboot_count); 5870 check_guest_watchdog_no_reboot(&guest, ¤t_reboot_count); 5871 check_guest_watchdog_one_reboot(&guest, &api_socket, &mut current_reboot_count); 5872 }); 5873 5874 let _ = child.kill(); 5875 let output = child.wait_with_output().unwrap(); 5876 5877 handle_child_output(r, &output); 5878 } 5879 5880 #[test] 5881 fn test_tap_from_fd() { 5882 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5883 let guest = Guest::new(Box::new(focal)); 5884 let kernel_path = direct_kernel_boot_path(); 5885 5886 // Create a TAP interface with multi-queue enabled 5887 let num_queue_pairs: usize = 2; 5888 5889 use std::str::FromStr; 5890 let taps = net_util::open_tap( 5891 Some("chtap0"), 5892 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 5893 None, 5894 &mut None, 5895 num_queue_pairs, 5896 Some(libc::O_RDWR | libc::O_NONBLOCK), 5897 ) 5898 .unwrap(); 5899 5900 let mut child = GuestCommand::new(&guest) 5901 .args(&["--cpus", &format!("boot={}", num_queue_pairs)]) 5902 .args(&["--memory", "size=512M"]) 5903 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5904 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5905 .default_disks() 5906 .args(&[ 5907 "--net", 5908 &format!( 5909 "fd=[{},{}],mac={},num_queues={}", 5910 taps[0].as_raw_fd(), 5911 taps[1].as_raw_fd(), 5912 guest.network.guest_mac, 5913 num_queue_pairs * 2 5914 ), 5915 ]) 5916 .capture_output() 5917 .spawn() 5918 .unwrap(); 5919 5920 let r = std::panic::catch_unwind(|| { 5921 guest.wait_vm_boot(None).unwrap(); 5922 5923 assert_eq!( 5924 guest 5925 .ssh_command("ip -o link | wc -l") 5926 .unwrap() 5927 .trim() 5928 .parse::<u32>() 5929 .unwrap_or_default(), 5930 2 5931 ); 5932 5933 guest.reboot_linux(0, None); 5934 5935 assert_eq!( 5936 guest 5937 .ssh_command("ip -o link | wc -l") 5938 .unwrap() 5939 .trim() 5940 .parse::<u32>() 5941 .unwrap_or_default(), 5942 2 5943 ); 5944 }); 5945 5946 let _ = child.kill(); 5947 let output = child.wait_with_output().unwrap(); 5948 5949 handle_child_output(r, &output); 5950 } 5951 5952 // By design, a guest VM won't be able to connect to the host 5953 // machine when using a macvtap network interface (while it can 5954 // communicate externally). As a workaround, this integration 5955 // test creates two macvtap interfaces in 'bridge' mode on the 5956 // same physical net interface, one for the guest and one for 5957 // the host. With additional setup on the IP address and the 5958 // routing table, it enables the communications between the 5959 // guest VM and the host machine. 5960 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 5961 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 5962 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5963 let guest = Guest::new(Box::new(focal)); 5964 let api_socket = temp_api_path(&guest.tmp_dir); 5965 5966 #[cfg(target_arch = "x86_64")] 5967 let kernel_path = direct_kernel_boot_path(); 5968 #[cfg(target_arch = "aarch64")] 5969 let kernel_path = edk2_path(); 5970 5971 let phy_net = "eth0"; 5972 5973 // Create a macvtap interface for the guest VM to use 5974 assert!(exec_host_command_status(&format!( 5975 "sudo ip link add link {} name {} type macvtap mod bridge", 5976 phy_net, guest_macvtap_name 5977 )) 5978 .success()); 5979 assert!(exec_host_command_status(&format!( 5980 "sudo ip link set {} address {} up", 5981 guest_macvtap_name, guest.network.guest_mac 5982 )) 5983 .success()); 5984 assert!( 5985 exec_host_command_status(&format!("sudo ip link show {}", guest_macvtap_name)) 5986 .success() 5987 ); 5988 5989 let tap_index = 5990 fs::read_to_string(&format!("/sys/class/net/{}/ifindex", guest_macvtap_name)).unwrap(); 5991 let tap_device = format!("/dev/tap{}", tap_index.trim()); 5992 5993 assert!( 5994 exec_host_command_status(&format!("sudo chown $UID.$UID {}", tap_device)).success() 5995 ); 5996 5997 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 5998 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 5999 assert!(tap_fd1 > 0); 6000 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6001 assert!(tap_fd2 > 0); 6002 6003 // Create a macvtap on the same physical net interface for 6004 // the host machine to use 6005 assert!(exec_host_command_status(&format!( 6006 "sudo ip link add link {} name {} type macvtap mod bridge", 6007 phy_net, host_macvtap_name 6008 )) 6009 .success()); 6010 // Use default mask "255.255.255.0" 6011 assert!(exec_host_command_status(&format!( 6012 "sudo ip address add {}/24 dev {}", 6013 guest.network.host_ip, host_macvtap_name 6014 )) 6015 .success()); 6016 assert!(exec_host_command_status(&format!( 6017 "sudo ip link set dev {} up", 6018 host_macvtap_name 6019 )) 6020 .success()); 6021 6022 let mut guest_command = GuestCommand::new(&guest); 6023 guest_command 6024 .args(&["--cpus", "boot=2"]) 6025 .args(&["--memory", "size=512M"]) 6026 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6027 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6028 .default_disks() 6029 .args(&["--api-socket", &api_socket]); 6030 6031 let net_params = format!( 6032 "fd=[{},{}],mac={},num_queues=4", 6033 tap_fd1, tap_fd2, guest.network.guest_mac 6034 ); 6035 6036 if !hotplug { 6037 guest_command.args(&["--net", &net_params]); 6038 } 6039 6040 let mut child = guest_command.capture_output().spawn().unwrap(); 6041 6042 if hotplug { 6043 // Give some time to the VMM process to listen to the API 6044 // socket. This is the only requirement to avoid the following 6045 // call to ch-remote from failing. 6046 thread::sleep(std::time::Duration::new(10, 0)); 6047 // Hotplug the virtio-net device 6048 let (cmd_success, cmd_output) = 6049 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6050 assert!(cmd_success); 6051 #[cfg(target_arch = "x86_64")] 6052 assert!(String::from_utf8_lossy(&cmd_output) 6053 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6054 #[cfg(target_arch = "aarch64")] 6055 assert!(String::from_utf8_lossy(&cmd_output) 6056 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6057 } 6058 6059 // The functional connectivity provided by the virtio-net device 6060 // gets tested through wait_vm_boot() as it expects to receive a 6061 // HTTP request, and through the SSH command as well. 6062 let r = std::panic::catch_unwind(|| { 6063 guest.wait_vm_boot(None).unwrap(); 6064 6065 assert_eq!( 6066 guest 6067 .ssh_command("ip -o link | wc -l") 6068 .unwrap() 6069 .trim() 6070 .parse::<u32>() 6071 .unwrap_or_default(), 6072 2 6073 ); 6074 }); 6075 6076 let _ = child.kill(); 6077 6078 exec_host_command_status(&format!("sudo ip link del {}", guest_macvtap_name)); 6079 exec_host_command_status(&format!("sudo ip link del {}", host_macvtap_name)); 6080 6081 let output = child.wait_with_output().unwrap(); 6082 6083 handle_child_output(r, &output); 6084 } 6085 6086 #[test] 6087 fn test_macvtap() { 6088 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6089 } 6090 6091 #[test] 6092 fn test_macvtap_hotplug() { 6093 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6094 } 6095 6096 #[test] 6097 #[cfg(not(feature = "mshv"))] 6098 fn test_ovs_dpdk() { 6099 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6100 let guest1 = Guest::new(Box::new(focal1)); 6101 6102 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6103 let guest2 = Guest::new(Box::new(focal2)); 6104 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6105 6106 let (mut child1, mut child2) = 6107 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6108 6109 // Create the snapshot directory 6110 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6111 6112 let r = std::panic::catch_unwind(|| { 6113 // Remove one of the two ports from the OVS bridge 6114 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6115 6116 // Spawn a new netcat listener in the first VM 6117 let guest_ip = guest1.network.guest_ip.clone(); 6118 thread::spawn(move || { 6119 ssh_command_ip( 6120 "nc -l 12345", 6121 &guest_ip, 6122 DEFAULT_SSH_RETRIES, 6123 DEFAULT_SSH_TIMEOUT, 6124 ) 6125 .unwrap(); 6126 }); 6127 6128 // Wait for the server to be listening 6129 thread::sleep(std::time::Duration::new(5, 0)); 6130 6131 // Check the connection fails this time 6132 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6133 6134 // Add the OVS port back 6135 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6136 6137 // And finally check the connection is functional again 6138 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6139 6140 // Pause the VM 6141 assert!(remote_command(&api_socket_source, "pause", None)); 6142 6143 // Take a snapshot from the VM 6144 assert!(remote_command( 6145 &api_socket_source, 6146 "snapshot", 6147 Some(format!("file://{}", snapshot_dir).as_str()), 6148 )); 6149 6150 // Wait to make sure the snapshot is completed 6151 thread::sleep(std::time::Duration::new(10, 0)); 6152 }); 6153 6154 // Shutdown the source VM 6155 let _ = child2.kill(); 6156 let output = child2.wait_with_output().unwrap(); 6157 handle_child_output(r, &output); 6158 6159 // Remove the vhost-user socket file. 6160 Command::new("rm") 6161 .arg("-f") 6162 .arg("/tmp/dpdkvhostclient2") 6163 .output() 6164 .unwrap(); 6165 6166 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6167 // Restore the VM from the snapshot 6168 let mut child2 = GuestCommand::new(&guest2) 6169 .args(&["--api-socket", &api_socket_restored]) 6170 .args(&[ 6171 "--restore", 6172 format!("source_url=file://{}", snapshot_dir).as_str(), 6173 ]) 6174 .capture_output() 6175 .spawn() 6176 .unwrap(); 6177 6178 // Wait for the VM to be restored 6179 thread::sleep(std::time::Duration::new(10, 0)); 6180 6181 let r = std::panic::catch_unwind(|| { 6182 // Resume the VM 6183 assert!(remote_command(&api_socket_restored, "resume", None)); 6184 6185 // Spawn a new netcat listener in the first VM 6186 let guest_ip = guest1.network.guest_ip.clone(); 6187 thread::spawn(move || { 6188 ssh_command_ip( 6189 "nc -l 12345", 6190 &guest_ip, 6191 DEFAULT_SSH_RETRIES, 6192 DEFAULT_SSH_TIMEOUT, 6193 ) 6194 .unwrap(); 6195 }); 6196 6197 // Wait for the server to be listening 6198 thread::sleep(std::time::Duration::new(5, 0)); 6199 6200 // And check the connection is still functional after restore 6201 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6202 }); 6203 6204 cleanup_ovs_dpdk(); 6205 6206 let _ = child1.kill(); 6207 let _ = child2.kill(); 6208 6209 let output = child1.wait_with_output().unwrap(); 6210 child2.wait().unwrap(); 6211 6212 handle_child_output(r, &output); 6213 } 6214 6215 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6216 cleanup_spdk_nvme(); 6217 6218 assert!(exec_host_command_status(&format!( 6219 "mkdir -p {}", 6220 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6221 )) 6222 .success()); 6223 assert!(exec_host_command_status(&format!( 6224 "truncate {} -s 128M", 6225 nvme_dir.join("test-disk.raw").to_str().unwrap() 6226 )) 6227 .success()); 6228 assert!(exec_host_command_status(&format!( 6229 "mkfs.ext4 {}", 6230 nvme_dir.join("test-disk.raw").to_str().unwrap() 6231 )) 6232 .success()); 6233 6234 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6235 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6236 .args(&["-i", "0", "-m", "0x1"]) 6237 .spawn() 6238 .unwrap(); 6239 thread::sleep(std::time::Duration::new(2, 0)); 6240 6241 assert!(exec_host_command_status( 6242 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER" 6243 ) 6244 .success()); 6245 assert!(exec_host_command_status(&format!( 6246 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6247 nvme_dir.join("test-disk.raw").to_str().unwrap() 6248 )) 6249 .success()); 6250 assert!(exec_host_command_status( 6251 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6252 ) 6253 .success()); 6254 assert!(exec_host_command_status( 6255 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6256 ) 6257 .success()); 6258 assert!(exec_host_command_status(&format!( 6259 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6260 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6261 )) 6262 .success()); 6263 } 6264 6265 fn cleanup_spdk_nvme() { 6266 exec_host_command_status("pkill -f nvmf_tgt"); 6267 } 6268 6269 #[test] 6270 fn test_vfio_user() { 6271 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6272 let jammy = UbuntuDiskConfig::new(jammy_image); 6273 let guest = Guest::new(Box::new(jammy)); 6274 6275 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6276 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6277 6278 let api_socket = temp_api_path(&guest.tmp_dir); 6279 let mut child = GuestCommand::new(&guest) 6280 .args(&["--api-socket", &api_socket]) 6281 .args(&["--cpus", "boot=1"]) 6282 .args(&["--memory", "size=512M,shared=on,hugepages=on"]) 6283 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6284 .args(&["--serial", "tty", "--console", "off"]) 6285 .default_disks() 6286 .default_net() 6287 .capture_output() 6288 .spawn() 6289 .unwrap(); 6290 6291 let r = std::panic::catch_unwind(|| { 6292 guest.wait_vm_boot(None).unwrap(); 6293 6294 // Hotplug the SPDK-NVMe device to the VM 6295 let (cmd_success, cmd_output) = remote_command_w_output( 6296 &api_socket, 6297 "add-user-device", 6298 Some(&format!( 6299 "socket={},id=vfio_user0", 6300 spdk_nvme_dir 6301 .as_path() 6302 .join("nvme-vfio-user/cntrl") 6303 .to_str() 6304 .unwrap(), 6305 )), 6306 ); 6307 assert!(cmd_success); 6308 assert!(String::from_utf8_lossy(&cmd_output) 6309 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6310 6311 thread::sleep(std::time::Duration::new(10, 0)); 6312 6313 // Check both if /dev/nvme exists and if the block size is 128M. 6314 assert_eq!( 6315 guest 6316 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6317 .unwrap() 6318 .trim() 6319 .parse::<u32>() 6320 .unwrap_or_default(), 6321 1 6322 ); 6323 6324 // Check changes persist after reboot 6325 assert_eq!( 6326 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6327 "" 6328 ); 6329 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6330 guest 6331 .ssh_command("echo test123 | sudo tee /mnt/test") 6332 .unwrap(); 6333 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6334 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6335 6336 guest.reboot_linux(0, None); 6337 assert_eq!( 6338 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6339 "" 6340 ); 6341 assert_eq!( 6342 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6343 "test123" 6344 ); 6345 }); 6346 6347 cleanup_spdk_nvme(); 6348 6349 let _ = child.kill(); 6350 let output = child.wait_with_output().unwrap(); 6351 6352 handle_child_output(r, &output); 6353 } 6354 6355 #[test] 6356 #[cfg(target_arch = "x86_64")] 6357 fn test_vdpa_block() { 6358 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6359 if !exec_host_command_status("lsmod | grep vdpa_sim_blk").success() { 6360 return; 6361 } 6362 6363 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6364 let guest = Guest::new(Box::new(focal)); 6365 let api_socket = temp_api_path(&guest.tmp_dir); 6366 6367 let kernel_path = direct_kernel_boot_path(); 6368 6369 let mut child = GuestCommand::new(&guest) 6370 .args(&["--cpus", "boot=2"]) 6371 .args(&["--memory", "size=512M,hugepages=on"]) 6372 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6373 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6374 .default_disks() 6375 .default_net() 6376 .args(&["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6377 .args(&["--platform", "num_pci_segments=2,iommu_segments=1"]) 6378 .args(&["--api-socket", &api_socket]) 6379 .capture_output() 6380 .spawn() 6381 .unwrap(); 6382 6383 let r = std::panic::catch_unwind(|| { 6384 guest.wait_vm_boot(None).unwrap(); 6385 6386 // Check both if /dev/vdc exists and if the block size is 128M. 6387 assert_eq!( 6388 guest 6389 .ssh_command("lsblk | grep vdc | grep -c 128M") 6390 .unwrap() 6391 .trim() 6392 .parse::<u32>() 6393 .unwrap_or_default(), 6394 1 6395 ); 6396 6397 // Check the content of the block device after we wrote to it. 6398 // The vpda-sim-blk should let us read what we previously wrote. 6399 guest 6400 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6401 .unwrap(); 6402 assert_eq!( 6403 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6404 "foobar" 6405 ); 6406 6407 // Hotplug an extra vDPA block device behind the vIOMMU 6408 // Add a new vDPA device to the VM 6409 let (cmd_success, cmd_output) = remote_command_w_output( 6410 &api_socket, 6411 "add-vdpa", 6412 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6413 ); 6414 assert!(cmd_success); 6415 assert!(String::from_utf8_lossy(&cmd_output) 6416 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6417 6418 thread::sleep(std::time::Duration::new(10, 0)); 6419 6420 // Check IOMMU setup 6421 assert!(guest 6422 .does_device_vendor_pair_match("0x1057", "0x1af4") 6423 .unwrap_or_default()); 6424 assert_eq!( 6425 guest 6426 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6427 .unwrap() 6428 .trim(), 6429 "0001:00:01.0" 6430 ); 6431 6432 // Check both if /dev/vdd exists and if the block size is 128M. 6433 assert_eq!( 6434 guest 6435 .ssh_command("lsblk | grep vdd | grep -c 128M") 6436 .unwrap() 6437 .trim() 6438 .parse::<u32>() 6439 .unwrap_or_default(), 6440 1 6441 ); 6442 6443 // Write some content to the block device we've just plugged. 6444 guest 6445 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6446 .unwrap(); 6447 6448 // Check we can read the content back. 6449 assert_eq!( 6450 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6451 "foobar" 6452 ); 6453 6454 // Unplug the device 6455 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6456 assert!(cmd_success); 6457 thread::sleep(std::time::Duration::new(10, 0)); 6458 6459 // Check /dev/vdd doesn't exist anymore 6460 assert_eq!( 6461 guest 6462 .ssh_command("lsblk | grep -c vdd || true") 6463 .unwrap() 6464 .trim() 6465 .parse::<u32>() 6466 .unwrap_or(1), 6467 0 6468 ); 6469 }); 6470 6471 let _ = child.kill(); 6472 let output = child.wait_with_output().unwrap(); 6473 6474 handle_child_output(r, &output); 6475 } 6476 6477 #[test] 6478 #[cfg(target_arch = "x86_64")] 6479 fn test_vdpa_net() { 6480 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6481 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6482 return; 6483 } 6484 6485 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6486 let guest = Guest::new(Box::new(focal)); 6487 6488 let kernel_path = direct_kernel_boot_path(); 6489 6490 let mut child = GuestCommand::new(&guest) 6491 .args(&["--cpus", "boot=2"]) 6492 .args(&["--memory", "size=512M,hugepages=on"]) 6493 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6494 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6495 .default_disks() 6496 .default_net() 6497 .args(&["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 6498 .capture_output() 6499 .spawn() 6500 .unwrap(); 6501 6502 let r = std::panic::catch_unwind(|| { 6503 guest.wait_vm_boot(None).unwrap(); 6504 6505 // Check we can find network interface related to vDPA device 6506 assert_eq!( 6507 guest 6508 .ssh_command("ip -o link | grep -c ens6") 6509 .unwrap() 6510 .trim() 6511 .parse::<u32>() 6512 .unwrap_or(0), 6513 1 6514 ); 6515 6516 guest 6517 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 6518 .unwrap(); 6519 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 6520 6521 // Check there is no packet yet on both TX/RX of the network interface 6522 assert_eq!( 6523 guest 6524 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 6525 .unwrap() 6526 .trim() 6527 .parse::<u32>() 6528 .unwrap_or(0), 6529 2 6530 ); 6531 6532 // Send 6 packets with ping command 6533 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 6534 6535 // Check we can find 6 packets on both TX/RX of the network interface 6536 assert_eq!( 6537 guest 6538 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 6539 .unwrap() 6540 .trim() 6541 .parse::<u32>() 6542 .unwrap_or(0), 6543 2 6544 ); 6545 6546 // No need to check for hotplug as we already tested it through 6547 // test_vdpa_block() 6548 }); 6549 6550 let _ = child.kill(); 6551 let output = child.wait_with_output().unwrap(); 6552 6553 handle_child_output(r, &output); 6554 } 6555 } 6556 6557 mod sequential { 6558 use crate::*; 6559 6560 #[test] 6561 fn test_memory_mergeable_on() { 6562 test_memory_mergeable(true) 6563 } 6564 } 6565 6566 #[cfg(target_arch = "x86_64")] 6567 mod windows { 6568 use crate::*; 6569 use once_cell::sync::Lazy; 6570 6571 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 6572 6573 struct WindowsGuest { 6574 guest: Guest, 6575 auth: PasswordAuth, 6576 } 6577 6578 trait FsType { 6579 const FS_FAT: u8; 6580 const FS_NTFS: u8; 6581 } 6582 impl FsType for WindowsGuest { 6583 const FS_FAT: u8 = 0; 6584 const FS_NTFS: u8 = 1; 6585 } 6586 6587 impl WindowsGuest { 6588 fn new() -> Self { 6589 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 6590 let guest = Guest::new(Box::new(disk)); 6591 let auth = PasswordAuth { 6592 username: String::from("administrator"), 6593 password: String::from("Admin123"), 6594 }; 6595 6596 WindowsGuest { guest, auth } 6597 } 6598 6599 fn guest(&self) -> &Guest { 6600 &self.guest 6601 } 6602 6603 fn ssh_cmd(&self, cmd: &str) -> String { 6604 ssh_command_ip_with_auth( 6605 cmd, 6606 &self.auth, 6607 &self.guest.network.guest_ip, 6608 DEFAULT_SSH_RETRIES, 6609 DEFAULT_SSH_TIMEOUT, 6610 ) 6611 .unwrap() 6612 } 6613 6614 fn cpu_count(&self) -> u8 { 6615 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 6616 .trim() 6617 .parse::<u8>() 6618 .unwrap_or(0) 6619 } 6620 6621 fn ram_size(&self) -> usize { 6622 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 6623 .trim() 6624 .parse::<usize>() 6625 .unwrap_or(0) 6626 } 6627 6628 fn netdev_count(&self) -> u8 { 6629 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6630 .trim() 6631 .parse::<u8>() 6632 .unwrap_or(0) 6633 } 6634 6635 fn disk_count(&self) -> u8 { 6636 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6637 .trim() 6638 .parse::<u8>() 6639 .unwrap_or(0) 6640 } 6641 6642 fn reboot(&self) { 6643 let _ = self.ssh_cmd("shutdown /r /t 0"); 6644 } 6645 6646 fn shutdown(&self) { 6647 let _ = self.ssh_cmd("shutdown /s /t 0"); 6648 } 6649 6650 fn run_dnsmasq(&self) -> std::process::Child { 6651 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 6652 let dhcp_host = format!( 6653 "--dhcp-host={},{}", 6654 self.guest.network.guest_mac, self.guest.network.guest_ip 6655 ); 6656 let dhcp_range = format!( 6657 "--dhcp-range=eth,{},{}", 6658 self.guest.network.guest_ip, self.guest.network.guest_ip 6659 ); 6660 6661 Command::new("dnsmasq") 6662 .arg("--no-daemon") 6663 .arg("--log-queries") 6664 .arg(listen_address.as_str()) 6665 .arg("--except-interface=lo") 6666 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 6667 .arg("--conf-file=/dev/null") 6668 .arg(dhcp_host.as_str()) 6669 .arg(dhcp_range.as_str()) 6670 .spawn() 6671 .unwrap() 6672 } 6673 6674 // TODO Cleanup image file explicitly after test, if there's some space issues. 6675 fn disk_new(&self, fs: u8, sz: usize) -> String { 6676 let mut guard = NEXT_DISK_ID.lock().unwrap(); 6677 let id = *guard; 6678 *guard = id + 1; 6679 6680 let img = PathBuf::from(format!("/tmp/test-hotplug-{}.raw", id)); 6681 let _ = fs::remove_file(&img); 6682 6683 // Create an image file 6684 let out = Command::new("qemu-img") 6685 .args(&[ 6686 "create", 6687 "-f", 6688 "raw", 6689 img.to_str().unwrap(), 6690 format!("{}m", sz).as_str(), 6691 ]) 6692 .output() 6693 .expect("qemu-img command failed") 6694 .stdout; 6695 println!("{:?}", out); 6696 6697 // Associate image to a loop device 6698 let out = Command::new("losetup") 6699 .args(&["--show", "-f", img.to_str().unwrap()]) 6700 .output() 6701 .expect("failed to create loop device") 6702 .stdout; 6703 let _tmp = String::from_utf8_lossy(&out); 6704 let loop_dev = _tmp.trim(); 6705 println!("{:?}", out); 6706 6707 // Create a partition table 6708 // echo 'type=7' | sudo sfdisk "${LOOP}" 6709 let mut child = Command::new("sfdisk") 6710 .args(&[loop_dev]) 6711 .stdin(Stdio::piped()) 6712 .spawn() 6713 .unwrap(); 6714 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 6715 stdin 6716 .write_all("type=7".as_bytes()) 6717 .expect("failed to write stdin"); 6718 let out = child.wait_with_output().expect("sfdisk failed").stdout; 6719 println!("{:?}", out); 6720 6721 // Disengage the loop device 6722 let out = Command::new("losetup") 6723 .args(&["-d", loop_dev]) 6724 .output() 6725 .expect("loop device not found") 6726 .stdout; 6727 println!("{:?}", out); 6728 6729 // Re-associate loop device pointing to the partition only 6730 let out = Command::new("losetup") 6731 .args(&[ 6732 "--show", 6733 "--offset", 6734 (512 * 2048).to_string().as_str(), 6735 "-f", 6736 img.to_str().unwrap(), 6737 ]) 6738 .output() 6739 .expect("failed to create loop device") 6740 .stdout; 6741 let _tmp = String::from_utf8_lossy(&out); 6742 let loop_dev = _tmp.trim(); 6743 println!("{:?}", out); 6744 6745 // Create filesystem. 6746 let fs_cmd = match fs { 6747 WindowsGuest::FS_FAT => "mkfs.msdos", 6748 WindowsGuest::FS_NTFS => "mkfs.ntfs", 6749 _ => panic!("Unknown filesystem type '{}'", fs), 6750 }; 6751 let out = Command::new(fs_cmd) 6752 .args(&[&loop_dev]) 6753 .output() 6754 .unwrap_or_else(|_| panic!("{} failed", fs_cmd)) 6755 .stdout; 6756 println!("{:?}", out); 6757 6758 // Disengage the loop device 6759 let out = Command::new("losetup") 6760 .args(&["-d", loop_dev]) 6761 .output() 6762 .unwrap_or_else(|_| panic!("loop device '{}' not found", loop_dev)) 6763 .stdout; 6764 println!("{:?}", out); 6765 6766 img.to_str().unwrap().to_string() 6767 } 6768 6769 fn disks_set_rw(&self) { 6770 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 6771 } 6772 6773 fn disks_online(&self) { 6774 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 6775 } 6776 6777 fn disk_file_put(&self, fname: &str, data: &str) { 6778 let _ = self.ssh_cmd(&format!( 6779 "powershell -Command \"'{}' | Set-Content -Path {}\"", 6780 data, fname 6781 )); 6782 } 6783 6784 fn disk_file_read(&self, fname: &str) -> String { 6785 self.ssh_cmd(&format!( 6786 "powershell -Command \"Get-Content -Path {}\"", 6787 fname 6788 )) 6789 } 6790 6791 fn wait_for_boot(&self) -> bool { 6792 let cmd = "dir /b c:\\ | find \"Windows\""; 6793 let tmo_max = 180; 6794 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 6795 // interval must be small. 6796 let tmo_int = 2; 6797 let out = ssh_command_ip_with_auth( 6798 cmd, 6799 &self.auth, 6800 &self.guest.network.guest_ip, 6801 { 6802 let mut ret = 1; 6803 let mut tmo_acc = 0; 6804 loop { 6805 tmo_acc += tmo_int * ret; 6806 if tmo_acc >= tmo_max { 6807 break; 6808 } 6809 ret += 1; 6810 } 6811 ret 6812 }, 6813 tmo_int, 6814 ) 6815 .unwrap(); 6816 6817 if "Windows" == out.trim() { 6818 return true; 6819 } 6820 6821 false 6822 } 6823 } 6824 6825 fn vcpu_threads_count(pid: u32) -> u8 { 6826 // ps -T -p 12345 | grep vcpu | wc -l 6827 let out = Command::new("ps") 6828 .args(&["-T", "-p", format!("{}", pid).as_str()]) 6829 .output() 6830 .expect("ps command failed") 6831 .stdout; 6832 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 6833 } 6834 6835 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 6836 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 6837 let out = Command::new("ps") 6838 .args(&["-T", "-p", format!("{}", pid).as_str()]) 6839 .output() 6840 .expect("ps command failed") 6841 .stdout; 6842 let mut n = 0; 6843 String::from_utf8_lossy(&out) 6844 .split_whitespace() 6845 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 6846 n 6847 } 6848 6849 fn disk_ctrl_threads_count(pid: u32) -> u8 { 6850 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 6851 let out = Command::new("ps") 6852 .args(&["-T", "-p", format!("{}", pid).as_str()]) 6853 .output() 6854 .expect("ps command failed") 6855 .stdout; 6856 let mut n = 0; 6857 String::from_utf8_lossy(&out) 6858 .split_whitespace() 6859 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 6860 n 6861 } 6862 6863 #[test] 6864 fn test_windows_guest() { 6865 let windows_guest = WindowsGuest::new(); 6866 6867 let mut ovmf_path = dirs::home_dir().unwrap(); 6868 ovmf_path.push("workloads"); 6869 ovmf_path.push(OVMF_NAME); 6870 6871 let mut child = GuestCommand::new(windows_guest.guest()) 6872 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6873 .args(&["--memory", "size=4G"]) 6874 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6875 .args(&["--serial", "tty"]) 6876 .args(&["--console", "off"]) 6877 .default_disks() 6878 .default_net() 6879 .capture_output() 6880 .spawn() 6881 .unwrap(); 6882 6883 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 6884 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6885 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 6886 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6887 6888 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 6889 6890 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6891 6892 let r = std::panic::catch_unwind(|| { 6893 // Wait to make sure Windows boots up 6894 assert!(windows_guest.wait_for_boot()); 6895 6896 windows_guest.shutdown(); 6897 }); 6898 6899 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6900 let _ = child.kill(); 6901 let output = child.wait_with_output().unwrap(); 6902 6903 let _ = child_dnsmasq.kill(); 6904 let _ = child_dnsmasq.wait(); 6905 6906 handle_child_output(r, &output); 6907 } 6908 6909 #[test] 6910 fn test_windows_guest_multiple_queues() { 6911 let windows_guest = WindowsGuest::new(); 6912 6913 let mut ovmf_path = dirs::home_dir().unwrap(); 6914 ovmf_path.push("workloads"); 6915 ovmf_path.push(OVMF_NAME); 6916 6917 let mut child = GuestCommand::new(windows_guest.guest()) 6918 .args(&["--cpus", "boot=4,kvm_hyperv=on"]) 6919 .args(&["--memory", "size=4G"]) 6920 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6921 .args(&["--serial", "tty"]) 6922 .args(&["--console", "off"]) 6923 .args(&[ 6924 "--disk", 6925 format!( 6926 "path={},num_queues=4", 6927 windows_guest 6928 .guest() 6929 .disk_config 6930 .disk(DiskType::OperatingSystem) 6931 .unwrap() 6932 ) 6933 .as_str(), 6934 ]) 6935 .args(&[ 6936 "--net", 6937 format!( 6938 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 6939 windows_guest.guest().network.guest_mac, 6940 windows_guest.guest().network.host_ip 6941 ) 6942 .as_str(), 6943 ]) 6944 .capture_output() 6945 .spawn() 6946 .unwrap(); 6947 6948 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 6949 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6950 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 6951 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6952 6953 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 6954 6955 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6956 6957 let r = std::panic::catch_unwind(|| { 6958 // Wait to make sure Windows boots up 6959 assert!(windows_guest.wait_for_boot()); 6960 6961 windows_guest.shutdown(); 6962 }); 6963 6964 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6965 let _ = child.kill(); 6966 let output = child.wait_with_output().unwrap(); 6967 6968 let _ = child_dnsmasq.kill(); 6969 let _ = child_dnsmasq.wait(); 6970 6971 handle_child_output(r, &output); 6972 } 6973 6974 #[test] 6975 #[cfg(not(feature = "mshv"))] 6976 #[ignore = "See #4327"] 6977 fn test_windows_guest_snapshot_restore() { 6978 let windows_guest = WindowsGuest::new(); 6979 6980 let mut ovmf_path = dirs::home_dir().unwrap(); 6981 ovmf_path.push("workloads"); 6982 ovmf_path.push(OVMF_NAME); 6983 6984 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6985 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 6986 6987 let mut child = GuestCommand::new(windows_guest.guest()) 6988 .args(&["--api-socket", &api_socket_source]) 6989 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6990 .args(&["--memory", "size=4G"]) 6991 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6992 .args(&["--serial", "tty"]) 6993 .args(&["--console", "off"]) 6994 .default_disks() 6995 .default_net() 6996 .capture_output() 6997 .spawn() 6998 .unwrap(); 6999 7000 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7001 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7002 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7003 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7004 7005 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7006 7007 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7008 7009 // Wait to make sure Windows boots up 7010 assert!(windows_guest.wait_for_boot()); 7011 7012 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 7013 7014 // Pause the VM 7015 assert!(remote_command(&api_socket_source, "pause", None)); 7016 7017 // Take a snapshot from the VM 7018 assert!(remote_command( 7019 &api_socket_source, 7020 "snapshot", 7021 Some(format!("file://{}", snapshot_dir).as_str()), 7022 )); 7023 7024 // Wait to make sure the snapshot is completed 7025 thread::sleep(std::time::Duration::new(30, 0)); 7026 7027 let _ = child.kill(); 7028 child.wait().unwrap(); 7029 7030 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 7031 7032 // Restore the VM from the snapshot 7033 let mut child = GuestCommand::new(windows_guest.guest()) 7034 .args(&["--api-socket", &api_socket_restored]) 7035 .args(&[ 7036 "--restore", 7037 format!("source_url=file://{}", snapshot_dir).as_str(), 7038 ]) 7039 .capture_output() 7040 .spawn() 7041 .unwrap(); 7042 7043 // Wait for the VM to be restored 7044 thread::sleep(std::time::Duration::new(20, 0)); 7045 7046 let r = std::panic::catch_unwind(|| { 7047 // Resume the VM 7048 assert!(remote_command(&api_socket_restored, "resume", None)); 7049 7050 windows_guest.shutdown(); 7051 }); 7052 7053 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7054 let _ = child.kill(); 7055 let output = child.wait_with_output().unwrap(); 7056 7057 let _ = child_dnsmasq.kill(); 7058 let _ = child_dnsmasq.wait(); 7059 7060 handle_child_output(r, &output); 7061 } 7062 7063 #[test] 7064 #[cfg(not(feature = "mshv"))] 7065 fn test_windows_guest_cpu_hotplug() { 7066 let windows_guest = WindowsGuest::new(); 7067 7068 let mut ovmf_path = dirs::home_dir().unwrap(); 7069 ovmf_path.push("workloads"); 7070 ovmf_path.push(OVMF_NAME); 7071 7072 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7073 let api_socket = temp_api_path(&tmp_dir); 7074 7075 let mut child = GuestCommand::new(windows_guest.guest()) 7076 .args(&["--api-socket", &api_socket]) 7077 .args(&["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 7078 .args(&["--memory", "size=4G"]) 7079 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7080 .args(&["--serial", "tty"]) 7081 .args(&["--console", "off"]) 7082 .default_disks() 7083 .default_net() 7084 .capture_output() 7085 .spawn() 7086 .unwrap(); 7087 7088 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7089 7090 let r = std::panic::catch_unwind(|| { 7091 // Wait to make sure Windows boots up 7092 assert!(windows_guest.wait_for_boot()); 7093 7094 let vcpu_num = 2; 7095 // Check the initial number of CPUs the guest sees 7096 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7097 // Check the initial number of vcpu threads in the CH process 7098 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7099 7100 let vcpu_num = 6; 7101 // Hotplug some CPUs 7102 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7103 // Wait to make sure CPUs are added 7104 thread::sleep(std::time::Duration::new(10, 0)); 7105 // Check the guest sees the correct number 7106 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7107 // Check the CH process has the correct number of vcpu threads 7108 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7109 7110 let vcpu_num = 4; 7111 // Remove some CPUs. Note that Windows doesn't support hot-remove. 7112 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7113 // Wait to make sure CPUs are removed 7114 thread::sleep(std::time::Duration::new(10, 0)); 7115 // Reboot to let Windows catch up 7116 windows_guest.reboot(); 7117 // Wait to make sure Windows completely rebooted 7118 thread::sleep(std::time::Duration::new(60, 0)); 7119 // Check the guest sees the correct number 7120 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7121 // Check the CH process has the correct number of vcpu threads 7122 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7123 7124 windows_guest.shutdown(); 7125 }); 7126 7127 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7128 let _ = child.kill(); 7129 let output = child.wait_with_output().unwrap(); 7130 7131 let _ = child_dnsmasq.kill(); 7132 let _ = child_dnsmasq.wait(); 7133 7134 handle_child_output(r, &output); 7135 } 7136 7137 #[test] 7138 #[cfg(not(feature = "mshv"))] 7139 fn test_windows_guest_ram_hotplug() { 7140 let windows_guest = WindowsGuest::new(); 7141 7142 let mut ovmf_path = dirs::home_dir().unwrap(); 7143 ovmf_path.push("workloads"); 7144 ovmf_path.push(OVMF_NAME); 7145 7146 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7147 let api_socket = temp_api_path(&tmp_dir); 7148 7149 let mut child = GuestCommand::new(windows_guest.guest()) 7150 .args(&["--api-socket", &api_socket]) 7151 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7152 .args(&["--memory", "size=2G,hotplug_size=5G"]) 7153 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7154 .args(&["--serial", "tty"]) 7155 .args(&["--console", "off"]) 7156 .default_disks() 7157 .default_net() 7158 .capture_output() 7159 .spawn() 7160 .unwrap(); 7161 7162 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7163 7164 let r = std::panic::catch_unwind(|| { 7165 // Wait to make sure Windows boots up 7166 assert!(windows_guest.wait_for_boot()); 7167 7168 let ram_size = 2 * 1024 * 1024 * 1024; 7169 // Check the initial number of RAM the guest sees 7170 let current_ram_size = windows_guest.ram_size(); 7171 // This size seems to be reserved by the system and thus the 7172 // reported amount differs by this constant value. 7173 let reserved_ram_size = ram_size - current_ram_size; 7174 // Verify that there's not more than 4mb constant diff wasted 7175 // by the reserved ram. 7176 assert!(reserved_ram_size < 4 * 1024 * 1024); 7177 7178 let ram_size = 4 * 1024 * 1024 * 1024; 7179 // Hotplug some RAM 7180 resize_command(&api_socket, None, Some(ram_size), None, None); 7181 // Wait to make sure RAM has been added 7182 thread::sleep(std::time::Duration::new(10, 0)); 7183 // Check the guest sees the correct number 7184 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7185 7186 let ram_size = 3 * 1024 * 1024 * 1024; 7187 // Unplug some RAM. Note that hot-remove most likely won't work. 7188 resize_command(&api_socket, None, Some(ram_size), None, None); 7189 // Wait to make sure RAM has been added 7190 thread::sleep(std::time::Duration::new(10, 0)); 7191 // Reboot to let Windows catch up 7192 windows_guest.reboot(); 7193 // Wait to make sure guest completely rebooted 7194 thread::sleep(std::time::Duration::new(60, 0)); 7195 // Check the guest sees the correct number 7196 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7197 7198 windows_guest.shutdown(); 7199 }); 7200 7201 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7202 let _ = child.kill(); 7203 let output = child.wait_with_output().unwrap(); 7204 7205 let _ = child_dnsmasq.kill(); 7206 let _ = child_dnsmasq.wait(); 7207 7208 handle_child_output(r, &output); 7209 } 7210 7211 #[test] 7212 #[cfg(not(feature = "mshv"))] 7213 fn test_windows_guest_netdev_hotplug() { 7214 let windows_guest = WindowsGuest::new(); 7215 7216 let mut ovmf_path = dirs::home_dir().unwrap(); 7217 ovmf_path.push("workloads"); 7218 ovmf_path.push(OVMF_NAME); 7219 7220 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7221 let api_socket = temp_api_path(&tmp_dir); 7222 7223 let mut child = GuestCommand::new(windows_guest.guest()) 7224 .args(&["--api-socket", &api_socket]) 7225 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7226 .args(&["--memory", "size=4G"]) 7227 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7228 .args(&["--serial", "tty"]) 7229 .args(&["--console", "off"]) 7230 .default_disks() 7231 .default_net() 7232 .capture_output() 7233 .spawn() 7234 .unwrap(); 7235 7236 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7237 7238 let r = std::panic::catch_unwind(|| { 7239 // Wait to make sure Windows boots up 7240 assert!(windows_guest.wait_for_boot()); 7241 7242 // Initially present network device 7243 let netdev_num = 1; 7244 assert_eq!(windows_guest.netdev_count(), netdev_num); 7245 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7246 7247 // Hotplug network device 7248 let (cmd_success, cmd_output) = remote_command_w_output( 7249 &api_socket, 7250 "add-net", 7251 Some(windows_guest.guest().default_net_string().as_str()), 7252 ); 7253 assert!(cmd_success); 7254 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 7255 thread::sleep(std::time::Duration::new(5, 0)); 7256 // Verify the device is on the system 7257 let netdev_num = 2; 7258 assert_eq!(windows_guest.netdev_count(), netdev_num); 7259 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7260 7261 // Remove network device 7262 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 7263 assert!(cmd_success); 7264 thread::sleep(std::time::Duration::new(5, 0)); 7265 // Verify the device has been removed 7266 let netdev_num = 1; 7267 assert_eq!(windows_guest.netdev_count(), netdev_num); 7268 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7269 7270 windows_guest.shutdown(); 7271 }); 7272 7273 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7274 let _ = child.kill(); 7275 let output = child.wait_with_output().unwrap(); 7276 7277 let _ = child_dnsmasq.kill(); 7278 let _ = child_dnsmasq.wait(); 7279 7280 handle_child_output(r, &output); 7281 } 7282 7283 #[test] 7284 #[cfg(not(feature = "mshv"))] 7285 fn test_windows_guest_disk_hotplug() { 7286 let windows_guest = WindowsGuest::new(); 7287 7288 let mut ovmf_path = dirs::home_dir().unwrap(); 7289 ovmf_path.push("workloads"); 7290 ovmf_path.push(OVMF_NAME); 7291 7292 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7293 let api_socket = temp_api_path(&tmp_dir); 7294 7295 let mut child = GuestCommand::new(windows_guest.guest()) 7296 .args(&["--api-socket", &api_socket]) 7297 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7298 .args(&["--memory", "size=4G"]) 7299 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7300 .args(&["--serial", "tty"]) 7301 .args(&["--console", "off"]) 7302 .default_disks() 7303 .default_net() 7304 .capture_output() 7305 .spawn() 7306 .unwrap(); 7307 7308 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7309 7310 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 7311 7312 let r = std::panic::catch_unwind(|| { 7313 // Wait to make sure Windows boots up 7314 assert!(windows_guest.wait_for_boot()); 7315 7316 // Initially present disk device 7317 let disk_num = 1; 7318 assert_eq!(windows_guest.disk_count(), disk_num); 7319 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7320 7321 // Hotplug disk device 7322 let (cmd_success, cmd_output) = remote_command_w_output( 7323 &api_socket, 7324 "add-disk", 7325 Some(format!("path={},readonly=off", disk).as_str()), 7326 ); 7327 assert!(cmd_success); 7328 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 7329 thread::sleep(std::time::Duration::new(5, 0)); 7330 // Online disk device 7331 windows_guest.disks_set_rw(); 7332 windows_guest.disks_online(); 7333 // Verify the device is on the system 7334 let disk_num = 2; 7335 assert_eq!(windows_guest.disk_count(), disk_num); 7336 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7337 7338 let data = "hello"; 7339 let fname = "d:\\world"; 7340 windows_guest.disk_file_put(fname, data); 7341 7342 // Unmount disk device 7343 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 7344 assert!(cmd_success); 7345 thread::sleep(std::time::Duration::new(5, 0)); 7346 // Verify the device has been removed 7347 let disk_num = 1; 7348 assert_eq!(windows_guest.disk_count(), disk_num); 7349 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7350 7351 // Remount and check the file exists with the expected contents 7352 let (cmd_success, _cmd_output) = remote_command_w_output( 7353 &api_socket, 7354 "add-disk", 7355 Some(format!("path={},readonly=off", disk).as_str()), 7356 ); 7357 assert!(cmd_success); 7358 thread::sleep(std::time::Duration::new(5, 0)); 7359 let out = windows_guest.disk_file_read(fname); 7360 assert_eq!(data, out.trim()); 7361 7362 // Intentionally no unmount, it'll happen at shutdown. 7363 7364 windows_guest.shutdown(); 7365 }); 7366 7367 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7368 let _ = child.kill(); 7369 let output = child.wait_with_output().unwrap(); 7370 7371 let _ = child_dnsmasq.kill(); 7372 let _ = child_dnsmasq.wait(); 7373 7374 handle_child_output(r, &output); 7375 } 7376 7377 #[test] 7378 #[cfg(not(feature = "mshv"))] 7379 fn test_windows_guest_disk_hotplug_multi() { 7380 let windows_guest = WindowsGuest::new(); 7381 7382 let mut ovmf_path = dirs::home_dir().unwrap(); 7383 ovmf_path.push("workloads"); 7384 ovmf_path.push(OVMF_NAME); 7385 7386 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7387 let api_socket = temp_api_path(&tmp_dir); 7388 7389 let mut child = GuestCommand::new(windows_guest.guest()) 7390 .args(&["--api-socket", &api_socket]) 7391 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7392 .args(&["--memory", "size=2G"]) 7393 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7394 .args(&["--serial", "tty"]) 7395 .args(&["--console", "off"]) 7396 .default_disks() 7397 .default_net() 7398 .capture_output() 7399 .spawn() 7400 .unwrap(); 7401 7402 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7403 7404 // Predefined data to used at various test stages 7405 let disk_test_data: [[String; 4]; 2] = [ 7406 [ 7407 "_disk2".to_string(), 7408 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 7409 "d:\\world".to_string(), 7410 "hello".to_string(), 7411 ], 7412 [ 7413 "_disk3".to_string(), 7414 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 7415 "e:\\hello".to_string(), 7416 "world".to_string(), 7417 ], 7418 ]; 7419 7420 let r = std::panic::catch_unwind(|| { 7421 // Wait to make sure Windows boots up 7422 assert!(windows_guest.wait_for_boot()); 7423 7424 // Initially present disk device 7425 let disk_num = 1; 7426 assert_eq!(windows_guest.disk_count(), disk_num); 7427 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7428 7429 for it in &disk_test_data { 7430 let disk_id = it[0].as_str(); 7431 let disk = it[1].as_str(); 7432 // Hotplug disk device 7433 let (cmd_success, cmd_output) = remote_command_w_output( 7434 &api_socket, 7435 "add-disk", 7436 Some(format!("path={},readonly=off", disk).as_str()), 7437 ); 7438 assert!(cmd_success); 7439 assert!(String::from_utf8_lossy(&cmd_output) 7440 .contains(format!("\"id\":\"{}\"", disk_id).as_str())); 7441 thread::sleep(std::time::Duration::new(5, 0)); 7442 // Online disk devices 7443 windows_guest.disks_set_rw(); 7444 windows_guest.disks_online(); 7445 } 7446 // Verify the devices are on the system 7447 let disk_num = (disk_test_data.len() + 1) as u8; 7448 assert_eq!(windows_guest.disk_count(), disk_num); 7449 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7450 7451 // Put test data 7452 for it in &disk_test_data { 7453 let fname = it[2].as_str(); 7454 let data = it[3].as_str(); 7455 windows_guest.disk_file_put(fname, data); 7456 } 7457 7458 // Unmount disk devices 7459 for it in &disk_test_data { 7460 let disk_id = it[0].as_str(); 7461 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 7462 assert!(cmd_success); 7463 thread::sleep(std::time::Duration::new(5, 0)); 7464 } 7465 7466 // Verify the devices have been removed 7467 let disk_num = 1; 7468 assert_eq!(windows_guest.disk_count(), disk_num); 7469 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7470 7471 // Remount 7472 for it in &disk_test_data { 7473 let disk = it[1].as_str(); 7474 let (cmd_success, _cmd_output) = remote_command_w_output( 7475 &api_socket, 7476 "add-disk", 7477 Some(format!("path={},readonly=off", disk).as_str()), 7478 ); 7479 assert!(cmd_success); 7480 thread::sleep(std::time::Duration::new(5, 0)); 7481 } 7482 7483 // Check the files exists with the expected contents 7484 for it in &disk_test_data { 7485 let fname = it[2].as_str(); 7486 let data = it[3].as_str(); 7487 let out = windows_guest.disk_file_read(fname); 7488 assert_eq!(data, out.trim()); 7489 } 7490 7491 // Intentionally no unmount, it'll happen at shutdown. 7492 7493 windows_guest.shutdown(); 7494 }); 7495 7496 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7497 let _ = child.kill(); 7498 let output = child.wait_with_output().unwrap(); 7499 7500 let _ = child_dnsmasq.kill(); 7501 let _ = child_dnsmasq.wait(); 7502 7503 handle_child_output(r, &output); 7504 } 7505 7506 #[test] 7507 #[cfg(not(feature = "mshv"))] 7508 fn test_windows_guest_netdev_multi() { 7509 let windows_guest = WindowsGuest::new(); 7510 7511 let mut ovmf_path = dirs::home_dir().unwrap(); 7512 ovmf_path.push("workloads"); 7513 ovmf_path.push(OVMF_NAME); 7514 7515 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7516 let api_socket = temp_api_path(&tmp_dir); 7517 7518 let mut child = GuestCommand::new(windows_guest.guest()) 7519 .args(&["--api-socket", &api_socket]) 7520 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7521 .args(&["--memory", "size=4G"]) 7522 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7523 .args(&["--serial", "tty"]) 7524 .args(&["--console", "off"]) 7525 .default_disks() 7526 // The multi net dev config is borrowed from test_multiple_network_interfaces 7527 .args(&[ 7528 "--net", 7529 windows_guest.guest().default_net_string().as_str(), 7530 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 7531 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 7532 ]) 7533 .capture_output() 7534 .spawn() 7535 .unwrap(); 7536 7537 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7538 7539 let r = std::panic::catch_unwind(|| { 7540 // Wait to make sure Windows boots up 7541 assert!(windows_guest.wait_for_boot()); 7542 7543 let netdev_num = 3; 7544 assert_eq!(windows_guest.netdev_count(), netdev_num); 7545 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7546 7547 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 7548 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 7549 7550 windows_guest.shutdown(); 7551 }); 7552 7553 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7554 let _ = child.kill(); 7555 let output = child.wait_with_output().unwrap(); 7556 7557 let _ = child_dnsmasq.kill(); 7558 let _ = child_dnsmasq.wait(); 7559 7560 handle_child_output(r, &output); 7561 } 7562 } 7563 7564 #[cfg(target_arch = "x86_64")] 7565 mod sgx { 7566 use crate::*; 7567 7568 #[test] 7569 fn test_sgx() { 7570 let focal = UbuntuDiskConfig::new(FOCAL_SGX_IMAGE_NAME.to_string()); 7571 let guest = Guest::new(Box::new(focal)); 7572 let mut workload_path = dirs::home_dir().unwrap(); 7573 workload_path.push("workloads"); 7574 7575 let mut kernel_path = workload_path; 7576 kernel_path.push("vmlinux_w_sgx"); 7577 7578 let mut child = GuestCommand::new(&guest) 7579 .args(&["--cpus", "boot=1"]) 7580 .args(&["--memory", "size=512M"]) 7581 .args(&["--kernel", kernel_path.to_str().unwrap()]) 7582 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7583 .default_disks() 7584 .default_net() 7585 .args(&["--sgx-epc", "id=epc0,size=64M"]) 7586 .capture_output() 7587 .spawn() 7588 .unwrap(); 7589 7590 let r = std::panic::catch_unwind(|| { 7591 guest.wait_vm_boot(None).unwrap(); 7592 7593 // Check if SGX is correctly detected in the guest. 7594 guest.check_sgx_support().unwrap(); 7595 7596 // Validate the SGX EPC section is 64MiB. 7597 assert_eq!( 7598 guest 7599 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 7600 .unwrap() 7601 .trim(), 7602 "0x0000000004000000" 7603 ); 7604 7605 // Run a test relying on SGX enclaves and check if it runs 7606 // successfully. 7607 assert!(guest 7608 .ssh_command("cd /linux-sgx/SampleCode/LocalAttestation/bin/ && sudo ./app") 7609 .unwrap() 7610 .trim() 7611 .contains( 7612 "succeed to load enclaves.\nsucceed to \ 7613 establish secure channel.\nSucceed to exchange \ 7614 secure message...\nSucceed to close Session..." 7615 )); 7616 }); 7617 7618 let _ = child.kill(); 7619 let output = child.wait_with_output().unwrap(); 7620 7621 handle_child_output(r, &output); 7622 } 7623 } 7624 7625 #[cfg(target_arch = "x86_64")] 7626 mod vfio { 7627 use crate::*; 7628 7629 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 7630 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 7631 let guest = Guest::new(Box::new(hirsute)); 7632 let api_socket = temp_api_path(&guest.tmp_dir); 7633 7634 let mut child = GuestCommand::new(&guest) 7635 .args(&["--cpus", "boot=4"]) 7636 .args(&[ 7637 "--memory", 7638 format!("size=4G,hotplug_size=4G,hotplug_method={}", hotplug_method).as_str(), 7639 ]) 7640 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7641 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 7642 .args(&["--api-socket", &api_socket]) 7643 .default_disks() 7644 .default_net() 7645 .capture_output() 7646 .spawn() 7647 .unwrap(); 7648 7649 let r = std::panic::catch_unwind(|| { 7650 guest.wait_vm_boot(None).unwrap(); 7651 7652 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 7653 7654 guest.enable_memory_hotplug(); 7655 7656 // Add RAM to the VM 7657 let desired_ram = 6 << 30; 7658 resize_command(&api_socket, None, Some(desired_ram), None, None); 7659 thread::sleep(std::time::Duration::new(30, 0)); 7660 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7661 7662 // Check the VFIO device works when RAM is increased to 6GiB 7663 guest.check_nvidia_gpu(); 7664 }); 7665 7666 let _ = child.kill(); 7667 let output = child.wait_with_output().unwrap(); 7668 7669 handle_child_output(r, &output); 7670 } 7671 7672 #[test] 7673 fn test_nvidia_card_memory_hotplug_acpi() { 7674 test_nvidia_card_memory_hotplug("acpi") 7675 } 7676 7677 #[test] 7678 fn test_nvidia_card_memory_hotplug_virtio_mem() { 7679 test_nvidia_card_memory_hotplug("virtio-mem") 7680 } 7681 7682 #[test] 7683 fn test_nvidia_card_pci_hotplug() { 7684 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 7685 let guest = Guest::new(Box::new(hirsute)); 7686 let api_socket = temp_api_path(&guest.tmp_dir); 7687 7688 let mut child = GuestCommand::new(&guest) 7689 .args(&["--cpus", "boot=4"]) 7690 .args(&["--memory", "size=4G"]) 7691 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7692 .args(&["--api-socket", &api_socket]) 7693 .default_disks() 7694 .default_net() 7695 .capture_output() 7696 .spawn() 7697 .unwrap(); 7698 7699 let r = std::panic::catch_unwind(|| { 7700 guest.wait_vm_boot(None).unwrap(); 7701 7702 // Hotplug the card to the VM 7703 let (cmd_success, cmd_output) = remote_command_w_output( 7704 &api_socket, 7705 "add-device", 7706 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 7707 ); 7708 assert!(cmd_success); 7709 assert!(String::from_utf8_lossy(&cmd_output) 7710 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 7711 7712 thread::sleep(std::time::Duration::new(10, 0)); 7713 7714 // Check the VFIO device works after hotplug 7715 guest.check_nvidia_gpu(); 7716 }); 7717 7718 let _ = child.kill(); 7719 let output = child.wait_with_output().unwrap(); 7720 7721 handle_child_output(r, &output); 7722 } 7723 7724 #[test] 7725 fn test_nvidia_card_reboot() { 7726 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 7727 let guest = Guest::new(Box::new(hirsute)); 7728 let api_socket = temp_api_path(&guest.tmp_dir); 7729 7730 let mut child = GuestCommand::new(&guest) 7731 .args(&["--cpus", "boot=4"]) 7732 .args(&["--memory", "size=4G"]) 7733 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7734 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 7735 .args(&["--api-socket", &api_socket]) 7736 .default_disks() 7737 .default_net() 7738 .capture_output() 7739 .spawn() 7740 .unwrap(); 7741 7742 let r = std::panic::catch_unwind(|| { 7743 guest.wait_vm_boot(None).unwrap(); 7744 7745 // Check the VFIO device works after boot 7746 guest.check_nvidia_gpu(); 7747 7748 guest.reboot_linux(0, None); 7749 7750 // Check the VFIO device works after reboot 7751 guest.check_nvidia_gpu(); 7752 }); 7753 7754 let _ = child.kill(); 7755 let output = child.wait_with_output().unwrap(); 7756 7757 handle_child_output(r, &output); 7758 } 7759 } 7760 7761 mod live_migration { 7762 use crate::*; 7763 7764 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 7765 // same host. It ensures the following behaviors: 7766 // 1. The source VM is up and functional (including various virtio-devices are working properly); 7767 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 7768 // 3. The source VM terminated gracefully after live migration; 7769 // 4. The destination VM is functional (including various virtio-devices are working properly) after 7770 // live migration; 7771 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 7772 fn _test_live_migration( 7773 upgrade_test: bool, 7774 numa: bool, 7775 local: bool, 7776 watchdog: bool, 7777 balloon: bool, 7778 ) { 7779 assert!( 7780 !(numa && balloon), 7781 "Invalid inputs: 'numa' and 'balloon' cannot be tested at the same time." 7782 ); 7783 7784 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7785 let guest = Guest::new(Box::new(focal)); 7786 let kernel_path = direct_kernel_boot_path(); 7787 let console_text = String::from("On a branch floating down river a cricket, singing."); 7788 let net_id = "net123"; 7789 let net_params = format!( 7790 "id={},tap=,mac={},ip={},mask=255.255.255.0", 7791 net_id, guest.network.guest_mac, guest.network.host_ip 7792 ); 7793 7794 let memory_param: &[&str] = match (balloon, local, numa) { 7795 (false, false, false) => &["--memory", "size=4G"], 7796 (false, false, true) => &[ 7797 "--memory", 7798 "size=0,hotplug_method=virtio-mem", 7799 "--memory-zone", 7800 "id=mem0,size=1G,hotplug_size=4G", 7801 "id=mem1,size=1G,hotplug_size=4G", 7802 "id=mem2,size=2G,hotplug_size=4G", 7803 "--numa", 7804 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 7805 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 7806 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 7807 ], 7808 (false, true, false) => &["--memory", "size=4G,shared=on"], 7809 (false, true, true) => &[ 7810 "--memory", 7811 "size=0,hotplug_method=virtio-mem,shared=on", 7812 "--memory-zone", 7813 "id=mem0,size=1G,hotplug_size=4G,shared=on", 7814 "id=mem1,size=1G,hotplug_size=4G,shared=on", 7815 "id=mem2,size=2G,hotplug_size=4G,shared=on", 7816 "--numa", 7817 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 7818 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 7819 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 7820 ], 7821 (true, false, _) => &[ 7822 "--memory", 7823 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 7824 "--balloon", 7825 "size=0", 7826 ], 7827 (true, true, _) => &[ 7828 "--memory", 7829 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 7830 "--balloon", 7831 "size=0", 7832 ], 7833 }; 7834 7835 let pmem_temp_file = TempFile::new().unwrap(); 7836 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 7837 std::process::Command::new("mkfs.ext4") 7838 .arg(pmem_temp_file.as_path()) 7839 .output() 7840 .expect("Expect creating disk image to succeed"); 7841 let pmem_path = String::from("/dev/pmem0"); 7842 7843 // Start the source VM 7844 let src_vm_path = if !upgrade_test { 7845 clh_command("cloud-hypervisor") 7846 } else { 7847 cloud_hypervisor_release_path() 7848 }; 7849 let src_api_socket = temp_api_path(&guest.tmp_dir); 7850 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 7851 src_vm_cmd 7852 .args(&["--cpus", "boot=6,max=12"]) 7853 .args(memory_param) 7854 .args(&["--kernel", kernel_path.to_str().unwrap()]) 7855 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7856 .default_disks() 7857 .args(&["--net", net_params.as_str()]) 7858 .args(&["--api-socket", &src_api_socket]) 7859 .args(&[ 7860 "--pmem", 7861 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 7862 ]); 7863 if watchdog { 7864 src_vm_cmd.args(&["--watchdog"]); 7865 } 7866 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 7867 7868 // Start the destination VM 7869 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 7870 dest_api_socket.push_str(".dest"); 7871 let mut dest_child = GuestCommand::new(&guest) 7872 .args(&["--api-socket", &dest_api_socket]) 7873 .capture_output() 7874 .spawn() 7875 .unwrap(); 7876 7877 let r = std::panic::catch_unwind(|| { 7878 guest.wait_vm_boot(None).unwrap(); 7879 7880 // Make sure the source VM is functaionl 7881 // Check the number of vCPUs 7882 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 6); 7883 7884 // Check the guest RAM 7885 if balloon { 7886 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 7887 // Increase the guest RAM 7888 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 7889 thread::sleep(std::time::Duration::new(5, 0)); 7890 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7891 // Use balloon to remove RAM from the VM 7892 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 7893 thread::sleep(std::time::Duration::new(5, 0)); 7894 let total_memory = guest.get_total_memory().unwrap_or_default(); 7895 assert!(total_memory > 4_800_000); 7896 assert!(total_memory < 5_760_000); 7897 } else if numa { 7898 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 7899 } else { 7900 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 7901 } 7902 7903 // Check the guest virtio-devices, e.g. block, rng, console, and net 7904 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 7905 7906 // Check the NUMA parameters are applied correctly and resize 7907 // each zone to test the case where we migrate a VM with the 7908 // virtio-mem regions being used. 7909 if numa { 7910 guest.check_numa_common( 7911 Some(&[960_000, 960_000, 1_920_000]), 7912 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 7913 Some(&["10 15 20", "20 10 25", "25 30 10"]), 7914 ); 7915 7916 // AArch64 currently does not support hotplug, and therefore we only 7917 // test hotplug-related function on x86_64 here. 7918 #[cfg(target_arch = "x86_64")] 7919 { 7920 guest.enable_memory_hotplug(); 7921 7922 // Resize every memory zone and check each associated NUMA node 7923 // has been assigned the right amount of memory. 7924 resize_zone_command(&src_api_socket, "mem0", "2G"); 7925 resize_zone_command(&src_api_socket, "mem1", "2G"); 7926 resize_zone_command(&src_api_socket, "mem2", "3G"); 7927 thread::sleep(std::time::Duration::new(5, 0)); 7928 7929 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 7930 } 7931 } 7932 7933 // x86_64: Following what's done in the `test_snapshot_restore`, we need 7934 // to make sure that removing and adding back the virtio-net device does 7935 // not break the live-migration support for virtio-pci. 7936 #[cfg(target_arch = "x86_64")] 7937 { 7938 assert!(remote_command( 7939 &src_api_socket, 7940 "remove-device", 7941 Some(net_id), 7942 )); 7943 thread::sleep(std::time::Duration::new(10, 0)); 7944 7945 // Plug the virtio-net device again 7946 assert!(remote_command( 7947 &src_api_socket, 7948 "add-net", 7949 Some(net_params.as_str()), 7950 )); 7951 thread::sleep(std::time::Duration::new(10, 0)); 7952 } 7953 7954 if watchdog { 7955 let mut current_reboot_count = 1; 7956 enable_guest_watchdog(&guest, &mut current_reboot_count); 7957 check_guest_watchdog_no_reboot(&guest, ¤t_reboot_count); 7958 } 7959 7960 // Start the live-migration 7961 let migration_socket = String::from( 7962 guest 7963 .tmp_dir 7964 .as_path() 7965 .join("live-migration.sock") 7966 .to_str() 7967 .unwrap(), 7968 ); 7969 // Start to receive migration from the destintion VM 7970 let mut receive_migration = Command::new(clh_command("ch-remote")) 7971 .args(&[ 7972 &format!("--api-socket={}", &dest_api_socket), 7973 "receive-migration", 7974 &format! {"unix:{}", migration_socket}, 7975 ]) 7976 .stderr(Stdio::piped()) 7977 .stdout(Stdio::piped()) 7978 .spawn() 7979 .unwrap(); 7980 // Give it '1s' to make sure the 'migration_socket' file is properly created 7981 thread::sleep(std::time::Duration::new(1, 0)); 7982 // Start to send migration from the source VM 7983 7984 let mut args = [ 7985 format!("--api-socket={}", &src_api_socket), 7986 "send-migration".to_string(), 7987 format! {"unix:{}", migration_socket}, 7988 ] 7989 .to_vec(); 7990 7991 if local { 7992 args.insert(2, "--local".to_string()); 7993 } 7994 7995 let mut send_migration = Command::new(clh_command("ch-remote")) 7996 .args(&args) 7997 .stderr(Stdio::piped()) 7998 .stdout(Stdio::piped()) 7999 .spawn() 8000 .unwrap(); 8001 8002 // The 'send-migration' command should be executed successfully within the given timeout 8003 let success = if let Some(status) = send_migration 8004 .wait_timeout(std::time::Duration::from_secs(30)) 8005 .unwrap() 8006 { 8007 status.success() 8008 } else { 8009 false 8010 }; 8011 8012 if !success { 8013 let _ = send_migration.kill(); 8014 let output = send_migration.wait_with_output().unwrap(); 8015 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 8016 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8017 } 8018 8019 // The 'receive-migration' command should be executed successfully within the given timeout 8020 let success = if let Some(status) = receive_migration 8021 .wait_timeout(std::time::Duration::from_secs(30)) 8022 .unwrap() 8023 { 8024 status.success() 8025 } else { 8026 false 8027 }; 8028 8029 if !success { 8030 let _ = receive_migration.kill(); 8031 let output = receive_migration.wait_with_output().unwrap(); 8032 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 8033 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8034 } 8035 assert!( 8036 success, 8037 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8038 ); 8039 }); 8040 8041 let print_and_panic = |src_vm: Child, dest_vm: Child, message: &str| -> ! { 8042 let mut src_vm = src_vm; 8043 let mut dest_vm = dest_vm; 8044 8045 let _ = src_vm.kill(); 8046 let src_output = src_vm.wait_with_output().unwrap(); 8047 eprintln!( 8048 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8049 String::from_utf8_lossy(&src_output.stdout) 8050 ); 8051 eprintln!( 8052 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8053 String::from_utf8_lossy(&src_output.stderr) 8054 ); 8055 let _ = dest_vm.kill(); 8056 let dest_output = dest_vm.wait_with_output().unwrap(); 8057 eprintln!( 8058 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8059 String::from_utf8_lossy(&dest_output.stdout) 8060 ); 8061 eprintln!( 8062 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8063 String::from_utf8_lossy(&dest_output.stderr) 8064 ); 8065 8066 panic!("Test failed: {}", message) 8067 }; 8068 8069 // Check and report any errors occured during the live-migration 8070 if r.is_err() { 8071 print_and_panic(src_child, dest_child, "Error occured during live-migration"); 8072 } 8073 8074 // Check the source vm has been terminated successful (give it '3s' to settle) 8075 thread::sleep(std::time::Duration::new(3, 0)); 8076 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8077 print_and_panic( 8078 src_child, 8079 dest_child, 8080 "source VM was not terminated successfully.", 8081 ); 8082 }; 8083 8084 // Post live-migration check to make sure the destination VM is funcational 8085 let r = std::panic::catch_unwind(|| { 8086 // Perform same checks to validate VM has been properly migrated 8087 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 6); 8088 #[cfg(target_arch = "x86_64")] 8089 if numa { 8090 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 8091 } else { 8092 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8093 } 8094 #[cfg(target_arch = "aarch64")] 8095 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8096 8097 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8098 8099 // Perform NUMA related checks 8100 if numa { 8101 #[cfg(target_arch = "aarch64")] 8102 { 8103 guest.check_numa_common( 8104 Some(&[960_000, 960_000, 1_920_000]), 8105 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8106 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8107 ); 8108 } 8109 8110 // AArch64 currently does not support hotplug, and therefore we only 8111 // test hotplug-related function on x86_64 here. 8112 #[cfg(target_arch = "x86_64")] 8113 { 8114 guest.check_numa_common( 8115 Some(&[1_920_000, 1_920_000, 2_880_000]), 8116 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8117 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8118 ); 8119 8120 guest.enable_memory_hotplug(); 8121 8122 // Resize every memory zone and check each associated NUMA node 8123 // has been assigned the right amount of memory. 8124 resize_zone_command(&dest_api_socket, "mem0", "4G"); 8125 resize_zone_command(&dest_api_socket, "mem1", "4G"); 8126 resize_zone_command(&dest_api_socket, "mem2", "4G"); 8127 // Resize to the maximum amount of CPUs and check each NUMA 8128 // node has been assigned the right CPUs set. 8129 resize_command(&dest_api_socket, Some(12), None, None, None); 8130 thread::sleep(std::time::Duration::new(5, 0)); 8131 8132 guest.check_numa_common( 8133 Some(&[3_840_000, 3_840_000, 3_840_000]), 8134 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 8135 None, 8136 ); 8137 } 8138 } 8139 8140 if watchdog { 8141 let mut current_reboot_count = 2; 8142 check_guest_watchdog_no_reboot(&guest, ¤t_reboot_count); 8143 check_guest_watchdog_one_reboot( 8144 &guest, 8145 &dest_api_socket, 8146 &mut current_reboot_count, 8147 ); 8148 } 8149 8150 if balloon { 8151 let total_memory = guest.get_total_memory().unwrap_or_default(); 8152 assert!(total_memory > 4_800_000); 8153 assert!(total_memory < 5_760_000); 8154 // Deflate balloon to restore entire RAM to the VM 8155 resize_command(&dest_api_socket, None, None, Some(0), None); 8156 thread::sleep(std::time::Duration::new(5, 0)); 8157 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8158 // Decrease guest RAM with virtio-mem 8159 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 8160 thread::sleep(std::time::Duration::new(5, 0)); 8161 let total_memory = guest.get_total_memory().unwrap_or_default(); 8162 assert!(total_memory > 4_800_000); 8163 assert!(total_memory < 5_760_000); 8164 } 8165 }); 8166 8167 // Clean-up the destination VM and make sure it terminated correctly 8168 let _ = dest_child.kill(); 8169 let dest_output = dest_child.wait_with_output().unwrap(); 8170 handle_child_output(r, &dest_output); 8171 8172 // Check the destination VM has the expected 'concole_text' from its output 8173 let r = std::panic::catch_unwind(|| { 8174 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8175 }); 8176 handle_child_output(r, &dest_output); 8177 } 8178 8179 #[test] 8180 fn test_live_migration_basic() { 8181 _test_live_migration(false, false, false, false, false) 8182 } 8183 8184 #[test] 8185 fn test_live_migration_local() { 8186 _test_live_migration(false, false, true, false, false) 8187 } 8188 8189 #[test] 8190 #[cfg(not(feature = "mshv"))] 8191 fn test_live_migration_numa() { 8192 _test_live_migration(false, true, false, false, false) 8193 } 8194 8195 #[test] 8196 #[cfg(not(feature = "mshv"))] 8197 fn test_live_migration_numa_local() { 8198 _test_live_migration(false, true, true, false, false) 8199 } 8200 8201 #[test] 8202 fn test_live_migration_watchdog() { 8203 _test_live_migration(false, false, false, true, false) 8204 } 8205 8206 #[test] 8207 fn test_live_migration_watchdog_local() { 8208 _test_live_migration(false, false, true, true, false) 8209 } 8210 8211 #[test] 8212 fn test_live_migration_balloon() { 8213 _test_live_migration(false, false, false, false, true) 8214 } 8215 8216 #[test] 8217 fn test_live_migration_balloon_local() { 8218 _test_live_migration(false, false, true, false, true) 8219 } 8220 8221 #[test] 8222 #[ignore] 8223 fn test_live_upgrade_basic() { 8224 _test_live_migration(true, false, false, false, false) 8225 } 8226 8227 #[test] 8228 #[ignore] 8229 fn test_live_upgrade_local() { 8230 _test_live_migration(true, false, true, false, false) 8231 } 8232 8233 #[test] 8234 #[ignore] 8235 #[cfg(not(feature = "mshv"))] 8236 fn test_live_upgrade_numa() { 8237 _test_live_migration(true, true, false, false, false) 8238 } 8239 8240 #[test] 8241 #[ignore] 8242 #[cfg(not(feature = "mshv"))] 8243 fn test_live_upgrade_numa_local() { 8244 _test_live_migration(true, true, true, false, false) 8245 } 8246 8247 #[test] 8248 #[ignore] 8249 fn test_live_upgrade_watchdog() { 8250 _test_live_migration(true, false, false, true, false) 8251 } 8252 8253 #[test] 8254 #[ignore] 8255 fn test_live_upgrade_watchdog_local() { 8256 _test_live_migration(true, false, true, true, false) 8257 } 8258 8259 #[test] 8260 #[ignore] 8261 fn test_live_upgrade_balloon() { 8262 _test_live_migration(true, false, false, false, true) 8263 } 8264 8265 #[test] 8266 #[ignore] 8267 fn test_live_upgrade_balloon_local() { 8268 _test_live_migration(true, false, true, false, true) 8269 } 8270 8271 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 8272 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8273 let ovs_guest = Guest::new(Box::new(ovs_focal)); 8274 8275 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8276 let migration_guest = Guest::new(Box::new(migration_focal)); 8277 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 8278 8279 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 8280 let (mut ovs_child, mut src_child) = 8281 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 8282 8283 // Start the destination VM 8284 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 8285 dest_api_socket.push_str(".dest"); 8286 let mut dest_child = GuestCommand::new(&migration_guest) 8287 .args(&["--api-socket", &dest_api_socket]) 8288 .capture_output() 8289 .spawn() 8290 .unwrap(); 8291 8292 let r = std::panic::catch_unwind(|| { 8293 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 8294 thread::sleep(std::time::Duration::new(1, 0)); 8295 8296 // Start the live-migration 8297 let migration_socket = String::from( 8298 migration_guest 8299 .tmp_dir 8300 .as_path() 8301 .join("live-migration.sock") 8302 .to_str() 8303 .unwrap(), 8304 ); 8305 // Start to receive migration from the destintion VM 8306 let mut receive_migration = Command::new(clh_command("ch-remote")) 8307 .args(&[ 8308 &format!("--api-socket={}", &dest_api_socket), 8309 "receive-migration", 8310 &format! {"unix:{}", migration_socket}, 8311 ]) 8312 .stderr(Stdio::piped()) 8313 .stdout(Stdio::piped()) 8314 .spawn() 8315 .unwrap(); 8316 // Give it '1s' to make sure the 'migration_socket' file is properly created 8317 thread::sleep(std::time::Duration::new(1, 0)); 8318 // Start to send migration from the source VM 8319 let mut args = [ 8320 format!("--api-socket={}", &src_api_socket), 8321 "send-migration".to_string(), 8322 format! {"unix:{}", migration_socket}, 8323 ] 8324 .to_vec(); 8325 8326 if local { 8327 args.insert(2, "--local".to_string()); 8328 } 8329 8330 let mut send_migration = Command::new(clh_command("ch-remote")) 8331 .args(&args) 8332 .stderr(Stdio::piped()) 8333 .stdout(Stdio::piped()) 8334 .spawn() 8335 .unwrap(); 8336 8337 // The 'send-migration' command should be executed successfully within the given timeout 8338 let success = if let Some(status) = send_migration 8339 .wait_timeout(std::time::Duration::from_secs(30)) 8340 .unwrap() 8341 { 8342 status.success() 8343 } else { 8344 false 8345 }; 8346 8347 if !success { 8348 let _ = send_migration.kill(); 8349 let output = send_migration.wait_with_output().unwrap(); 8350 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 8351 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8352 } 8353 8354 // The 'receive-migration' command should be executed successfully within the given timeout 8355 let success = if let Some(status) = receive_migration 8356 .wait_timeout(std::time::Duration::from_secs(30)) 8357 .unwrap() 8358 { 8359 status.success() 8360 } else { 8361 false 8362 }; 8363 8364 if !success { 8365 let _ = receive_migration.kill(); 8366 let output = receive_migration.wait_with_output().unwrap(); 8367 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 8368 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8369 } 8370 assert!( 8371 success, 8372 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8373 ); 8374 }); 8375 8376 let print_and_panic = |src_vm: Child, dest_vm: Child, ovs_vm: Child, message: &str| -> ! { 8377 let mut src_vm = src_vm; 8378 let mut dest_vm = dest_vm; 8379 let mut ovs_vm = ovs_vm; 8380 8381 let _ = src_vm.kill(); 8382 let src_output = src_vm.wait_with_output().unwrap(); 8383 eprintln!( 8384 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8385 String::from_utf8_lossy(&src_output.stdout) 8386 ); 8387 eprintln!( 8388 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8389 String::from_utf8_lossy(&src_output.stderr) 8390 ); 8391 let _ = dest_vm.kill(); 8392 let dest_output = dest_vm.wait_with_output().unwrap(); 8393 eprintln!( 8394 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8395 String::from_utf8_lossy(&dest_output.stdout) 8396 ); 8397 eprintln!( 8398 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8399 String::from_utf8_lossy(&dest_output.stderr) 8400 ); 8401 let _ = ovs_vm.kill(); 8402 let ovs_output = ovs_vm.wait_with_output().unwrap(); 8403 eprintln!( 8404 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 8405 String::from_utf8_lossy(&ovs_output.stdout) 8406 ); 8407 eprintln!( 8408 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 8409 String::from_utf8_lossy(&ovs_output.stderr) 8410 ); 8411 8412 cleanup_ovs_dpdk(); 8413 8414 panic!("Test failed: {}", message) 8415 }; 8416 8417 // Check and report any errors occured during the live-migration 8418 if r.is_err() { 8419 print_and_panic( 8420 src_child, 8421 dest_child, 8422 ovs_child, 8423 "Error occured during live-migration", 8424 ); 8425 } 8426 8427 // Check the source vm has been terminated successful (give it '3s' to settle) 8428 thread::sleep(std::time::Duration::new(3, 0)); 8429 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8430 print_and_panic( 8431 src_child, 8432 dest_child, 8433 ovs_child, 8434 "source VM was not terminated successfully.", 8435 ); 8436 }; 8437 8438 // Post live-migration check to make sure the destination VM is funcational 8439 let r = std::panic::catch_unwind(|| { 8440 // Perform same checks to validate VM has been properly migrated 8441 // Spawn a new netcat listener in the OVS VM 8442 let guest_ip = ovs_guest.network.guest_ip.clone(); 8443 thread::spawn(move || { 8444 ssh_command_ip( 8445 "nc -l 12345", 8446 &guest_ip, 8447 DEFAULT_SSH_RETRIES, 8448 DEFAULT_SSH_TIMEOUT, 8449 ) 8450 .unwrap(); 8451 }); 8452 8453 // Wait for the server to be listening 8454 thread::sleep(std::time::Duration::new(5, 0)); 8455 8456 // And check the connection is still functional after live-migration 8457 migration_guest 8458 .ssh_command("nc -vz 172.100.0.1 12345") 8459 .unwrap(); 8460 8461 cleanup_ovs_dpdk(); 8462 }); 8463 8464 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 8465 let _ = dest_child.kill(); 8466 let _ = ovs_child.kill(); 8467 let dest_output = dest_child.wait_with_output().unwrap(); 8468 handle_child_output(r, &dest_output); 8469 let ovs_output = ovs_child.wait_with_output().unwrap(); 8470 handle_child_output(Ok(()), &ovs_output); 8471 } 8472 8473 #[test] 8474 #[cfg(not(feature = "mshv"))] 8475 fn test_live_migration_ovs_dpdk() { 8476 _test_live_migration_ovs_dpdk(false, false); 8477 } 8478 8479 #[test] 8480 #[cfg(not(feature = "mshv"))] 8481 fn test_live_migration_ovs_dpdk_local() { 8482 _test_live_migration_ovs_dpdk(false, true); 8483 } 8484 8485 #[test] 8486 #[ignore] 8487 #[cfg(not(feature = "mshv"))] 8488 fn test_live_upgrade_ovs_dpdk() { 8489 _test_live_migration_ovs_dpdk(true, false); 8490 } 8491 8492 #[test] 8493 #[ignore] 8494 #[cfg(not(feature = "mshv"))] 8495 fn test_live_upgrade_ovs_dpdk_local() { 8496 _test_live_migration_ovs_dpdk(true, true); 8497 } 8498 } 8499 8500 #[cfg(target_arch = "aarch64")] 8501 mod aarch64_acpi { 8502 use crate::*; 8503 8504 #[test] 8505 fn test_simple_launch_acpi() { 8506 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8507 8508 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 8509 let guest = Guest::new(disk_config); 8510 8511 let mut child = GuestCommand::new(&guest) 8512 .args(&["--cpus", "boot=1"]) 8513 .args(&["--memory", "size=512M"]) 8514 .args(&["--kernel", edk2_path().to_str().unwrap()]) 8515 .default_disks() 8516 .default_net() 8517 .args(&["--serial", "tty", "--console", "off"]) 8518 .capture_output() 8519 .spawn() 8520 .unwrap(); 8521 8522 let r = std::panic::catch_unwind(|| { 8523 guest.wait_vm_boot(Some(120)).unwrap(); 8524 8525 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 8526 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 8527 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 8528 }); 8529 8530 let _ = child.kill(); 8531 let output = child.wait_with_output().unwrap(); 8532 8533 handle_child_output(r, &output); 8534 }); 8535 } 8536 8537 #[test] 8538 fn test_guest_numa_nodes_acpi() { 8539 _test_guest_numa_nodes(true); 8540 } 8541 8542 #[test] 8543 fn test_cpu_topology_421_acpi() { 8544 test_cpu_topology(4, 2, 1, true); 8545 } 8546 8547 #[test] 8548 fn test_cpu_topology_142_acpi() { 8549 test_cpu_topology(1, 4, 2, true); 8550 } 8551 8552 #[test] 8553 fn test_cpu_topology_262_acpi() { 8554 test_cpu_topology(2, 6, 2, true); 8555 } 8556 8557 #[test] 8558 fn test_power_button_acpi() { 8559 _test_power_button(true); 8560 } 8561 8562 #[test] 8563 fn test_virtio_iommu() { 8564 _test_virtio_iommu(true) 8565 } 8566 } 8567