1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 #[cfg(target_arch = "x86_64")] 28 use std::sync::Mutex; 29 use std::thread; 30 use test_infra::*; 31 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 32 use wait_timeout::ChildExt; 33 34 #[cfg(target_arch = "x86_64")] 35 mod x86_64 { 36 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw"; 37 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 38 pub const FOCAL_SGX_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-sgx.raw"; 39 pub const HIRSUTE_NVIDIA_IMAGE_NAME: &str = "hirsute-server-cloudimg-amd64-nvidia.raw"; 40 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 41 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 42 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 43 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20220329-0.raw"; 44 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2019.raw"; 45 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 46 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 47 } 48 49 #[cfg(target_arch = "x86_64")] 50 use x86_64::*; 51 52 #[cfg(target_arch = "aarch64")] 53 mod aarch64 { 54 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 55 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 56 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 57 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 58 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 59 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 60 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 61 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 62 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 63 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 64 } 65 66 #[cfg(target_arch = "aarch64")] 67 use aarch64::*; 68 69 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 70 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 71 72 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 73 74 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 75 let mut workload_path = dirs::home_dir().unwrap(); 76 workload_path.push("workloads"); 77 78 let mut virtiofsd_path = workload_path; 79 virtiofsd_path.push("virtiofsd"); 80 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 81 82 let virtiofsd_socket_path = 83 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 84 85 // Start the daemon 86 let child = Command::new(virtiofsd_path.as_str()) 87 .args(&["--shared-dir", shared_dir]) 88 .args(&["--socket-path", virtiofsd_socket_path.as_str()]) 89 .args(&["--cache", "never"]) 90 .spawn() 91 .unwrap(); 92 93 thread::sleep(std::time::Duration::new(10, 0)); 94 95 (child, virtiofsd_socket_path) 96 } 97 98 fn prepare_vubd( 99 tmp_dir: &TempDir, 100 blk_img: &str, 101 num_queues: usize, 102 rdonly: bool, 103 direct: bool, 104 ) -> (std::process::Child, String) { 105 let mut workload_path = dirs::home_dir().unwrap(); 106 workload_path.push("workloads"); 107 108 let mut blk_file_path = workload_path; 109 blk_file_path.push(blk_img); 110 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 111 112 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 113 114 // Start the daemon 115 let child = Command::new(clh_command("vhost_user_block")) 116 .args(&[ 117 "--block-backend", 118 format!( 119 "path={},socket={},num_queues={},readonly={},direct={}", 120 blk_file_path, vubd_socket_path, num_queues, rdonly, direct 121 ) 122 .as_str(), 123 ]) 124 .spawn() 125 .unwrap(); 126 127 thread::sleep(std::time::Duration::new(10, 0)); 128 129 (child, vubd_socket_path) 130 } 131 132 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 133 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 134 } 135 136 fn temp_api_path(tmp_dir: &TempDir) -> String { 137 String::from( 138 tmp_dir 139 .as_path() 140 .join("cloud-hypervisor.sock") 141 .to_str() 142 .unwrap(), 143 ) 144 } 145 146 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 147 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 148 } 149 150 // Creates the directory and returns the path. 151 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 152 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 153 std::fs::create_dir(&snapshot_dir).unwrap(); 154 snapshot_dir 155 } 156 157 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 158 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 159 vmcore_file 160 } 161 162 // Creates the path for direct kernel boot and return the path. 163 // For x86_64, this function returns the vmlinux kernel path. 164 // For AArch64, this function returns the PE kernel path. 165 fn direct_kernel_boot_path() -> PathBuf { 166 let mut workload_path = dirs::home_dir().unwrap(); 167 workload_path.push("workloads"); 168 169 let mut kernel_path = workload_path; 170 #[cfg(target_arch = "x86_64")] 171 kernel_path.push("vmlinux"); 172 #[cfg(target_arch = "aarch64")] 173 kernel_path.push("Image"); 174 175 kernel_path 176 } 177 178 #[cfg(target_arch = "aarch64")] 179 fn edk2_path() -> PathBuf { 180 let mut workload_path = dirs::home_dir().unwrap(); 181 workload_path.push("workloads"); 182 let mut edk2_path = workload_path; 183 edk2_path.push("CLOUDHV_EFI.fd"); 184 185 edk2_path 186 } 187 188 fn cloud_hypervisor_release_path() -> String { 189 let mut workload_path = dirs::home_dir().unwrap(); 190 workload_path.push("workloads"); 191 192 let mut ch_release_path = workload_path; 193 #[cfg(target_arch = "x86_64")] 194 ch_release_path.push("cloud-hypervisor-static"); 195 #[cfg(target_arch = "aarch64")] 196 ch_release_path.push("cloud-hypervisor-static-aarch64"); 197 198 ch_release_path.into_os_string().into_string().unwrap() 199 } 200 201 fn prepare_vhost_user_net_daemon( 202 tmp_dir: &TempDir, 203 ip: &str, 204 tap: Option<&str>, 205 num_queues: usize, 206 client_mode: bool, 207 ) -> (std::process::Command, String) { 208 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 209 210 // Start the daemon 211 let net_params = if let Some(tap_str) = tap { 212 format!( 213 "tap={},ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 214 tap_str, ip, vunet_socket_path, num_queues, client_mode 215 ) 216 } else { 217 format!( 218 "ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 219 ip, vunet_socket_path, num_queues, client_mode 220 ) 221 }; 222 223 let mut command = Command::new(clh_command("vhost_user_net")); 224 command.args(&["--net-backend", net_params.as_str()]); 225 226 (command, vunet_socket_path) 227 } 228 229 fn curl_command(api_socket: &str, method: &str, url: &str, http_body: Option<&str>) { 230 let mut curl_args: Vec<&str> = ["--unix-socket", api_socket, "-i", "-X", method, url].to_vec(); 231 232 if let Some(body) = http_body { 233 curl_args.push("-H"); 234 curl_args.push("Accept: application/json"); 235 curl_args.push("-H"); 236 curl_args.push("Content-Type: application/json"); 237 curl_args.push("-d"); 238 curl_args.push(body); 239 } 240 241 let status = Command::new("curl") 242 .args(curl_args) 243 .status() 244 .expect("Failed to launch curl command"); 245 246 assert!(status.success()); 247 } 248 249 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 250 let mut cmd = Command::new(clh_command("ch-remote")); 251 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 252 253 if let Some(arg) = arg { 254 cmd.arg(arg); 255 } 256 let output = cmd.output().unwrap(); 257 if output.status.success() { 258 true 259 } else { 260 eprintln!("Error running ch-remote command: {:?}", &cmd); 261 let stderr = String::from_utf8_lossy(&output.stderr); 262 eprintln!("stderr: {}", stderr); 263 false 264 } 265 } 266 267 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 268 let mut cmd = Command::new(clh_command("ch-remote")); 269 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 270 271 if let Some(arg) = arg { 272 cmd.arg(arg); 273 } 274 275 let output = cmd.output().expect("Failed to launch ch-remote"); 276 277 (output.status.success(), output.stdout) 278 } 279 280 fn resize_command( 281 api_socket: &str, 282 desired_vcpus: Option<u8>, 283 desired_ram: Option<usize>, 284 desired_balloon: Option<usize>, 285 event_file: Option<&str>, 286 ) -> bool { 287 let mut cmd = Command::new(clh_command("ch-remote")); 288 cmd.args(&[&format!("--api-socket={}", api_socket), "resize"]); 289 290 if let Some(desired_vcpus) = desired_vcpus { 291 cmd.arg(format!("--cpus={}", desired_vcpus)); 292 } 293 294 if let Some(desired_ram) = desired_ram { 295 cmd.arg(format!("--memory={}", desired_ram)); 296 } 297 298 if let Some(desired_balloon) = desired_balloon { 299 cmd.arg(format!("--balloon={}", desired_balloon)); 300 } 301 302 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 303 304 if let Some(event_path) = event_file { 305 let latest_events = [ 306 &MetaEvent { 307 event: "resizing".to_string(), 308 device_id: None, 309 }, 310 &MetaEvent { 311 event: "resized".to_string(), 312 device_id: None, 313 }, 314 ]; 315 assert!(check_latest_events_exact(&latest_events, event_path)); 316 } 317 318 ret 319 } 320 321 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 322 let mut cmd = Command::new(clh_command("ch-remote")); 323 cmd.args(&[ 324 &format!("--api-socket={}", api_socket), 325 "resize-zone", 326 &format!("--id={}", id), 327 &format!("--size={}", desired_size), 328 ]); 329 330 cmd.status().expect("Failed to launch ch-remote").success() 331 } 332 333 // setup OVS-DPDK bridge and ports 334 fn setup_ovs_dpdk() { 335 // setup OVS-DPDK 336 assert!(exec_host_command_status("service openvswitch-switch start").success()); 337 assert!(exec_host_command_status("ovs-vsctl init").success()); 338 assert!( 339 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 340 .success() 341 ); 342 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 343 344 // Create OVS-DPDK bridge and ports 345 assert!(exec_host_command_status( 346 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 347 ) 348 .success()); 349 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 350 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 351 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 352 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 353 } 354 fn cleanup_ovs_dpdk() { 355 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 356 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 357 } 358 // Setup two guests and ensure they are connected through ovs-dpdk 359 fn setup_ovs_dpdk_guests( 360 guest1: &Guest, 361 guest2: &Guest, 362 api_socket: &str, 363 release_binary: bool, 364 ) -> (Child, Child) { 365 setup_ovs_dpdk(); 366 367 let clh_path = if !release_binary { 368 clh_command("cloud-hypervisor") 369 } else { 370 cloud_hypervisor_release_path() 371 }; 372 373 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 374 .args(&["--cpus", "boot=2"]) 375 .args(&["--memory", "size=0,shared=on"]) 376 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 377 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 378 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 379 .default_disks() 380 .args(&["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 381 .capture_output() 382 .spawn() 383 .unwrap(); 384 385 #[cfg(target_arch = "x86_64")] 386 let guest_net_iface = "ens5"; 387 #[cfg(target_arch = "aarch64")] 388 let guest_net_iface = "enp0s5"; 389 390 let r = std::panic::catch_unwind(|| { 391 guest1.wait_vm_boot(None).unwrap(); 392 393 guest1 394 .ssh_command(&format!( 395 "sudo ip addr add 172.100.0.1/24 dev {}", 396 guest_net_iface 397 )) 398 .unwrap(); 399 guest1 400 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 401 .unwrap(); 402 403 let guest_ip = guest1.network.guest_ip.clone(); 404 thread::spawn(move || { 405 ssh_command_ip( 406 "nc -l 12345", 407 &guest_ip, 408 DEFAULT_SSH_RETRIES, 409 DEFAULT_SSH_TIMEOUT, 410 ) 411 .unwrap(); 412 }); 413 }); 414 if r.is_err() { 415 cleanup_ovs_dpdk(); 416 417 let _ = child1.kill(); 418 let output = child1.wait_with_output().unwrap(); 419 handle_child_output(r, &output); 420 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 421 } 422 423 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 424 .args(&["--api-socket", api_socket]) 425 .args(&["--cpus", "boot=2"]) 426 .args(&["--memory", "size=0,shared=on"]) 427 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 428 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 429 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 430 .default_disks() 431 .args(&["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 432 .capture_output() 433 .spawn() 434 .unwrap(); 435 436 let r = std::panic::catch_unwind(|| { 437 guest2.wait_vm_boot(None).unwrap(); 438 439 guest2 440 .ssh_command(&format!( 441 "sudo ip addr add 172.100.0.2/24 dev {}", 442 guest_net_iface 443 )) 444 .unwrap(); 445 guest2 446 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 447 .unwrap(); 448 449 // Check the connection works properly between the two VMs 450 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 451 }); 452 if r.is_err() { 453 cleanup_ovs_dpdk(); 454 455 let _ = child1.kill(); 456 let _ = child2.kill(); 457 let output = child2.wait_with_output().unwrap(); 458 handle_child_output(r, &output); 459 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 460 } 461 462 (child1, child2) 463 } 464 465 enum FwType { 466 Ovmf, 467 RustHypervisorFirmware, 468 } 469 470 fn fw_path(_fw_type: FwType) -> String { 471 let mut workload_path = dirs::home_dir().unwrap(); 472 workload_path.push("workloads"); 473 474 let mut fw_path = workload_path; 475 #[cfg(target_arch = "aarch64")] 476 fw_path.push("CLOUDHV_EFI.fd"); 477 #[cfg(target_arch = "x86_64")] 478 { 479 match _fw_type { 480 FwType::Ovmf => fw_path.push(OVMF_NAME), 481 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 482 } 483 } 484 485 fw_path.to_str().unwrap().to_string() 486 } 487 488 struct MetaEvent { 489 event: String, 490 device_id: Option<String>, 491 } 492 493 impl MetaEvent { 494 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 495 let mut matched = false; 496 if v["event"].as_str().unwrap() == self.event { 497 if let Some(device_id) = &self.device_id { 498 if v["properties"]["id"].as_str().unwrap() == device_id { 499 matched = true 500 } 501 } else { 502 matched = true; 503 } 504 } 505 matched 506 } 507 } 508 509 // Parse the event_monitor file based on the format that each event 510 // is followed by a double newline 511 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 512 let content = fs::read(event_file).unwrap(); 513 let mut ret = Vec::new(); 514 for entry in String::from_utf8_lossy(&content) 515 .trim() 516 .split("\n\n") 517 .collect::<Vec<&str>>() 518 { 519 ret.push(serde_json::from_str(entry).unwrap()); 520 } 521 522 ret 523 } 524 525 // Return true if all events from the input 'expected_events' are matched sequentially 526 // with events from the 'event_file' 527 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 528 let json_events = parse_event_file(event_file); 529 let len = expected_events.len(); 530 let mut idx = 0; 531 for e in &json_events { 532 if idx == len { 533 break; 534 } 535 if expected_events[idx].match_with_json_event(e) { 536 idx += 1; 537 } 538 } 539 540 idx == len 541 } 542 543 // Return true if all events from the input 'expected_events' are matched exactly 544 // with events from the 'event_file' 545 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 546 let json_events = parse_event_file(event_file); 547 assert!(expected_events.len() <= json_events.len()); 548 let json_events = &json_events[..expected_events.len()]; 549 550 for (idx, e) in json_events.iter().enumerate() { 551 if !expected_events[idx].match_with_json_event(e) { 552 return false; 553 } 554 } 555 556 true 557 } 558 559 // Return true if events from the input 'expected_events' are matched exactly 560 // with the most recent events from the 'event_file' 561 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 562 let json_events = parse_event_file(event_file); 563 assert!(latest_events.len() <= json_events.len()); 564 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 565 566 for (idx, e) in json_events.iter().enumerate() { 567 if !latest_events[idx].match_with_json_event(e) { 568 return false; 569 } 570 } 571 572 true 573 } 574 575 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 576 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 577 let guest = Guest::new(Box::new(focal)); 578 let total_vcpus = threads_per_core * cores_per_package * packages; 579 let direct_kernel_boot_path = direct_kernel_boot_path(); 580 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 581 let fw_path = fw_path(FwType::RustHypervisorFirmware); 582 if use_fw { 583 kernel_path = fw_path.as_str(); 584 } 585 586 let mut child = GuestCommand::new(&guest) 587 .args(&[ 588 "--cpus", 589 &format!( 590 "boot={},topology={}:{}:1:{}", 591 total_vcpus, threads_per_core, cores_per_package, packages 592 ), 593 ]) 594 .args(&["--memory", "size=512M"]) 595 .args(&["--kernel", kernel_path]) 596 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 597 .default_disks() 598 .default_net() 599 .capture_output() 600 .spawn() 601 .unwrap(); 602 603 let r = std::panic::catch_unwind(|| { 604 guest.wait_vm_boot(None).unwrap(); 605 assert_eq!( 606 guest.get_cpu_count().unwrap_or_default(), 607 u32::from(total_vcpus) 608 ); 609 assert_eq!( 610 guest 611 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 612 .unwrap() 613 .trim() 614 .parse::<u8>() 615 .unwrap_or(0), 616 threads_per_core 617 ); 618 619 assert_eq!( 620 guest 621 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 622 .unwrap() 623 .trim() 624 .parse::<u8>() 625 .unwrap_or(0), 626 cores_per_package 627 ); 628 629 assert_eq!( 630 guest 631 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 632 .unwrap() 633 .trim() 634 .parse::<u8>() 635 .unwrap_or(0), 636 packages 637 ); 638 }); 639 640 let _ = child.kill(); 641 let output = child.wait_with_output().unwrap(); 642 643 handle_child_output(r, &output); 644 } 645 646 #[allow(unused_variables)] 647 fn _test_guest_numa_nodes(acpi: bool) { 648 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 649 let guest = Guest::new(Box::new(focal)); 650 let api_socket = temp_api_path(&guest.tmp_dir); 651 #[cfg(target_arch = "x86_64")] 652 let kernel_path = direct_kernel_boot_path(); 653 #[cfg(target_arch = "aarch64")] 654 let kernel_path = if acpi { 655 edk2_path() 656 } else { 657 direct_kernel_boot_path() 658 }; 659 660 let mut child = GuestCommand::new(&guest) 661 .args(&["--cpus", "boot=6,max=12"]) 662 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 663 .args(&[ 664 "--memory-zone", 665 "id=mem0,size=1G,hotplug_size=3G", 666 "id=mem1,size=2G,hotplug_size=3G", 667 "id=mem2,size=3G,hotplug_size=3G", 668 ]) 669 .args(&[ 670 "--numa", 671 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 672 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 673 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 674 ]) 675 .args(&["--kernel", kernel_path.to_str().unwrap()]) 676 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 677 .args(&["--api-socket", &api_socket]) 678 .capture_output() 679 .default_disks() 680 .default_net() 681 .spawn() 682 .unwrap(); 683 684 let r = std::panic::catch_unwind(|| { 685 guest.wait_vm_boot(None).unwrap(); 686 687 guest.check_numa_common( 688 Some(&[960_000, 1_920_000, 2_880_000]), 689 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 690 Some(&["10 15 20", "20 10 25", "25 30 10"]), 691 ); 692 693 // AArch64 currently does not support hotplug, and therefore we only 694 // test hotplug-related function on x86_64 here. 695 #[cfg(target_arch = "x86_64")] 696 { 697 guest.enable_memory_hotplug(); 698 699 // Resize every memory zone and check each associated NUMA node 700 // has been assigned the right amount of memory. 701 resize_zone_command(&api_socket, "mem0", "4G"); 702 resize_zone_command(&api_socket, "mem1", "4G"); 703 resize_zone_command(&api_socket, "mem2", "4G"); 704 // Resize to the maximum amount of CPUs and check each NUMA 705 // node has been assigned the right CPUs set. 706 resize_command(&api_socket, Some(12), None, None, None); 707 thread::sleep(std::time::Duration::new(5, 0)); 708 709 guest.check_numa_common( 710 Some(&[3_840_000, 3_840_000, 3_840_000]), 711 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 712 None, 713 ); 714 } 715 }); 716 717 let _ = child.kill(); 718 let output = child.wait_with_output().unwrap(); 719 720 handle_child_output(r, &output); 721 } 722 723 #[allow(unused_variables)] 724 fn _test_power_button(acpi: bool) { 725 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 726 let guest = Guest::new(Box::new(focal)); 727 let mut cmd = GuestCommand::new(&guest); 728 let api_socket = temp_api_path(&guest.tmp_dir); 729 730 #[cfg(target_arch = "x86_64")] 731 let kernel_path = direct_kernel_boot_path(); 732 #[cfg(target_arch = "aarch64")] 733 let kernel_path = if acpi { 734 edk2_path() 735 } else { 736 direct_kernel_boot_path() 737 }; 738 739 cmd.args(&["--cpus", "boot=1"]) 740 .args(&["--memory", "size=512M"]) 741 .args(&["--kernel", kernel_path.to_str().unwrap()]) 742 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 743 .capture_output() 744 .default_disks() 745 .default_net() 746 .args(&["--api-socket", &api_socket]); 747 748 let child = cmd.spawn().unwrap(); 749 750 let r = std::panic::catch_unwind(|| { 751 guest.wait_vm_boot(None).unwrap(); 752 assert!(remote_command(&api_socket, "power-button", None)); 753 }); 754 755 let output = child.wait_with_output().unwrap(); 756 assert!(output.status.success()); 757 handle_child_output(r, &output); 758 } 759 760 type PrepareNetDaemon = 761 dyn Fn(&TempDir, &str, Option<&str>, usize, bool) -> (std::process::Command, String); 762 763 fn test_vhost_user_net( 764 tap: Option<&str>, 765 num_queues: usize, 766 prepare_daemon: &PrepareNetDaemon, 767 generate_host_mac: bool, 768 client_mode_daemon: bool, 769 ) { 770 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 771 let guest = Guest::new(Box::new(focal)); 772 let api_socket = temp_api_path(&guest.tmp_dir); 773 774 let kernel_path = direct_kernel_boot_path(); 775 776 let host_mac = if generate_host_mac { 777 Some(MacAddr::local_random()) 778 } else { 779 None 780 }; 781 782 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 783 &guest.tmp_dir, 784 &guest.network.host_ip, 785 tap, 786 num_queues, 787 client_mode_daemon, 788 ); 789 790 let net_params = format!( 791 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={}", 792 guest.network.guest_mac, 793 vunet_socket_path, 794 num_queues, 795 if let Some(host_mac) = host_mac { 796 format!(",host_mac={}", host_mac) 797 } else { 798 "".to_owned() 799 }, 800 if client_mode_daemon { 801 "server" 802 } else { 803 "client" 804 }, 805 ); 806 807 let mut ch_command = GuestCommand::new(&guest); 808 ch_command 809 .args(&["--cpus", format!("boot={}", num_queues / 2).as_str()]) 810 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 811 .args(&["--kernel", kernel_path.to_str().unwrap()]) 812 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 813 .default_disks() 814 .args(&["--net", net_params.as_str()]) 815 .args(&["--api-socket", &api_socket]) 816 .capture_output(); 817 818 let mut daemon_child: std::process::Child; 819 let mut child: std::process::Child; 820 821 if client_mode_daemon { 822 child = ch_command.spawn().unwrap(); 823 // Make sure the VMM is waiting for the backend to connect 824 thread::sleep(std::time::Duration::new(10, 0)); 825 daemon_child = daemon_command.spawn().unwrap(); 826 } else { 827 daemon_child = daemon_command.spawn().unwrap(); 828 // Make sure the backend is waiting for the VMM to connect 829 thread::sleep(std::time::Duration::new(10, 0)); 830 child = ch_command.spawn().unwrap(); 831 } 832 833 let r = std::panic::catch_unwind(|| { 834 guest.wait_vm_boot(None).unwrap(); 835 836 if let Some(tap_name) = tap { 837 let tap_count = exec_host_command_output(&format!("ip link | grep -c {}", tap_name)); 838 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 839 } 840 841 if let Some(host_mac) = tap { 842 let mac_count = exec_host_command_output(&format!("ip link | grep -c {}", host_mac)); 843 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 844 } 845 846 // 1 network interface + default localhost ==> 2 interfaces 847 // It's important to note that this test is fully exercising the 848 // vhost-user-net implementation and the associated backend since 849 // it does not define any --net network interface. That means all 850 // the ssh communication in that test happens through the network 851 // interface backed by vhost-user-net. 852 assert_eq!( 853 guest 854 .ssh_command("ip -o link | wc -l") 855 .unwrap() 856 .trim() 857 .parse::<u32>() 858 .unwrap_or_default(), 859 2 860 ); 861 862 // The following pci devices will appear on guest with PCI-MSI 863 // interrupt vectors assigned. 864 // 1 virtio-console with 3 vectors: config, Rx, Tx 865 // 1 virtio-blk with 2 vectors: config, Request 866 // 1 virtio-blk with 2 vectors: config, Request 867 // 1 virtio-rng with 2 vectors: config, Request 868 // Since virtio-net has 2 queue pairs, its vectors is as follows: 869 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 870 // Based on the above, the total vectors should 14. 871 #[cfg(target_arch = "x86_64")] 872 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 873 #[cfg(target_arch = "aarch64")] 874 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 875 assert_eq!( 876 guest 877 .ssh_command(grep_cmd) 878 .unwrap() 879 .trim() 880 .parse::<u32>() 881 .unwrap_or_default(), 882 10 + (num_queues as u32) 883 ); 884 885 // ACPI feature is needed. 886 #[cfg(target_arch = "x86_64")] 887 { 888 guest.enable_memory_hotplug(); 889 890 // Add RAM to the VM 891 let desired_ram = 1024 << 20; 892 resize_command(&api_socket, None, Some(desired_ram), None, None); 893 894 thread::sleep(std::time::Duration::new(10, 0)); 895 896 // Here by simply checking the size (through ssh), we validate 897 // the connection is still working, which means vhost-user-net 898 // keeps working after the resize. 899 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 900 } 901 }); 902 903 let _ = child.kill(); 904 let output = child.wait_with_output().unwrap(); 905 906 thread::sleep(std::time::Duration::new(5, 0)); 907 let _ = daemon_child.kill(); 908 let _ = daemon_child.wait(); 909 910 handle_child_output(r, &output); 911 } 912 913 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 914 915 fn test_vhost_user_blk( 916 num_queues: usize, 917 readonly: bool, 918 direct: bool, 919 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 920 ) { 921 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 922 let guest = Guest::new(Box::new(focal)); 923 let api_socket = temp_api_path(&guest.tmp_dir); 924 925 let kernel_path = direct_kernel_boot_path(); 926 927 let (blk_params, daemon_child) = { 928 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 929 // Start the daemon 930 let (daemon_child, vubd_socket_path) = 931 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 932 933 ( 934 format!( 935 "vhost_user=true,socket={},num_queues={},queue_size=128", 936 vubd_socket_path, num_queues, 937 ), 938 Some(daemon_child), 939 ) 940 }; 941 942 let mut child = GuestCommand::new(&guest) 943 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 944 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 945 .args(&["--kernel", kernel_path.to_str().unwrap()]) 946 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 947 .args(&[ 948 "--disk", 949 format!( 950 "path={}", 951 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 952 ) 953 .as_str(), 954 format!( 955 "path={}", 956 guest.disk_config.disk(DiskType::CloudInit).unwrap() 957 ) 958 .as_str(), 959 blk_params.as_str(), 960 ]) 961 .default_net() 962 .args(&["--api-socket", &api_socket]) 963 .capture_output() 964 .spawn() 965 .unwrap(); 966 967 let r = std::panic::catch_unwind(|| { 968 guest.wait_vm_boot(None).unwrap(); 969 970 // Check both if /dev/vdc exists and if the block size is 16M. 971 assert_eq!( 972 guest 973 .ssh_command("lsblk | grep vdc | grep -c 16M") 974 .unwrap() 975 .trim() 976 .parse::<u32>() 977 .unwrap_or_default(), 978 1 979 ); 980 981 // Check if this block is RO or RW. 982 assert_eq!( 983 guest 984 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 985 .unwrap() 986 .trim() 987 .parse::<u32>() 988 .unwrap_or_default(), 989 readonly as u32 990 ); 991 992 // Check if the number of queues in /sys/block/vdc/mq matches the 993 // expected num_queues. 994 assert_eq!( 995 guest 996 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 997 .unwrap() 998 .trim() 999 .parse::<u32>() 1000 .unwrap_or_default(), 1001 num_queues as u32 1002 ); 1003 1004 // Mount the device 1005 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1006 guest.ssh_command("mkdir mount_image").unwrap(); 1007 guest 1008 .ssh_command( 1009 format!( 1010 "sudo mount -o {} -t ext4 /dev/vdc mount_image/", 1011 mount_ro_rw_flag 1012 ) 1013 .as_str(), 1014 ) 1015 .unwrap(); 1016 1017 // Check the content of the block device. The file "foo" should 1018 // contain "bar". 1019 assert_eq!( 1020 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1021 "bar" 1022 ); 1023 1024 // ACPI feature is needed. 1025 #[cfg(target_arch = "x86_64")] 1026 { 1027 guest.enable_memory_hotplug(); 1028 1029 // Add RAM to the VM 1030 let desired_ram = 1024 << 20; 1031 resize_command(&api_socket, None, Some(desired_ram), None, None); 1032 1033 thread::sleep(std::time::Duration::new(10, 0)); 1034 1035 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1036 1037 // Check again the content of the block device after the resize 1038 // has been performed. 1039 assert_eq!( 1040 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1041 "bar" 1042 ); 1043 } 1044 1045 // Unmount the device 1046 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1047 guest.ssh_command("rm -r mount_image").unwrap(); 1048 }); 1049 1050 let _ = child.kill(); 1051 let output = child.wait_with_output().unwrap(); 1052 1053 if let Some(mut daemon_child) = daemon_child { 1054 thread::sleep(std::time::Duration::new(5, 0)); 1055 let _ = daemon_child.kill(); 1056 let _ = daemon_child.wait(); 1057 } 1058 1059 handle_child_output(r, &output); 1060 } 1061 1062 fn test_boot_from_vhost_user_blk( 1063 num_queues: usize, 1064 readonly: bool, 1065 direct: bool, 1066 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1067 ) { 1068 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1069 let guest = Guest::new(Box::new(focal)); 1070 1071 let kernel_path = direct_kernel_boot_path(); 1072 1073 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1074 1075 let (blk_boot_params, daemon_child) = { 1076 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1077 // Start the daemon 1078 let (daemon_child, vubd_socket_path) = prepare_daemon( 1079 &guest.tmp_dir, 1080 disk_path.as_str(), 1081 num_queues, 1082 readonly, 1083 direct, 1084 ); 1085 1086 ( 1087 format!( 1088 "vhost_user=true,socket={},num_queues={},queue_size=128", 1089 vubd_socket_path, num_queues, 1090 ), 1091 Some(daemon_child), 1092 ) 1093 }; 1094 1095 let mut child = GuestCommand::new(&guest) 1096 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 1097 .args(&["--memory", "size=512M,shared=on"]) 1098 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1099 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1100 .args(&[ 1101 "--disk", 1102 blk_boot_params.as_str(), 1103 format!( 1104 "path={}", 1105 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1106 ) 1107 .as_str(), 1108 ]) 1109 .default_net() 1110 .capture_output() 1111 .spawn() 1112 .unwrap(); 1113 1114 let r = std::panic::catch_unwind(|| { 1115 guest.wait_vm_boot(None).unwrap(); 1116 1117 // Just check the VM booted correctly. 1118 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1119 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1120 }); 1121 let _ = child.kill(); 1122 let output = child.wait_with_output().unwrap(); 1123 1124 if let Some(mut daemon_child) = daemon_child { 1125 thread::sleep(std::time::Duration::new(5, 0)); 1126 let _ = daemon_child.kill(); 1127 let _ = daemon_child.wait(); 1128 } 1129 1130 handle_child_output(r, &output); 1131 } 1132 1133 fn _test_virtio_fs( 1134 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1135 hotplug: bool, 1136 pci_segment: Option<u16>, 1137 ) { 1138 #[cfg(target_arch = "aarch64")] 1139 let focal_image = if hotplug { 1140 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1141 } else { 1142 FOCAL_IMAGE_NAME.to_string() 1143 }; 1144 #[cfg(target_arch = "x86_64")] 1145 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1146 let focal = UbuntuDiskConfig::new(focal_image); 1147 let guest = Guest::new(Box::new(focal)); 1148 let api_socket = temp_api_path(&guest.tmp_dir); 1149 1150 let mut workload_path = dirs::home_dir().unwrap(); 1151 workload_path.push("workloads"); 1152 1153 let mut shared_dir = workload_path; 1154 shared_dir.push("shared_dir"); 1155 1156 #[cfg(target_arch = "x86_64")] 1157 let kernel_path = direct_kernel_boot_path(); 1158 #[cfg(target_arch = "aarch64")] 1159 let kernel_path = if hotplug { 1160 edk2_path() 1161 } else { 1162 direct_kernel_boot_path() 1163 }; 1164 1165 let (mut daemon_child, virtiofsd_socket_path) = 1166 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1167 1168 let mut guest_command = GuestCommand::new(&guest); 1169 guest_command 1170 .args(&["--cpus", "boot=1"]) 1171 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1172 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1173 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1174 .default_disks() 1175 .default_net() 1176 .args(&["--api-socket", &api_socket]); 1177 if pci_segment.is_some() { 1178 guest_command.args(&["--platform", "num_pci_segments=16"]); 1179 } 1180 1181 let fs_params = format!( 1182 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1183 virtiofsd_socket_path, 1184 if let Some(pci_segment) = pci_segment { 1185 format!(",pci_segment={}", pci_segment) 1186 } else { 1187 "".to_owned() 1188 } 1189 ); 1190 1191 if !hotplug { 1192 guest_command.args(&["--fs", fs_params.as_str()]); 1193 } 1194 1195 let mut child = guest_command.capture_output().spawn().unwrap(); 1196 1197 let r = std::panic::catch_unwind(|| { 1198 guest.wait_vm_boot(None).unwrap(); 1199 1200 if hotplug { 1201 // Add fs to the VM 1202 let (cmd_success, cmd_output) = 1203 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1204 assert!(cmd_success); 1205 1206 if let Some(pci_segment) = pci_segment { 1207 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1208 "{{\"id\":\"myfs0\",\"bdf\":\"{:04x}:00:01.0\"}}", 1209 pci_segment 1210 ))); 1211 } else { 1212 assert!(String::from_utf8_lossy(&cmd_output) 1213 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1214 } 1215 1216 thread::sleep(std::time::Duration::new(10, 0)); 1217 } 1218 1219 // Mount shared directory through virtio_fs filesystem 1220 guest 1221 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1222 .unwrap(); 1223 1224 // Check file1 exists and its content is "foo" 1225 assert_eq!( 1226 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1227 "foo" 1228 ); 1229 // Check file2 does not exist 1230 guest 1231 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1232 .unwrap(); 1233 1234 // Check file3 exists and its content is "bar" 1235 assert_eq!( 1236 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1237 "bar" 1238 ); 1239 1240 // ACPI feature is needed. 1241 #[cfg(target_arch = "x86_64")] 1242 { 1243 guest.enable_memory_hotplug(); 1244 1245 // Add RAM to the VM 1246 let desired_ram = 1024 << 20; 1247 resize_command(&api_socket, None, Some(desired_ram), None, None); 1248 1249 thread::sleep(std::time::Duration::new(30, 0)); 1250 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1251 1252 // After the resize, check again that file1 exists and its 1253 // content is "foo". 1254 assert_eq!( 1255 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1256 "foo" 1257 ); 1258 } 1259 1260 if hotplug { 1261 // Remove from VM 1262 guest.ssh_command("sudo umount mount_dir").unwrap(); 1263 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1264 } 1265 }); 1266 1267 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1268 thread::sleep(std::time::Duration::new(10, 0)); 1269 let (daemon_child, virtiofsd_socket_path) = 1270 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1271 1272 let r = std::panic::catch_unwind(|| { 1273 thread::sleep(std::time::Duration::new(10, 0)); 1274 let fs_params = format!( 1275 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1276 virtiofsd_socket_path, 1277 if let Some(pci_segment) = pci_segment { 1278 format!(",pci_segment={}", pci_segment) 1279 } else { 1280 "".to_owned() 1281 } 1282 ); 1283 1284 // Add back and check it works 1285 let (cmd_success, cmd_output) = 1286 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1287 assert!(cmd_success); 1288 if let Some(pci_segment) = pci_segment { 1289 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1290 "{{\"id\":\"myfs0\",\"bdf\":\"{:04x}:00:01.0\"}}", 1291 pci_segment 1292 ))); 1293 } else { 1294 assert!(String::from_utf8_lossy(&cmd_output) 1295 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1296 } 1297 1298 thread::sleep(std::time::Duration::new(10, 0)); 1299 // Mount shared directory through virtio_fs filesystem 1300 guest 1301 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1302 .unwrap(); 1303 1304 // Check file1 exists and its content is "foo" 1305 assert_eq!( 1306 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1307 "foo" 1308 ); 1309 }); 1310 1311 (r, Some(daemon_child)) 1312 } else { 1313 (r, None) 1314 }; 1315 1316 let _ = child.kill(); 1317 let output = child.wait_with_output().unwrap(); 1318 1319 let _ = daemon_child.kill(); 1320 let _ = daemon_child.wait(); 1321 1322 if let Some(mut daemon_child) = hotplug_daemon_child { 1323 let _ = daemon_child.kill(); 1324 let _ = daemon_child.wait(); 1325 } 1326 1327 handle_child_output(r, &output); 1328 } 1329 1330 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1331 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1332 let guest = Guest::new(Box::new(focal)); 1333 1334 let kernel_path = direct_kernel_boot_path(); 1335 1336 let pmem_temp_file = TempFile::new().unwrap(); 1337 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1338 1339 std::process::Command::new("mkfs.ext4") 1340 .arg(pmem_temp_file.as_path()) 1341 .output() 1342 .expect("Expect creating disk image to succeed"); 1343 1344 let mut child = GuestCommand::new(&guest) 1345 .args(&["--cpus", "boot=1"]) 1346 .args(&["--memory", "size=512M"]) 1347 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1348 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1349 .default_disks() 1350 .default_net() 1351 .args(&[ 1352 "--pmem", 1353 format!( 1354 "file={}{}{}", 1355 pmem_temp_file.as_path().to_str().unwrap(), 1356 if specify_size { ",size=128M" } else { "" }, 1357 if discard_writes { 1358 ",discard_writes=on" 1359 } else { 1360 "" 1361 } 1362 ) 1363 .as_str(), 1364 ]) 1365 .capture_output() 1366 .spawn() 1367 .unwrap(); 1368 1369 let r = std::panic::catch_unwind(|| { 1370 guest.wait_vm_boot(None).unwrap(); 1371 1372 // Check for the presence of /dev/pmem0 1373 assert_eq!( 1374 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1375 "/dev/pmem0" 1376 ); 1377 1378 // Check changes persist after reboot 1379 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1380 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1381 guest 1382 .ssh_command("echo test123 | sudo tee /mnt/test") 1383 .unwrap(); 1384 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1385 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1386 1387 guest.reboot_linux(0, None); 1388 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1389 assert_eq!( 1390 guest 1391 .ssh_command("sudo cat /mnt/test || true") 1392 .unwrap() 1393 .trim(), 1394 if discard_writes { "" } else { "test123" } 1395 ); 1396 }); 1397 1398 let _ = child.kill(); 1399 let output = child.wait_with_output().unwrap(); 1400 1401 handle_child_output(r, &output); 1402 } 1403 1404 fn get_fd_count(pid: u32) -> usize { 1405 fs::read_dir(format!("/proc/{}/fd", pid)).unwrap().count() 1406 } 1407 1408 fn _test_virtio_vsock(hotplug: bool) { 1409 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1410 let guest = Guest::new(Box::new(focal)); 1411 1412 #[cfg(target_arch = "x86_64")] 1413 let kernel_path = direct_kernel_boot_path(); 1414 #[cfg(target_arch = "aarch64")] 1415 let kernel_path = if hotplug { 1416 edk2_path() 1417 } else { 1418 direct_kernel_boot_path() 1419 }; 1420 1421 let socket = temp_vsock_path(&guest.tmp_dir); 1422 let api_socket = temp_api_path(&guest.tmp_dir); 1423 1424 let mut cmd = GuestCommand::new(&guest); 1425 cmd.args(&["--api-socket", &api_socket]); 1426 cmd.args(&["--cpus", "boot=1"]); 1427 cmd.args(&["--memory", "size=512M"]); 1428 cmd.args(&["--kernel", kernel_path.to_str().unwrap()]); 1429 cmd.args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1430 cmd.default_disks(); 1431 cmd.default_net(); 1432 1433 if !hotplug { 1434 cmd.args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]); 1435 } 1436 1437 let mut child = cmd.capture_output().spawn().unwrap(); 1438 1439 let r = std::panic::catch_unwind(|| { 1440 guest.wait_vm_boot(None).unwrap(); 1441 1442 if hotplug { 1443 let (cmd_success, cmd_output) = remote_command_w_output( 1444 &api_socket, 1445 "add-vsock", 1446 Some(format!("cid=3,socket={},id=test0", socket).as_str()), 1447 ); 1448 assert!(cmd_success); 1449 assert!(String::from_utf8_lossy(&cmd_output) 1450 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1451 thread::sleep(std::time::Duration::new(10, 0)); 1452 // Check adding a second one fails 1453 assert!(!remote_command( 1454 &api_socket, 1455 "add-vsock", 1456 Some("cid=1234,socket=/tmp/fail") 1457 )); 1458 } 1459 1460 // Validate vsock works as expected. 1461 guest.check_vsock(socket.as_str()); 1462 guest.reboot_linux(0, None); 1463 // Validate vsock still works after a reboot. 1464 guest.check_vsock(socket.as_str()); 1465 1466 if hotplug { 1467 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1468 } 1469 }); 1470 1471 let _ = child.kill(); 1472 let output = child.wait_with_output().unwrap(); 1473 1474 handle_child_output(r, &output); 1475 } 1476 1477 fn get_ksm_pages_shared() -> u32 { 1478 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1479 .unwrap() 1480 .trim() 1481 .parse::<u32>() 1482 .unwrap() 1483 } 1484 1485 fn test_memory_mergeable(mergeable: bool) { 1486 let memory_param = if mergeable { 1487 "mergeable=on" 1488 } else { 1489 "mergeable=off" 1490 }; 1491 1492 // We are assuming the rest of the system in our CI is not using mergeable memeory 1493 let ksm_ps_init = get_ksm_pages_shared(); 1494 assert!(ksm_ps_init == 0); 1495 1496 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1497 let guest1 = Guest::new(Box::new(focal1)); 1498 let mut child1 = GuestCommand::new(&guest1) 1499 .args(&["--cpus", "boot=1"]) 1500 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1501 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1502 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1503 .default_disks() 1504 .args(&["--net", guest1.default_net_string().as_str()]) 1505 .args(&["--serial", "tty", "--console", "off"]) 1506 .capture_output() 1507 .spawn() 1508 .unwrap(); 1509 1510 let r = std::panic::catch_unwind(|| { 1511 guest1.wait_vm_boot(None).unwrap(); 1512 }); 1513 if r.is_err() { 1514 let _ = child1.kill(); 1515 let output = child1.wait_with_output().unwrap(); 1516 handle_child_output(r, &output); 1517 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1518 } 1519 1520 let ksm_ps_guest1 = get_ksm_pages_shared(); 1521 1522 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1523 let guest2 = Guest::new(Box::new(focal2)); 1524 let mut child2 = GuestCommand::new(&guest2) 1525 .args(&["--cpus", "boot=1"]) 1526 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1527 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1528 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1529 .default_disks() 1530 .args(&["--net", guest2.default_net_string().as_str()]) 1531 .args(&["--serial", "tty", "--console", "off"]) 1532 .capture_output() 1533 .spawn() 1534 .unwrap(); 1535 1536 let r = std::panic::catch_unwind(|| { 1537 guest2.wait_vm_boot(None).unwrap(); 1538 let ksm_ps_guest2 = get_ksm_pages_shared(); 1539 1540 if mergeable { 1541 println!( 1542 "ksm pages_shared after vm1 booted '{}', ksm pages_shared after vm2 booted '{}'", 1543 ksm_ps_guest1, ksm_ps_guest2 1544 ); 1545 // We are expecting the number of shared pages to increase as the number of VM increases 1546 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1547 } else { 1548 assert!(ksm_ps_guest1 == 0); 1549 assert!(ksm_ps_guest2 == 0); 1550 } 1551 }); 1552 1553 let _ = child1.kill(); 1554 let _ = child2.kill(); 1555 1556 let output = child1.wait_with_output().unwrap(); 1557 child2.wait().unwrap(); 1558 1559 handle_child_output(r, &output); 1560 } 1561 1562 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 1563 let smaps = fs::File::open(format!("/proc/{}/smaps", pid)).unwrap(); 1564 let reader = io::BufReader::new(smaps); 1565 1566 let mut skip_map: bool = false; 1567 let mut region_name: String = "".to_string(); 1568 let mut region_maps = HashMap::new(); 1569 for line in reader.lines() { 1570 let l = line.unwrap(); 1571 1572 if l.contains('-') { 1573 let values: Vec<&str> = l.split_whitespace().collect(); 1574 region_name = values.last().unwrap().trim().to_string(); 1575 if region_name == "0" { 1576 region_name = "anonymous".to_string() 1577 } 1578 } 1579 1580 // Each section begins with something that looks like: 1581 // Size: 2184 kB 1582 if l.starts_with("Size:") { 1583 let values: Vec<&str> = l.split_whitespace().collect(); 1584 let map_size = values[1].parse::<u32>().unwrap(); 1585 // We skip the assigned guest RAM map, its RSS is only 1586 // dependent on the guest actual memory usage. 1587 // Everything else can be added to the VMM overhead. 1588 skip_map = map_size >= guest_memory_size; 1589 continue; 1590 } 1591 1592 // If this is a map we're taking into account, then we only 1593 // count the RSS. The sum of all counted RSS is the VMM overhead. 1594 if !skip_map && l.starts_with("Rss:") { 1595 let values: Vec<&str> = l.split_whitespace().collect(); 1596 let value = values[1].trim().parse::<u32>().unwrap(); 1597 *region_maps.entry(region_name.clone()).or_insert(0) += value; 1598 } 1599 } 1600 1601 region_maps 1602 } 1603 1604 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 1605 let mut total = 0; 1606 1607 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 1608 eprintln!("{}: {}", region_name, value); 1609 total += value; 1610 } 1611 1612 total 1613 } 1614 1615 fn process_rss_kib(pid: u32) -> usize { 1616 let command = format!("ps -q {} -o rss=", pid); 1617 let rss = exec_host_command_output(&command); 1618 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 1619 } 1620 1621 // 10MB is our maximum accepted overhead. 1622 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 1623 1624 #[derive(PartialEq, Eq, PartialOrd)] 1625 struct Counters { 1626 rx_bytes: u64, 1627 rx_frames: u64, 1628 tx_bytes: u64, 1629 tx_frames: u64, 1630 read_bytes: u64, 1631 write_bytes: u64, 1632 read_ops: u64, 1633 write_ops: u64, 1634 } 1635 1636 fn get_counters(api_socket: &str) -> Counters { 1637 // Get counters 1638 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 1639 assert!(cmd_success); 1640 1641 let counters: HashMap<&str, HashMap<&str, u64>> = 1642 serde_json::from_slice(&cmd_output).unwrap_or_default(); 1643 1644 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 1645 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 1646 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 1647 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 1648 1649 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 1650 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 1651 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 1652 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 1653 1654 Counters { 1655 rx_bytes, 1656 rx_frames, 1657 tx_bytes, 1658 tx_frames, 1659 read_bytes, 1660 write_bytes, 1661 read_ops, 1662 write_ops, 1663 } 1664 } 1665 1666 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 1667 let (tx, rx) = mpsc::channel::<String>(); 1668 thread::spawn(move || loop { 1669 thread::sleep(std::time::Duration::new(1, 0)); 1670 let mut buf = [0; 512]; 1671 match pty.read(&mut buf) { 1672 Ok(_) => { 1673 let output = std::str::from_utf8(&buf).unwrap().to_string(); 1674 match tx.send(output) { 1675 Ok(_) => (), 1676 Err(_) => break, 1677 } 1678 } 1679 Err(_) => break, 1680 } 1681 }); 1682 rx 1683 } 1684 1685 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 1686 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1687 assert!(cmd_success); 1688 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1689 assert_eq!("Pty", info["config"][pty_type]["mode"]); 1690 PathBuf::from( 1691 info["config"][pty_type]["file"] 1692 .as_str() 1693 .expect("Missing pty path"), 1694 ) 1695 } 1696 1697 // VFIO test network setup. 1698 // We reserve a different IP class for it: 172.18.0.0/24. 1699 #[cfg(target_arch = "x86_64")] 1700 fn setup_vfio_network_interfaces() { 1701 // 'vfio-br0' 1702 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 1703 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 1704 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 1705 // 'vfio-tap0' 1706 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 1707 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 1708 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 1709 // 'vfio-tap1' 1710 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 1711 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 1712 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 1713 // 'vfio-tap2' 1714 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 1715 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 1716 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 1717 // 'vfio-tap3' 1718 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 1719 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 1720 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 1721 } 1722 1723 // Tear VFIO test network down 1724 #[cfg(target_arch = "x86_64")] 1725 fn cleanup_vfio_network_interfaces() { 1726 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 1727 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 1728 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 1729 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 1730 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 1731 } 1732 1733 fn balloon_size(api_socket: &str) -> u64 { 1734 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1735 assert!(cmd_success); 1736 1737 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1738 let total_mem = &info["config"]["memory"]["size"] 1739 .to_string() 1740 .parse::<u64>() 1741 .unwrap(); 1742 let actual_mem = &info["memory_actual_size"] 1743 .to_string() 1744 .parse::<u64>() 1745 .unwrap(); 1746 total_mem - actual_mem 1747 } 1748 1749 // This test validates that it can find the virtio-iommu device at first. 1750 // It also verifies that both disks and the network card are attached to 1751 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 1752 // The last interesting part of this test is that it exercises the network 1753 // interface attached to the virtual IOMMU since this is the one used to 1754 // send all commands through SSH. 1755 fn _test_virtio_iommu(acpi: bool) { 1756 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 1757 // Focal image is still old. 1758 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 1759 // the kernel binary has been updated. 1760 #[cfg(target_arch = "aarch64")] 1761 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 1762 #[cfg(target_arch = "x86_64")] 1763 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1764 let focal = UbuntuDiskConfig::new(focal_image); 1765 let guest = Guest::new(Box::new(focal)); 1766 1767 #[cfg(target_arch = "x86_64")] 1768 let kernel_path = direct_kernel_boot_path(); 1769 #[cfg(target_arch = "aarch64")] 1770 let kernel_path = if acpi { 1771 edk2_path() 1772 } else { 1773 direct_kernel_boot_path() 1774 }; 1775 1776 let mut child = GuestCommand::new(&guest) 1777 .args(&["--cpus", "boot=1"]) 1778 .args(&["--memory", "size=512M"]) 1779 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1780 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1781 .args(&[ 1782 "--disk", 1783 format!( 1784 "path={},iommu=on", 1785 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1786 ) 1787 .as_str(), 1788 format!( 1789 "path={},iommu=on", 1790 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1791 ) 1792 .as_str(), 1793 ]) 1794 .args(&["--net", guest.default_net_string_w_iommu().as_str()]) 1795 .capture_output() 1796 .spawn() 1797 .unwrap(); 1798 1799 let r = std::panic::catch_unwind(|| { 1800 guest.wait_vm_boot(None).unwrap(); 1801 1802 // Verify the virtio-iommu device is present. 1803 assert!(guest 1804 .does_device_vendor_pair_match("0x1057", "0x1af4") 1805 .unwrap_or_default()); 1806 1807 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 1808 // different with ACPI. 1809 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 1810 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 1811 // and appear under folder '/sys/kernel/iommu_groups/'. 1812 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 1813 // which is the console. The first disk "0000:00:02.0" is in group '1'. 1814 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 1815 // contains "0000:00:02.0" which is the first disk. 1816 // 1817 // Verify the iommu group of the first disk. 1818 let iommu_group = if acpi { 0 } else { 1 }; 1819 assert_eq!( 1820 guest 1821 .ssh_command( 1822 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1823 ) 1824 .unwrap() 1825 .trim(), 1826 "0000:00:02.0" 1827 ); 1828 1829 // Verify the iommu group of the second disk. 1830 let iommu_group = if acpi { 1 } else { 2 }; 1831 assert_eq!( 1832 guest 1833 .ssh_command( 1834 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1835 ) 1836 .unwrap() 1837 .trim(), 1838 "0000:00:03.0" 1839 ); 1840 1841 // Verify the iommu group of the network card. 1842 let iommu_group = if acpi { 2 } else { 3 }; 1843 assert_eq!( 1844 guest 1845 .ssh_command( 1846 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1847 ) 1848 .unwrap() 1849 .trim(), 1850 "0000:00:04.0" 1851 ); 1852 }); 1853 1854 let _ = child.kill(); 1855 let output = child.wait_with_output().unwrap(); 1856 1857 handle_child_output(r, &output); 1858 } 1859 1860 fn enable_guest_watchdog(guest: &Guest, current_reboot_count: &mut u32) { 1861 // Check for PCI device 1862 assert!(guest 1863 .does_device_vendor_pair_match("0x1063", "0x1af4") 1864 .unwrap_or_default()); 1865 1866 // Enable systemd watchdog 1867 guest 1868 .ssh_command("echo RuntimeWatchdogSec=15s | sudo tee -a /etc/systemd/system.conf") 1869 .unwrap(); 1870 1871 guest.ssh_command("sudo reboot").unwrap(); 1872 1873 guest.wait_vm_boot(None).unwrap(); 1874 1875 // Check that systemd has activated the watchdog 1876 assert_eq!( 1877 guest 1878 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 1879 .unwrap() 1880 .trim() 1881 .parse::<u32>() 1882 .unwrap_or_default(), 1883 2 1884 ); 1885 1886 // Ensure that the current boot journal is written so reboot counts are valid 1887 guest.ssh_command("sudo journalctl --sync").unwrap(); 1888 1889 *current_reboot_count += 1; 1890 1891 let boot_count = guest 1892 .ssh_command("sudo journalctl --list-boots | wc -l") 1893 .unwrap() 1894 .trim() 1895 .parse::<u32>() 1896 .unwrap_or_default(); 1897 assert_eq!(boot_count, *current_reboot_count); 1898 } 1899 1900 fn check_guest_watchdog_no_reboot(guest: &Guest, current_reboot_count: &u32) { 1901 // Allow some normal time to elapse to check we don't get spurious reboots 1902 thread::sleep(std::time::Duration::new(40, 0)); 1903 1904 // Check no reboot 1905 let boot_count = guest 1906 .ssh_command("sudo journalctl --list-boots | wc -l") 1907 .unwrap() 1908 .trim() 1909 .parse::<u32>() 1910 .unwrap_or_default(); 1911 assert_eq!(boot_count, *current_reboot_count); 1912 } 1913 1914 fn check_guest_watchdog_one_reboot( 1915 guest: &Guest, 1916 _api_socket: &str, 1917 current_reboot_count: &mut u32, 1918 ) { 1919 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 1920 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 1921 1922 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 1923 guest.wait_vm_boot(Some(50)).unwrap(); 1924 1925 // Check that watchdog triggered reboot 1926 *current_reboot_count += 1; 1927 let boot_count = guest 1928 .ssh_command("sudo journalctl --list-boots | wc -l") 1929 .unwrap() 1930 .trim() 1931 .parse::<u32>() 1932 .unwrap_or_default(); 1933 assert_eq!(boot_count, *current_reboot_count); 1934 1935 #[cfg(target_arch = "x86_64")] 1936 { 1937 // Now pause the VM and remain offline for 30s 1938 assert!(remote_command(_api_socket, "pause", None)); 1939 thread::sleep(std::time::Duration::new(30, 0)); 1940 assert!(remote_command(_api_socket, "resume", None)); 1941 1942 // Check no reboot 1943 let boot_count = guest 1944 .ssh_command("sudo journalctl --list-boots | wc -l") 1945 .unwrap() 1946 .trim() 1947 .parse::<u32>() 1948 .unwrap_or_default(); 1949 assert_eq!(boot_count, *current_reboot_count); 1950 } 1951 } 1952 1953 mod parallel { 1954 use std::io::SeekFrom; 1955 1956 use crate::*; 1957 1958 #[test] 1959 #[cfg(target_arch = "x86_64")] 1960 fn test_bionic_hypervisor_fw() { 1961 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), BIONIC_IMAGE_NAME) 1962 } 1963 1964 #[test] 1965 #[cfg(target_arch = "x86_64")] 1966 fn test_focal_hypervisor_fw() { 1967 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 1968 } 1969 1970 #[test] 1971 #[cfg(target_arch = "x86_64")] 1972 fn test_bionic_ovmf() { 1973 test_simple_launch(fw_path(FwType::Ovmf), BIONIC_IMAGE_NAME) 1974 } 1975 1976 #[test] 1977 #[cfg(target_arch = "x86_64")] 1978 fn test_focal_ovmf() { 1979 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 1980 } 1981 1982 #[cfg(target_arch = "x86_64")] 1983 fn test_simple_launch(fw_path: String, disk_path: &str) { 1984 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 1985 let guest = Guest::new(disk_config); 1986 let event_path = temp_event_monitor_path(&guest.tmp_dir); 1987 1988 let mut child = GuestCommand::new(&guest) 1989 .args(&["--cpus", "boot=1"]) 1990 .args(&["--memory", "size=512M"]) 1991 .args(&["--kernel", fw_path.as_str()]) 1992 .default_disks() 1993 .default_net() 1994 .args(&["--serial", "tty", "--console", "off"]) 1995 .args(&["--event-monitor", format!("path={}", event_path).as_str()]) 1996 .capture_output() 1997 .spawn() 1998 .unwrap(); 1999 2000 let r = std::panic::catch_unwind(|| { 2001 guest.wait_vm_boot(Some(120)).unwrap(); 2002 2003 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2004 assert_eq!(guest.get_initial_apicid().unwrap_or(1), 0); 2005 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2006 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2007 2008 let expected_sequential_events = [ 2009 &MetaEvent { 2010 event: "starting".to_string(), 2011 device_id: None, 2012 }, 2013 &MetaEvent { 2014 event: "booting".to_string(), 2015 device_id: None, 2016 }, 2017 &MetaEvent { 2018 event: "booted".to_string(), 2019 device_id: None, 2020 }, 2021 &MetaEvent { 2022 event: "activated".to_string(), 2023 device_id: Some("_disk0".to_string()), 2024 }, 2025 &MetaEvent { 2026 event: "reset".to_string(), 2027 device_id: Some("_disk0".to_string()), 2028 }, 2029 ]; 2030 assert!(check_sequential_events( 2031 &expected_sequential_events, 2032 &event_path 2033 )); 2034 2035 guest.ssh_command("sudo poweroff").unwrap(); 2036 thread::sleep(std::time::Duration::new(5, 0)); 2037 let latest_events = [ 2038 &MetaEvent { 2039 event: "shutdown".to_string(), 2040 device_id: None, 2041 }, 2042 &MetaEvent { 2043 event: "deleted".to_string(), 2044 device_id: None, 2045 }, 2046 &MetaEvent { 2047 event: "shutdown".to_string(), 2048 device_id: None, 2049 }, 2050 ]; 2051 assert!(check_latest_events_exact(&latest_events, &event_path)); 2052 }); 2053 2054 let _ = child.kill(); 2055 let output = child.wait_with_output().unwrap(); 2056 2057 handle_child_output(r, &output); 2058 } 2059 2060 #[test] 2061 fn test_multi_cpu() { 2062 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 2063 let guest = Guest::new(Box::new(bionic)); 2064 let mut cmd = GuestCommand::new(&guest); 2065 cmd.args(&["--cpus", "boot=2,max=4"]) 2066 .args(&["--memory", "size=512M"]) 2067 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2068 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2069 .capture_output() 2070 .default_disks() 2071 .default_net(); 2072 2073 let mut child = cmd.spawn().unwrap(); 2074 2075 let r = std::panic::catch_unwind(|| { 2076 guest.wait_vm_boot(Some(120)).unwrap(); 2077 2078 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2079 2080 #[cfg(target_arch = "x86_64")] 2081 assert_eq!( 2082 guest 2083 .ssh_command(r#"dmesg | grep "smpboot: Allowing" | sed "s/\[\ *[0-9.]*\] //""#) 2084 .unwrap() 2085 .trim(), 2086 "smpboot: Allowing 4 CPUs, 2 hotplug CPUs" 2087 ); 2088 #[cfg(target_arch = "aarch64")] 2089 assert_eq!( 2090 guest 2091 .ssh_command(r#"dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""#) 2092 .unwrap() 2093 .trim(), 2094 "smp: Brought up 1 node, 2 CPUs" 2095 ); 2096 }); 2097 2098 let _ = child.kill(); 2099 let output = child.wait_with_output().unwrap(); 2100 2101 handle_child_output(r, &output); 2102 } 2103 2104 #[test] 2105 #[cfg(not(feature = "mshv"))] 2106 fn test_cpu_topology_421() { 2107 test_cpu_topology(4, 2, 1, false); 2108 } 2109 2110 #[test] 2111 #[cfg(not(feature = "mshv"))] 2112 fn test_cpu_topology_142() { 2113 test_cpu_topology(1, 4, 2, false); 2114 } 2115 2116 #[test] 2117 #[cfg(not(feature = "mshv"))] 2118 fn test_cpu_topology_262() { 2119 test_cpu_topology(2, 6, 2, false); 2120 } 2121 2122 #[test] 2123 #[cfg(target_arch = "x86_64")] 2124 #[cfg(not(feature = "mshv"))] 2125 fn test_cpu_physical_bits() { 2126 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2127 let guest = Guest::new(Box::new(focal)); 2128 let max_phys_bits: u8 = 36; 2129 let mut child = GuestCommand::new(&guest) 2130 .args(&["--cpus", &format!("max_phys_bits={}", max_phys_bits)]) 2131 .args(&["--memory", "size=512M"]) 2132 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2133 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2134 .default_disks() 2135 .default_net() 2136 .capture_output() 2137 .spawn() 2138 .unwrap(); 2139 2140 let r = std::panic::catch_unwind(|| { 2141 guest.wait_vm_boot(None).unwrap(); 2142 2143 assert!( 2144 guest 2145 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2146 .unwrap() 2147 .trim() 2148 .parse::<u8>() 2149 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2150 ); 2151 }); 2152 2153 let _ = child.kill(); 2154 let output = child.wait_with_output().unwrap(); 2155 2156 handle_child_output(r, &output); 2157 } 2158 2159 #[test] 2160 fn test_cpu_affinity() { 2161 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2162 let guest = Guest::new(Box::new(focal)); 2163 2164 // We need the host to have at least 4 CPUs if we want to be able 2165 // to run this test. 2166 let host_cpus_count = exec_host_command_output("nproc"); 2167 assert!( 2168 String::from_utf8_lossy(&host_cpus_count.stdout) 2169 .trim() 2170 .parse::<u8>() 2171 .unwrap_or(0) 2172 >= 4 2173 ); 2174 2175 let mut child = GuestCommand::new(&guest) 2176 .args(&["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2177 .args(&["--memory", "size=512M"]) 2178 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2179 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2180 .default_disks() 2181 .default_net() 2182 .capture_output() 2183 .spawn() 2184 .unwrap(); 2185 2186 let r = std::panic::catch_unwind(|| { 2187 guest.wait_vm_boot(None).unwrap(); 2188 let pid = child.id(); 2189 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"", pid).as_str()); 2190 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2191 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"", pid).as_str()); 2192 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2193 }); 2194 2195 let _ = child.kill(); 2196 let output = child.wait_with_output().unwrap(); 2197 2198 handle_child_output(r, &output); 2199 } 2200 2201 #[test] 2202 #[cfg(not(feature = "mshv"))] 2203 fn test_large_vm() { 2204 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2205 let guest = Guest::new(Box::new(focal)); 2206 let mut cmd = GuestCommand::new(&guest); 2207 cmd.args(&["--cpus", "boot=48"]) 2208 .args(&["--memory", "size=5120M"]) 2209 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2210 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2211 .args(&["--serial", "tty"]) 2212 .args(&["--console", "off"]) 2213 .capture_output() 2214 .default_disks() 2215 .default_net(); 2216 2217 let mut child = cmd.spawn().unwrap(); 2218 2219 guest.wait_vm_boot(None).unwrap(); 2220 2221 let r = std::panic::catch_unwind(|| { 2222 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2223 assert_eq!( 2224 guest 2225 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2226 .unwrap() 2227 .trim(), 2228 "0-47" 2229 ); 2230 2231 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2232 }); 2233 2234 let _ = child.kill(); 2235 let output = child.wait_with_output().unwrap(); 2236 2237 handle_child_output(r, &output); 2238 } 2239 2240 #[test] 2241 #[cfg(not(feature = "mshv"))] 2242 fn test_huge_memory() { 2243 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2244 let guest = Guest::new(Box::new(focal)); 2245 let mut cmd = GuestCommand::new(&guest); 2246 cmd.args(&["--cpus", "boot=1"]) 2247 .args(&["--memory", "size=128G"]) 2248 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2249 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2250 .capture_output() 2251 .default_disks() 2252 .default_net(); 2253 2254 let mut child = cmd.spawn().unwrap(); 2255 2256 guest.wait_vm_boot(Some(120)).unwrap(); 2257 2258 let r = std::panic::catch_unwind(|| { 2259 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2260 }); 2261 2262 let _ = child.kill(); 2263 let output = child.wait_with_output().unwrap(); 2264 2265 handle_child_output(r, &output); 2266 } 2267 2268 #[test] 2269 fn test_power_button() { 2270 _test_power_button(false); 2271 } 2272 2273 #[test] 2274 #[cfg(not(feature = "mshv"))] 2275 fn test_user_defined_memory_regions() { 2276 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2277 let guest = Guest::new(Box::new(focal)); 2278 let api_socket = temp_api_path(&guest.tmp_dir); 2279 2280 let kernel_path = direct_kernel_boot_path(); 2281 2282 let mut child = GuestCommand::new(&guest) 2283 .args(&["--cpus", "boot=1"]) 2284 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 2285 .args(&[ 2286 "--memory-zone", 2287 "id=mem0,size=1G,hotplug_size=2G", 2288 "id=mem1,size=1G,file=/dev/shm", 2289 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2290 ]) 2291 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2292 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2293 .args(&["--api-socket", &api_socket]) 2294 .capture_output() 2295 .default_disks() 2296 .default_net() 2297 .spawn() 2298 .unwrap(); 2299 2300 let r = std::panic::catch_unwind(|| { 2301 guest.wait_vm_boot(None).unwrap(); 2302 2303 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2304 2305 guest.enable_memory_hotplug(); 2306 2307 resize_zone_command(&api_socket, "mem0", "3G"); 2308 thread::sleep(std::time::Duration::new(5, 0)); 2309 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2310 resize_zone_command(&api_socket, "mem2", "3G"); 2311 thread::sleep(std::time::Duration::new(5, 0)); 2312 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2313 resize_zone_command(&api_socket, "mem0", "2G"); 2314 thread::sleep(std::time::Duration::new(5, 0)); 2315 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2316 resize_zone_command(&api_socket, "mem2", "2G"); 2317 thread::sleep(std::time::Duration::new(5, 0)); 2318 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2319 2320 guest.reboot_linux(0, None); 2321 2322 // Check the amount of RAM after reboot 2323 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2324 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2325 2326 // Check if we can still resize down to the initial 'boot'size 2327 resize_zone_command(&api_socket, "mem0", "1G"); 2328 thread::sleep(std::time::Duration::new(5, 0)); 2329 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2330 resize_zone_command(&api_socket, "mem2", "1G"); 2331 thread::sleep(std::time::Duration::new(5, 0)); 2332 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2333 }); 2334 2335 let _ = child.kill(); 2336 let output = child.wait_with_output().unwrap(); 2337 2338 handle_child_output(r, &output); 2339 } 2340 2341 #[test] 2342 #[cfg(not(feature = "mshv"))] 2343 fn test_guest_numa_nodes() { 2344 _test_guest_numa_nodes(false); 2345 } 2346 2347 #[test] 2348 #[cfg(target_arch = "x86_64")] 2349 fn test_iommu_segments() { 2350 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2351 let focal = UbuntuDiskConfig::new(focal_image); 2352 let guest = Guest::new(Box::new(focal)); 2353 2354 // Prepare another disk file for the virtio-disk device 2355 let test_disk_path = String::from( 2356 guest 2357 .tmp_dir 2358 .as_path() 2359 .join("test-disk.raw") 2360 .to_str() 2361 .unwrap(), 2362 ); 2363 assert!( 2364 exec_host_command_status(format!("truncate {} -s 4M", test_disk_path).as_str()) 2365 .success() 2366 ); 2367 assert!( 2368 exec_host_command_status(format!("mkfs.ext4 {}", test_disk_path).as_str()).success() 2369 ); 2370 2371 let api_socket = temp_api_path(&guest.tmp_dir); 2372 let mut cmd = GuestCommand::new(&guest); 2373 2374 cmd.args(&["--cpus", "boot=1"]) 2375 .args(&["--api-socket", &api_socket]) 2376 .args(&["--memory", "size=512M"]) 2377 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2378 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2379 .args(&["--platform", "num_pci_segments=16,iommu_segments=[1]"]) 2380 .default_disks() 2381 .capture_output() 2382 .default_net(); 2383 2384 let mut child = cmd.spawn().unwrap(); 2385 2386 guest.wait_vm_boot(None).unwrap(); 2387 2388 let r = std::panic::catch_unwind(|| { 2389 let (cmd_success, cmd_output) = remote_command_w_output( 2390 &api_socket, 2391 "add-disk", 2392 Some( 2393 format!( 2394 "path={},id=test0,pci_segment=1,iommu=on", 2395 test_disk_path.as_str() 2396 ) 2397 .as_str(), 2398 ), 2399 ); 2400 assert!(cmd_success); 2401 assert!(String::from_utf8_lossy(&cmd_output) 2402 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2403 2404 // Check IOMMU setup 2405 assert!(guest 2406 .does_device_vendor_pair_match("0x1057", "0x1af4") 2407 .unwrap_or_default()); 2408 assert_eq!( 2409 guest 2410 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2411 .unwrap() 2412 .trim(), 2413 "0001:00:01.0" 2414 ); 2415 }); 2416 2417 let _ = child.kill(); 2418 let output = child.wait_with_output().unwrap(); 2419 2420 handle_child_output(r, &output); 2421 } 2422 2423 #[test] 2424 fn test_pci_msi() { 2425 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2426 let guest = Guest::new(Box::new(focal)); 2427 let mut cmd = GuestCommand::new(&guest); 2428 cmd.args(&["--cpus", "boot=1"]) 2429 .args(&["--memory", "size=512M"]) 2430 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2431 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2432 .capture_output() 2433 .default_disks() 2434 .default_net(); 2435 2436 let mut child = cmd.spawn().unwrap(); 2437 2438 guest.wait_vm_boot(None).unwrap(); 2439 2440 #[cfg(target_arch = "x86_64")] 2441 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2442 #[cfg(target_arch = "aarch64")] 2443 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2444 2445 let r = std::panic::catch_unwind(|| { 2446 assert_eq!( 2447 guest 2448 .ssh_command(grep_cmd) 2449 .unwrap() 2450 .trim() 2451 .parse::<u32>() 2452 .unwrap_or_default(), 2453 12 2454 ); 2455 }); 2456 2457 let _ = child.kill(); 2458 let output = child.wait_with_output().unwrap(); 2459 2460 handle_child_output(r, &output); 2461 } 2462 2463 #[test] 2464 fn test_virtio_net_ctrl_queue() { 2465 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2466 let guest = Guest::new(Box::new(focal)); 2467 let mut cmd = GuestCommand::new(&guest); 2468 cmd.args(&["--cpus", "boot=1"]) 2469 .args(&["--memory", "size=512M"]) 2470 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2471 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2472 .capture_output() 2473 .default_disks() 2474 .default_net(); 2475 2476 let mut child = cmd.spawn().unwrap(); 2477 2478 guest.wait_vm_boot(None).unwrap(); 2479 2480 #[cfg(target_arch = "aarch64")] 2481 let iface = "enp0s4"; 2482 #[cfg(target_arch = "x86_64")] 2483 let iface = "ens4"; 2484 2485 let r = std::panic::catch_unwind(|| { 2486 assert_eq!( 2487 guest 2488 .ssh_command( 2489 format!("sudo ethtool -K {} rx-gro-hw off && echo success", iface).as_str() 2490 ) 2491 .unwrap() 2492 .trim(), 2493 "success" 2494 ); 2495 }); 2496 2497 let _ = child.kill(); 2498 let output = child.wait_with_output().unwrap(); 2499 2500 handle_child_output(r, &output); 2501 } 2502 2503 #[test] 2504 #[cfg(not(feature = "mshv"))] 2505 fn test_pci_multiple_segments() { 2506 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2507 let guest = Guest::new(Box::new(focal)); 2508 2509 // Prepare another disk file for the virtio-disk device 2510 let test_disk_path = String::from( 2511 guest 2512 .tmp_dir 2513 .as_path() 2514 .join("test-disk.raw") 2515 .to_str() 2516 .unwrap(), 2517 ); 2518 assert!( 2519 exec_host_command_status(format!("truncate {} -s 4M", test_disk_path).as_str()) 2520 .success() 2521 ); 2522 assert!( 2523 exec_host_command_status(format!("mkfs.ext4 {}", test_disk_path).as_str()).success() 2524 ); 2525 2526 let mut cmd = GuestCommand::new(&guest); 2527 cmd.args(&["--cpus", "boot=1"]) 2528 .args(&["--memory", "size=512M"]) 2529 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2530 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2531 .args(&["--platform", "num_pci_segments=16"]) 2532 .args(&[ 2533 "--disk", 2534 format!( 2535 "path={}", 2536 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2537 ) 2538 .as_str(), 2539 format!( 2540 "path={}", 2541 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2542 ) 2543 .as_str(), 2544 format!("path={},pci_segment=15", test_disk_path).as_str(), 2545 ]) 2546 .capture_output() 2547 .default_net(); 2548 2549 let mut child = cmd.spawn().unwrap(); 2550 2551 guest.wait_vm_boot(None).unwrap(); 2552 2553 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2554 2555 let r = std::panic::catch_unwind(|| { 2556 // There should be 16 PCI host bridges in the guest. 2557 assert_eq!( 2558 guest 2559 .ssh_command(grep_cmd) 2560 .unwrap() 2561 .trim() 2562 .parse::<u32>() 2563 .unwrap_or_default(), 2564 16 2565 ); 2566 2567 // Check both if /dev/vdc exists and if the block size is 4M. 2568 assert_eq!( 2569 guest 2570 .ssh_command("lsblk | grep vdc | grep -c 4M") 2571 .unwrap() 2572 .trim() 2573 .parse::<u32>() 2574 .unwrap_or_default(), 2575 1 2576 ); 2577 2578 // Mount the device. 2579 guest.ssh_command("mkdir mount_image").unwrap(); 2580 guest 2581 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 2582 .unwrap(); 2583 // Grant all users with write permission. 2584 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 2585 2586 // Write something to the device. 2587 guest 2588 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 2589 .unwrap(); 2590 2591 // Check the content of the block device. The file "foo" should 2592 // contain "bar". 2593 assert_eq!( 2594 guest 2595 .ssh_command("sudo cat mount_image/foo") 2596 .unwrap() 2597 .trim(), 2598 "bar" 2599 ); 2600 }); 2601 2602 let _ = child.kill(); 2603 let output = child.wait_with_output().unwrap(); 2604 2605 handle_child_output(r, &output); 2606 } 2607 2608 #[test] 2609 fn test_direct_kernel_boot() { 2610 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2611 let guest = Guest::new(Box::new(focal)); 2612 2613 let kernel_path = direct_kernel_boot_path(); 2614 2615 let mut child = GuestCommand::new(&guest) 2616 .args(&["--cpus", "boot=1"]) 2617 .args(&["--memory", "size=512M"]) 2618 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2619 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2620 .default_disks() 2621 .default_net() 2622 .capture_output() 2623 .spawn() 2624 .unwrap(); 2625 2626 let r = std::panic::catch_unwind(|| { 2627 guest.wait_vm_boot(None).unwrap(); 2628 2629 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2630 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2631 2632 let grep_cmd = if cfg!(target_arch = "x86_64") { 2633 "grep -c PCI-MSI /proc/interrupts" 2634 } else { 2635 "grep -c ITS-MSI /proc/interrupts" 2636 }; 2637 assert_eq!( 2638 guest 2639 .ssh_command(grep_cmd) 2640 .unwrap() 2641 .trim() 2642 .parse::<u32>() 2643 .unwrap_or_default(), 2644 12 2645 ); 2646 }); 2647 2648 let _ = child.kill(); 2649 let output = child.wait_with_output().unwrap(); 2650 2651 handle_child_output(r, &output); 2652 } 2653 2654 fn _test_virtio_block(image_name: &str, disable_io_uring: bool) { 2655 let focal = UbuntuDiskConfig::new(image_name.to_string()); 2656 let guest = Guest::new(Box::new(focal)); 2657 2658 let mut workload_path = dirs::home_dir().unwrap(); 2659 workload_path.push("workloads"); 2660 2661 let mut blk_file_path = workload_path; 2662 blk_file_path.push("blk.img"); 2663 2664 let kernel_path = direct_kernel_boot_path(); 2665 2666 let mut cloud_child = GuestCommand::new(&guest) 2667 .args(&["--cpus", "boot=4"]) 2668 .args(&["--memory", "size=512M,shared=on"]) 2669 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2670 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2671 .args(&[ 2672 "--disk", 2673 format!( 2674 "path={}", 2675 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2676 ) 2677 .as_str(), 2678 format!( 2679 "path={}", 2680 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2681 ) 2682 .as_str(), 2683 format!( 2684 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={}", 2685 blk_file_path.to_str().unwrap(), 2686 disable_io_uring 2687 ) 2688 .as_str(), 2689 ]) 2690 .default_net() 2691 .capture_output() 2692 .spawn() 2693 .unwrap(); 2694 2695 let r = std::panic::catch_unwind(|| { 2696 guest.wait_vm_boot(None).unwrap(); 2697 2698 // Check both if /dev/vdc exists and if the block size is 16M. 2699 assert_eq!( 2700 guest 2701 .ssh_command("lsblk | grep vdc | grep -c 16M") 2702 .unwrap() 2703 .trim() 2704 .parse::<u32>() 2705 .unwrap_or_default(), 2706 1 2707 ); 2708 2709 // Check both if /dev/vdc exists and if this block is RO. 2710 assert_eq!( 2711 guest 2712 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 2713 .unwrap() 2714 .trim() 2715 .parse::<u32>() 2716 .unwrap_or_default(), 2717 1 2718 ); 2719 2720 // Check if the number of queues is 4. 2721 assert_eq!( 2722 guest 2723 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 2724 .unwrap() 2725 .trim() 2726 .parse::<u32>() 2727 .unwrap_or_default(), 2728 4 2729 ); 2730 }); 2731 2732 let _ = cloud_child.kill(); 2733 let output = cloud_child.wait_with_output().unwrap(); 2734 2735 handle_child_output(r, &output); 2736 } 2737 2738 #[test] 2739 fn test_virtio_block() { 2740 _test_virtio_block(FOCAL_IMAGE_NAME, false) 2741 } 2742 2743 #[test] 2744 fn test_virtio_block_disable_io_uring() { 2745 _test_virtio_block(FOCAL_IMAGE_NAME, true) 2746 } 2747 2748 #[test] 2749 fn test_virtio_block_qcow2() { 2750 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false) 2751 } 2752 2753 #[test] 2754 fn test_virtio_block_vhd() { 2755 let mut workload_path = dirs::home_dir().unwrap(); 2756 workload_path.push("workloads"); 2757 2758 let mut raw_file_path = workload_path.clone(); 2759 let mut vhd_file_path = workload_path; 2760 raw_file_path.push(FOCAL_IMAGE_NAME); 2761 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 2762 2763 // Generate VHD file from RAW file 2764 std::process::Command::new("qemu-img") 2765 .arg("convert") 2766 .arg("-p") 2767 .args(&["-f", "raw"]) 2768 .args(&["-O", "vpc"]) 2769 .args(&["-o", "subformat=fixed"]) 2770 .arg(raw_file_path.to_str().unwrap()) 2771 .arg(vhd_file_path.to_str().unwrap()) 2772 .output() 2773 .expect("Expect generating VHD image from RAW image"); 2774 2775 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false) 2776 } 2777 2778 #[test] 2779 fn test_virtio_block_vhdx() { 2780 let mut workload_path = dirs::home_dir().unwrap(); 2781 workload_path.push("workloads"); 2782 2783 let mut raw_file_path = workload_path.clone(); 2784 let mut vhdx_file_path = workload_path; 2785 raw_file_path.push(FOCAL_IMAGE_NAME); 2786 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 2787 2788 // Generate dynamic VHDX file from RAW file 2789 std::process::Command::new("qemu-img") 2790 .arg("convert") 2791 .arg("-p") 2792 .args(&["-f", "raw"]) 2793 .args(&["-O", "vhdx"]) 2794 .arg(raw_file_path.to_str().unwrap()) 2795 .arg(vhdx_file_path.to_str().unwrap()) 2796 .output() 2797 .expect("Expect generating dynamic VHDx image from RAW image"); 2798 2799 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false) 2800 } 2801 2802 #[test] 2803 fn test_virtio_block_dynamic_vhdx_expand() { 2804 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 2805 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 2806 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 2807 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 2808 2809 let mut workload_path = dirs::home_dir().unwrap(); 2810 workload_path.push("workloads"); 2811 2812 let mut vhdx_file_path = workload_path; 2813 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 2814 let vhdx_path = vhdx_file_path.to_str().unwrap(); 2815 2816 // Generate a 100 MiB dynamic VHDX file 2817 std::process::Command::new("qemu-img") 2818 .arg("create") 2819 .args(&["-f", "vhdx"]) 2820 .arg(vhdx_path) 2821 .arg(VIRTUAL_DISK_SIZE.to_string()) 2822 .output() 2823 .expect("Expect generating dynamic VHDx image from RAW image"); 2824 2825 // Check if the size matches with empty VHDx file size 2826 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 2827 2828 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2829 let guest = Guest::new(Box::new(focal)); 2830 let kernel_path = direct_kernel_boot_path(); 2831 2832 let mut cloud_child = GuestCommand::new(&guest) 2833 .args(&["--cpus", "boot=1"]) 2834 .args(&["--memory", "size=512M"]) 2835 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2836 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2837 .args(&[ 2838 "--disk", 2839 format!( 2840 "path={}", 2841 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2842 ) 2843 .as_str(), 2844 format!( 2845 "path={}", 2846 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2847 ) 2848 .as_str(), 2849 format!("path={}", vhdx_path).as_str(), 2850 ]) 2851 .default_net() 2852 .capture_output() 2853 .spawn() 2854 .unwrap(); 2855 2856 let r = std::panic::catch_unwind(|| { 2857 guest.wait_vm_boot(None).unwrap(); 2858 2859 // Check both if /dev/vdc exists and if the block size is 100 MiB. 2860 assert_eq!( 2861 guest 2862 .ssh_command("lsblk | grep vdc | grep -c 100M") 2863 .unwrap() 2864 .trim() 2865 .parse::<u32>() 2866 .unwrap_or_default(), 2867 1 2868 ); 2869 2870 // Write 100 MB of data to the VHDx disk 2871 guest 2872 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 2873 .unwrap(); 2874 }); 2875 2876 // Check if the size matches with expected expanded VHDx file size 2877 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 2878 2879 let _ = cloud_child.kill(); 2880 let output = cloud_child.wait_with_output().unwrap(); 2881 2882 handle_child_output(r, &output); 2883 } 2884 2885 fn vhdx_image_size(disk_name: &str) -> u64 { 2886 std::fs::File::open(disk_name) 2887 .unwrap() 2888 .seek(SeekFrom::End(0)) 2889 .unwrap() 2890 } 2891 2892 #[test] 2893 fn test_virtio_block_direct_and_firmware() { 2894 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2895 let guest = Guest::new(Box::new(focal)); 2896 2897 // The OS disk must be copied to a location that is not backed by 2898 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 2899 // with EINVAL because tmpfs doesn't support this flag. 2900 let mut workloads_path = dirs::home_dir().unwrap(); 2901 workloads_path.push("workloads"); 2902 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 2903 let mut os_path = os_dir.as_path().to_path_buf(); 2904 os_path.push("osdisk.img"); 2905 rate_limited_copy( 2906 &guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 2907 os_path.as_path(), 2908 ) 2909 .expect("copying of OS disk failed"); 2910 2911 let mut child = GuestCommand::new(&guest) 2912 .args(&["--cpus", "boot=1"]) 2913 .args(&["--memory", "size=512M"]) 2914 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 2915 .args(&[ 2916 "--disk", 2917 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 2918 format!( 2919 "path={}", 2920 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2921 ) 2922 .as_str(), 2923 ]) 2924 .default_net() 2925 .capture_output() 2926 .spawn() 2927 .unwrap(); 2928 2929 let r = std::panic::catch_unwind(|| { 2930 guest.wait_vm_boot(Some(120)).unwrap(); 2931 }); 2932 2933 let _ = child.kill(); 2934 let output = child.wait_with_output().unwrap(); 2935 2936 handle_child_output(r, &output); 2937 } 2938 2939 #[test] 2940 fn test_vhost_user_net_default() { 2941 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 2942 } 2943 2944 #[test] 2945 fn test_vhost_user_net_named_tap() { 2946 test_vhost_user_net( 2947 Some("mytap0"), 2948 2, 2949 &prepare_vhost_user_net_daemon, 2950 false, 2951 false, 2952 ) 2953 } 2954 2955 #[test] 2956 fn test_vhost_user_net_existing_tap() { 2957 test_vhost_user_net( 2958 Some("vunet-tap0"), 2959 2, 2960 &prepare_vhost_user_net_daemon, 2961 false, 2962 false, 2963 ) 2964 } 2965 2966 #[test] 2967 fn test_vhost_user_net_multiple_queues() { 2968 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 2969 } 2970 2971 #[test] 2972 fn test_vhost_user_net_tap_multiple_queues() { 2973 test_vhost_user_net( 2974 Some("vunet-tap1"), 2975 4, 2976 &prepare_vhost_user_net_daemon, 2977 false, 2978 false, 2979 ) 2980 } 2981 2982 #[test] 2983 fn test_vhost_user_net_host_mac() { 2984 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 2985 } 2986 2987 #[test] 2988 fn test_vhost_user_net_client_mode() { 2989 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 2990 } 2991 2992 #[test] 2993 fn test_vhost_user_blk_default() { 2994 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 2995 } 2996 2997 #[test] 2998 fn test_vhost_user_blk_readonly() { 2999 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3000 } 3001 3002 #[test] 3003 fn test_vhost_user_blk_direct() { 3004 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3005 } 3006 3007 #[test] 3008 fn test_boot_from_vhost_user_blk_default() { 3009 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3010 } 3011 3012 #[test] 3013 #[cfg(target_arch = "x86_64")] 3014 fn test_split_irqchip() { 3015 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3016 let guest = Guest::new(Box::new(focal)); 3017 3018 let mut child = GuestCommand::new(&guest) 3019 .args(&["--cpus", "boot=1"]) 3020 .args(&["--memory", "size=512M"]) 3021 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3022 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3023 .default_disks() 3024 .default_net() 3025 .capture_output() 3026 .spawn() 3027 .unwrap(); 3028 3029 let r = std::panic::catch_unwind(|| { 3030 guest.wait_vm_boot(None).unwrap(); 3031 3032 assert_eq!( 3033 guest 3034 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3035 .unwrap() 3036 .trim() 3037 .parse::<u32>() 3038 .unwrap_or(1), 3039 0 3040 ); 3041 assert_eq!( 3042 guest 3043 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3044 .unwrap() 3045 .trim() 3046 .parse::<u32>() 3047 .unwrap_or(1), 3048 0 3049 ); 3050 }); 3051 3052 let _ = child.kill(); 3053 let output = child.wait_with_output().unwrap(); 3054 3055 handle_child_output(r, &output); 3056 } 3057 3058 #[test] 3059 #[cfg(target_arch = "x86_64")] 3060 fn test_dmi_serial_number() { 3061 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3062 let guest = Guest::new(Box::new(focal)); 3063 3064 let mut child = GuestCommand::new(&guest) 3065 .args(&["--cpus", "boot=1"]) 3066 .args(&["--memory", "size=512M"]) 3067 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3068 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3069 .args(&["--platform", "serial_number=a=b;c=d"]) 3070 .default_disks() 3071 .default_net() 3072 .capture_output() 3073 .spawn() 3074 .unwrap(); 3075 3076 let r = std::panic::catch_unwind(|| { 3077 guest.wait_vm_boot(None).unwrap(); 3078 3079 assert_eq!( 3080 guest 3081 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3082 .unwrap() 3083 .trim(), 3084 "a=b;c=d" 3085 ); 3086 }); 3087 3088 let _ = child.kill(); 3089 let output = child.wait_with_output().unwrap(); 3090 3091 handle_child_output(r, &output); 3092 } 3093 3094 #[test] 3095 fn test_virtio_fs() { 3096 _test_virtio_fs(&prepare_virtiofsd, false, None) 3097 } 3098 3099 #[test] 3100 fn test_virtio_fs_hotplug() { 3101 _test_virtio_fs(&prepare_virtiofsd, true, None) 3102 } 3103 3104 #[test] 3105 #[cfg(not(feature = "mshv"))] 3106 fn test_virtio_fs_multi_segment_hotplug() { 3107 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3108 } 3109 3110 #[test] 3111 #[cfg(not(feature = "mshv"))] 3112 fn test_virtio_fs_multi_segment() { 3113 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3114 } 3115 3116 #[test] 3117 fn test_virtio_pmem_persist_writes() { 3118 test_virtio_pmem(false, false) 3119 } 3120 3121 #[test] 3122 fn test_virtio_pmem_discard_writes() { 3123 test_virtio_pmem(true, false) 3124 } 3125 3126 #[test] 3127 fn test_virtio_pmem_with_size() { 3128 test_virtio_pmem(true, true) 3129 } 3130 3131 #[test] 3132 fn test_boot_from_virtio_pmem() { 3133 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3134 let guest = Guest::new(Box::new(focal)); 3135 3136 let kernel_path = direct_kernel_boot_path(); 3137 3138 let mut child = GuestCommand::new(&guest) 3139 .args(&["--cpus", "boot=1"]) 3140 .args(&["--memory", "size=512M"]) 3141 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3142 .args(&[ 3143 "--disk", 3144 format!( 3145 "path={}", 3146 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3147 ) 3148 .as_str(), 3149 ]) 3150 .default_net() 3151 .args(&[ 3152 "--pmem", 3153 format!( 3154 "file={},size={}", 3155 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3156 fs::metadata(&guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3157 .unwrap() 3158 .len() 3159 ) 3160 .as_str(), 3161 ]) 3162 .args(&[ 3163 "--cmdline", 3164 DIRECT_KERNEL_BOOT_CMDLINE 3165 .replace("vda1", "pmem0p1") 3166 .as_str(), 3167 ]) 3168 .capture_output() 3169 .spawn() 3170 .unwrap(); 3171 3172 let r = std::panic::catch_unwind(|| { 3173 guest.wait_vm_boot(None).unwrap(); 3174 3175 // Simple checks to validate the VM booted properly 3176 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3177 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3178 }); 3179 3180 let _ = child.kill(); 3181 let output = child.wait_with_output().unwrap(); 3182 3183 handle_child_output(r, &output); 3184 } 3185 3186 #[test] 3187 fn test_multiple_network_interfaces() { 3188 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3189 let guest = Guest::new(Box::new(focal)); 3190 3191 let kernel_path = direct_kernel_boot_path(); 3192 3193 let mut child = GuestCommand::new(&guest) 3194 .args(&["--cpus", "boot=1"]) 3195 .args(&["--memory", "size=512M"]) 3196 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3197 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3198 .default_disks() 3199 .args(&[ 3200 "--net", 3201 guest.default_net_string().as_str(), 3202 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3203 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3204 ]) 3205 .capture_output() 3206 .spawn() 3207 .unwrap(); 3208 3209 let r = std::panic::catch_unwind(|| { 3210 guest.wait_vm_boot(None).unwrap(); 3211 3212 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3213 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3214 3215 // 3 network interfaces + default localhost ==> 4 interfaces 3216 assert_eq!( 3217 guest 3218 .ssh_command("ip -o link | wc -l") 3219 .unwrap() 3220 .trim() 3221 .parse::<u32>() 3222 .unwrap_or_default(), 3223 4 3224 ); 3225 }); 3226 3227 let _ = child.kill(); 3228 let output = child.wait_with_output().unwrap(); 3229 3230 handle_child_output(r, &output); 3231 } 3232 3233 #[test] 3234 #[cfg(target_arch = "aarch64")] 3235 fn test_pmu_on() { 3236 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3237 let guest = Guest::new(Box::new(focal)); 3238 let mut child = GuestCommand::new(&guest) 3239 .args(&["--cpus", "boot=1"]) 3240 .args(&["--memory", "size=512M"]) 3241 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3242 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3243 .default_disks() 3244 .default_net() 3245 .capture_output() 3246 .spawn() 3247 .unwrap(); 3248 3249 let r = std::panic::catch_unwind(|| { 3250 guest.wait_vm_boot(None).unwrap(); 3251 3252 // Test that PMU exists. 3253 assert_eq!( 3254 guest 3255 .ssh_command(GREP_PMU_IRQ_CMD) 3256 .unwrap() 3257 .trim() 3258 .parse::<u32>() 3259 .unwrap_or_default(), 3260 1 3261 ); 3262 }); 3263 3264 let _ = child.kill(); 3265 let output = child.wait_with_output().unwrap(); 3266 3267 handle_child_output(r, &output); 3268 } 3269 3270 #[test] 3271 fn test_serial_off() { 3272 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3273 let guest = Guest::new(Box::new(focal)); 3274 let mut child = GuestCommand::new(&guest) 3275 .args(&["--cpus", "boot=1"]) 3276 .args(&["--memory", "size=512M"]) 3277 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3278 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3279 .default_disks() 3280 .default_net() 3281 .args(&["--serial", "off"]) 3282 .capture_output() 3283 .spawn() 3284 .unwrap(); 3285 3286 let r = std::panic::catch_unwind(|| { 3287 guest.wait_vm_boot(None).unwrap(); 3288 3289 // Test that there is no ttyS0 3290 assert_eq!( 3291 guest 3292 .ssh_command(GREP_SERIAL_IRQ_CMD) 3293 .unwrap() 3294 .trim() 3295 .parse::<u32>() 3296 .unwrap_or(1), 3297 0 3298 ); 3299 }); 3300 3301 let _ = child.kill(); 3302 let output = child.wait_with_output().unwrap(); 3303 3304 handle_child_output(r, &output); 3305 } 3306 3307 #[test] 3308 fn test_serial_null() { 3309 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3310 let guest = Guest::new(Box::new(focal)); 3311 let mut cmd = GuestCommand::new(&guest); 3312 #[cfg(target_arch = "x86_64")] 3313 let console_str: &str = "console=ttyS0"; 3314 #[cfg(target_arch = "aarch64")] 3315 let console_str: &str = "console=ttyAMA0"; 3316 3317 cmd.args(&["--cpus", "boot=1"]) 3318 .args(&["--memory", "size=512M"]) 3319 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3320 .args(&[ 3321 "--cmdline", 3322 DIRECT_KERNEL_BOOT_CMDLINE 3323 .replace("console=hvc0 ", console_str) 3324 .as_str(), 3325 ]) 3326 .default_disks() 3327 .default_net() 3328 .args(&["--serial", "null"]) 3329 .args(&["--console", "off"]) 3330 .capture_output(); 3331 3332 let mut child = cmd.spawn().unwrap(); 3333 3334 let r = std::panic::catch_unwind(|| { 3335 guest.wait_vm_boot(None).unwrap(); 3336 3337 // Test that there is a ttyS0 3338 assert_eq!( 3339 guest 3340 .ssh_command(GREP_SERIAL_IRQ_CMD) 3341 .unwrap() 3342 .trim() 3343 .parse::<u32>() 3344 .unwrap_or_default(), 3345 1 3346 ); 3347 }); 3348 3349 let _ = child.kill(); 3350 let output = child.wait_with_output().unwrap(); 3351 handle_child_output(r, &output); 3352 3353 let r = std::panic::catch_unwind(|| { 3354 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3355 }); 3356 3357 handle_child_output(r, &output); 3358 } 3359 3360 #[test] 3361 fn test_serial_tty() { 3362 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3363 let guest = Guest::new(Box::new(focal)); 3364 3365 let kernel_path = direct_kernel_boot_path(); 3366 3367 #[cfg(target_arch = "x86_64")] 3368 let console_str: &str = "console=ttyS0"; 3369 #[cfg(target_arch = "aarch64")] 3370 let console_str: &str = "console=ttyAMA0"; 3371 3372 let mut child = GuestCommand::new(&guest) 3373 .args(&["--cpus", "boot=1"]) 3374 .args(&["--memory", "size=512M"]) 3375 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3376 .args(&[ 3377 "--cmdline", 3378 DIRECT_KERNEL_BOOT_CMDLINE 3379 .replace("console=hvc0 ", console_str) 3380 .as_str(), 3381 ]) 3382 .default_disks() 3383 .default_net() 3384 .args(&["--serial", "tty"]) 3385 .args(&["--console", "off"]) 3386 .capture_output() 3387 .spawn() 3388 .unwrap(); 3389 3390 let r = std::panic::catch_unwind(|| { 3391 guest.wait_vm_boot(None).unwrap(); 3392 3393 // Test that there is a ttyS0 3394 assert_eq!( 3395 guest 3396 .ssh_command(GREP_SERIAL_IRQ_CMD) 3397 .unwrap() 3398 .trim() 3399 .parse::<u32>() 3400 .unwrap_or_default(), 3401 1 3402 ); 3403 }); 3404 3405 // This sleep is needed to wait for the login prompt 3406 thread::sleep(std::time::Duration::new(2, 0)); 3407 3408 let _ = child.kill(); 3409 let output = child.wait_with_output().unwrap(); 3410 handle_child_output(r, &output); 3411 3412 let r = std::panic::catch_unwind(|| { 3413 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3414 }); 3415 3416 handle_child_output(r, &output); 3417 } 3418 3419 #[test] 3420 fn test_serial_file() { 3421 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3422 let guest = Guest::new(Box::new(focal)); 3423 3424 let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output"); 3425 #[cfg(target_arch = "x86_64")] 3426 let console_str: &str = "console=ttyS0"; 3427 #[cfg(target_arch = "aarch64")] 3428 let console_str: &str = "console=ttyAMA0"; 3429 3430 let mut child = GuestCommand::new(&guest) 3431 .args(&["--cpus", "boot=1"]) 3432 .args(&["--memory", "size=512M"]) 3433 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3434 .args(&[ 3435 "--cmdline", 3436 DIRECT_KERNEL_BOOT_CMDLINE 3437 .replace("console=hvc0 ", console_str) 3438 .as_str(), 3439 ]) 3440 .default_disks() 3441 .default_net() 3442 .args(&[ 3443 "--serial", 3444 format!("file={}", serial_path.to_str().unwrap()).as_str(), 3445 ]) 3446 .capture_output() 3447 .spawn() 3448 .unwrap(); 3449 3450 let r = std::panic::catch_unwind(|| { 3451 guest.wait_vm_boot(None).unwrap(); 3452 3453 // Test that there is a ttyS0 3454 assert_eq!( 3455 guest 3456 .ssh_command(GREP_SERIAL_IRQ_CMD) 3457 .unwrap() 3458 .trim() 3459 .parse::<u32>() 3460 .unwrap_or_default(), 3461 1 3462 ); 3463 3464 guest.ssh_command("sudo shutdown -h now").unwrap(); 3465 }); 3466 3467 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3468 let _ = child.kill(); 3469 let output = child.wait_with_output().unwrap(); 3470 handle_child_output(r, &output); 3471 3472 let r = std::panic::catch_unwind(|| { 3473 // Check that the cloud-hypervisor binary actually terminated 3474 assert!(output.status.success()); 3475 3476 // Do this check after shutdown of the VM as an easy way to ensure 3477 // all writes are flushed to disk 3478 let mut f = std::fs::File::open(serial_path).unwrap(); 3479 let mut buf = String::new(); 3480 f.read_to_string(&mut buf).unwrap(); 3481 assert!(buf.contains(CONSOLE_TEST_STRING)); 3482 }); 3483 3484 handle_child_output(r, &output); 3485 } 3486 3487 #[test] 3488 fn test_pty_interaction() { 3489 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3490 let guest = Guest::new(Box::new(focal)); 3491 let api_socket = temp_api_path(&guest.tmp_dir); 3492 let serial_option = if cfg!(target_arch = "x86_64") { 3493 " console=ttyS0" 3494 } else { 3495 " console=ttyAMA0" 3496 }; 3497 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 3498 3499 let mut child = GuestCommand::new(&guest) 3500 .args(&["--cpus", "boot=1"]) 3501 .args(&["--memory", "size=512M"]) 3502 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3503 .args(&["--cmdline", &cmdline]) 3504 .default_disks() 3505 .default_net() 3506 .args(&["--serial", "null"]) 3507 .args(&["--console", "pty"]) 3508 .args(&["--api-socket", &api_socket]) 3509 .spawn() 3510 .unwrap(); 3511 3512 let r = std::panic::catch_unwind(|| { 3513 guest.wait_vm_boot(None).unwrap(); 3514 // Get pty fd for console 3515 let console_path = get_pty_path(&api_socket, "console"); 3516 // TODO: Get serial pty test working 3517 let mut cf = std::fs::OpenOptions::new() 3518 .write(true) 3519 .read(true) 3520 .open(console_path) 3521 .unwrap(); 3522 3523 // Some dumb sleeps but we don't want to write 3524 // before the console is up and we don't want 3525 // to try and write the next line before the 3526 // login process is ready. 3527 thread::sleep(std::time::Duration::new(5, 0)); 3528 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 3529 thread::sleep(std::time::Duration::new(2, 0)); 3530 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 3531 thread::sleep(std::time::Duration::new(2, 0)); 3532 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 3533 thread::sleep(std::time::Duration::new(2, 0)); 3534 3535 // read pty and ensure they have a login shell 3536 // some fairly hacky workarounds to avoid looping 3537 // forever in case the channel is blocked getting output 3538 let ptyc = pty_read(cf); 3539 let mut empty = 0; 3540 let mut prev = String::new(); 3541 loop { 3542 thread::sleep(std::time::Duration::new(2, 0)); 3543 match ptyc.try_recv() { 3544 Ok(line) => { 3545 empty = 0; 3546 prev = prev + &line; 3547 if prev.contains("test_pty_console") { 3548 break; 3549 } 3550 } 3551 Err(mpsc::TryRecvError::Empty) => { 3552 empty += 1; 3553 assert!(empty <= 5, "No login on pty"); 3554 } 3555 _ => panic!("No login on pty"), 3556 } 3557 } 3558 3559 guest.ssh_command("sudo shutdown -h now").unwrap(); 3560 }); 3561 3562 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3563 let _ = child.kill(); 3564 let output = child.wait_with_output().unwrap(); 3565 handle_child_output(r, &output); 3566 3567 let r = std::panic::catch_unwind(|| { 3568 // Check that the cloud-hypervisor binary actually terminated 3569 assert!(output.status.success()) 3570 }); 3571 handle_child_output(r, &output); 3572 } 3573 3574 #[test] 3575 fn test_virtio_console() { 3576 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3577 let guest = Guest::new(Box::new(focal)); 3578 3579 let kernel_path = direct_kernel_boot_path(); 3580 3581 let mut child = GuestCommand::new(&guest) 3582 .args(&["--cpus", "boot=1"]) 3583 .args(&["--memory", "size=512M"]) 3584 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3585 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3586 .default_disks() 3587 .default_net() 3588 .args(&["--console", "tty"]) 3589 .args(&["--serial", "null"]) 3590 .capture_output() 3591 .spawn() 3592 .unwrap(); 3593 3594 let text = String::from("On a branch floating down river a cricket, singing."); 3595 let cmd = format!("echo {} | sudo tee /dev/hvc0", text); 3596 3597 let r = std::panic::catch_unwind(|| { 3598 guest.wait_vm_boot(None).unwrap(); 3599 3600 assert!(guest 3601 .does_device_vendor_pair_match("0x1043", "0x1af4") 3602 .unwrap_or_default()); 3603 3604 guest.ssh_command(&cmd).unwrap(); 3605 }); 3606 3607 let _ = child.kill(); 3608 let output = child.wait_with_output().unwrap(); 3609 handle_child_output(r, &output); 3610 3611 let r = std::panic::catch_unwind(|| { 3612 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 3613 }); 3614 3615 handle_child_output(r, &output); 3616 } 3617 3618 #[test] 3619 fn test_console_file() { 3620 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3621 let guest = Guest::new(Box::new(focal)); 3622 3623 let console_path = guest.tmp_dir.as_path().join("/tmp/console-output"); 3624 let mut child = GuestCommand::new(&guest) 3625 .args(&["--cpus", "boot=1"]) 3626 .args(&["--memory", "size=512M"]) 3627 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3628 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3629 .default_disks() 3630 .default_net() 3631 .args(&[ 3632 "--console", 3633 format!("file={}", console_path.to_str().unwrap()).as_str(), 3634 ]) 3635 .capture_output() 3636 .spawn() 3637 .unwrap(); 3638 3639 guest.wait_vm_boot(None).unwrap(); 3640 3641 guest.ssh_command("sudo shutdown -h now").unwrap(); 3642 3643 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3644 let _ = child.kill(); 3645 let output = child.wait_with_output().unwrap(); 3646 3647 let r = std::panic::catch_unwind(|| { 3648 // Check that the cloud-hypervisor binary actually terminated 3649 assert!(output.status.success()); 3650 3651 // Do this check after shutdown of the VM as an easy way to ensure 3652 // all writes are flushed to disk 3653 let mut f = std::fs::File::open(console_path).unwrap(); 3654 let mut buf = String::new(); 3655 f.read_to_string(&mut buf).unwrap(); 3656 3657 if !buf.contains(CONSOLE_TEST_STRING) { 3658 eprintln!( 3659 "\n\n==== Console file output ====\n\n{}\n\n==== End console file output ====", 3660 buf 3661 ); 3662 } 3663 assert!(buf.contains(CONSOLE_TEST_STRING)); 3664 }); 3665 3666 handle_child_output(r, &output); 3667 } 3668 3669 #[test] 3670 #[cfg(target_arch = "x86_64")] 3671 #[cfg(not(feature = "mshv"))] 3672 #[ignore = "See #4324"] 3673 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 3674 // backed networking interfaces, bound through a simple bridge on the host. 3675 // So if the nested cloud-hypervisor succeeds in getting a directly 3676 // assigned interface from its cloud-hypervisor host, we should be able to 3677 // ssh into it, and verify that it's running with the right kernel command 3678 // line (We tag the command line from cloud-hypervisor for that purpose). 3679 // The third device is added to validate that hotplug works correctly since 3680 // it is being added to the L2 VM through hotplugging mechanism. 3681 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 3682 // vfio device support 3683 fn test_vfio() { 3684 setup_vfio_network_interfaces(); 3685 3686 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3687 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 3688 3689 let mut workload_path = dirs::home_dir().unwrap(); 3690 workload_path.push("workloads"); 3691 3692 let kernel_path = direct_kernel_boot_path(); 3693 3694 let mut vfio_path = workload_path.clone(); 3695 vfio_path.push("vfio"); 3696 3697 let mut cloud_init_vfio_base_path = vfio_path.clone(); 3698 cloud_init_vfio_base_path.push("cloudinit.img"); 3699 3700 // We copy our cloudinit into the vfio mount point, for the nested 3701 // cloud-hypervisor guest to use. 3702 rate_limited_copy( 3703 &guest.disk_config.disk(DiskType::CloudInit).unwrap(), 3704 &cloud_init_vfio_base_path, 3705 ) 3706 .expect("copying of cloud-init disk failed"); 3707 3708 let mut vfio_disk_path = workload_path.clone(); 3709 vfio_disk_path.push("vfio.img"); 3710 3711 // Create the vfio disk image 3712 let output = Command::new("mkfs.ext4") 3713 .arg("-d") 3714 .arg(vfio_path.to_str().unwrap()) 3715 .arg(vfio_disk_path.to_str().unwrap()) 3716 .arg("2g") 3717 .output() 3718 .unwrap(); 3719 if !output.status.success() { 3720 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 3721 panic!("mkfs.ext4 command generated an error"); 3722 } 3723 3724 let mut blk_file_path = workload_path; 3725 blk_file_path.push("blk.img"); 3726 3727 let vfio_tap0 = "vfio-tap0"; 3728 let vfio_tap1 = "vfio-tap1"; 3729 let vfio_tap2 = "vfio-tap2"; 3730 let vfio_tap3 = "vfio-tap3"; 3731 3732 let mut child = GuestCommand::new(&guest) 3733 .args(&["--cpus", "boot=4"]) 3734 .args(&["--memory", "size=2G,hugepages=on,shared=on"]) 3735 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3736 .args(&[ 3737 "--disk", 3738 format!( 3739 "path={}", 3740 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3741 ) 3742 .as_str(), 3743 format!( 3744 "path={}", 3745 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3746 ) 3747 .as_str(), 3748 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 3749 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 3750 ]) 3751 .args(&[ 3752 "--cmdline", 3753 format!( 3754 "{} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts", 3755 DIRECT_KERNEL_BOOT_CMDLINE 3756 ) 3757 .as_str(), 3758 ]) 3759 .args(&[ 3760 "--net", 3761 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 3762 format!( 3763 "tap={},mac={},iommu=on", 3764 vfio_tap1, guest.network.l2_guest_mac1 3765 ) 3766 .as_str(), 3767 format!( 3768 "tap={},mac={},iommu=on", 3769 vfio_tap2, guest.network.l2_guest_mac2 3770 ) 3771 .as_str(), 3772 format!( 3773 "tap={},mac={},iommu=on", 3774 vfio_tap3, guest.network.l2_guest_mac3 3775 ) 3776 .as_str(), 3777 ]) 3778 .capture_output() 3779 .spawn() 3780 .unwrap(); 3781 3782 thread::sleep(std::time::Duration::new(30, 0)); 3783 3784 let r = std::panic::catch_unwind(|| { 3785 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 3786 thread::sleep(std::time::Duration::new(120, 0)); 3787 3788 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 3789 // added to its kernel command line. 3790 // Let's ssh into it and verify that it's there. If it is it means 3791 // we're in the right guest (The L2 one) because the QEMU L1 guest 3792 // does not have this command line tag. 3793 assert_eq!( 3794 guest 3795 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 3796 .unwrap() 3797 .trim() 3798 .parse::<u32>() 3799 .unwrap_or_default(), 3800 1 3801 ); 3802 3803 // Let's also verify from the second virtio-net device passed to 3804 // the L2 VM. 3805 assert_eq!( 3806 guest 3807 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 3808 .unwrap() 3809 .trim() 3810 .parse::<u32>() 3811 .unwrap_or_default(), 3812 1 3813 ); 3814 3815 // Check the amount of PCI devices appearing in L2 VM. 3816 assert_eq!( 3817 guest 3818 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3819 .unwrap() 3820 .trim() 3821 .parse::<u32>() 3822 .unwrap_or_default(), 3823 8, 3824 ); 3825 3826 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 3827 assert_eq!( 3828 guest 3829 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 3830 .unwrap() 3831 .trim() 3832 .parse::<u32>() 3833 .unwrap_or_default(), 3834 1 3835 ); 3836 3837 // Hotplug an extra virtio-net device through L2 VM. 3838 guest 3839 .ssh_command_l1( 3840 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 3841 ) 3842 .unwrap(); 3843 guest 3844 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 3845 .unwrap(); 3846 let vfio_hotplug_output = guest 3847 .ssh_command_l1( 3848 "sudo /mnt/ch-remote \ 3849 --api-socket=/tmp/ch_api.sock \ 3850 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 3851 ) 3852 .unwrap(); 3853 assert!(vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}")); 3854 3855 thread::sleep(std::time::Duration::new(10, 0)); 3856 3857 // Let's also verify from the third virtio-net device passed to 3858 // the L2 VM. This third device has been hotplugged through the L2 3859 // VM, so this is our way to validate hotplug works for VFIO PCI. 3860 assert_eq!( 3861 guest 3862 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 3863 .unwrap() 3864 .trim() 3865 .parse::<u32>() 3866 .unwrap_or_default(), 3867 1 3868 ); 3869 3870 // Check the amount of PCI devices appearing in L2 VM. 3871 // There should be one more device than before, raising the count 3872 // up to 9 PCI devices. 3873 assert_eq!( 3874 guest 3875 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3876 .unwrap() 3877 .trim() 3878 .parse::<u32>() 3879 .unwrap_or_default(), 3880 9, 3881 ); 3882 3883 // Let's now verify that we can correctly remove the virtio-net 3884 // device through the "remove-device" command responsible for 3885 // unplugging VFIO devices. 3886 guest 3887 .ssh_command_l1( 3888 "sudo /mnt/ch-remote \ 3889 --api-socket=/tmp/ch_api.sock \ 3890 remove-device vfio123", 3891 ) 3892 .unwrap(); 3893 thread::sleep(std::time::Duration::new(10, 0)); 3894 3895 // Check the amount of PCI devices appearing in L2 VM is back down 3896 // to 8 devices. 3897 assert_eq!( 3898 guest 3899 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3900 .unwrap() 3901 .trim() 3902 .parse::<u32>() 3903 .unwrap_or_default(), 3904 8, 3905 ); 3906 3907 // Perform memory hotplug in L2 and validate the memory is showing 3908 // up as expected. In order to check, we will use the virtio-net 3909 // device already passed through L2 as a VFIO device, this will 3910 // verify that VFIO devices are functional with memory hotplug. 3911 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 3912 guest 3913 .ssh_command_l2_1( 3914 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 3915 ) 3916 .unwrap(); 3917 guest 3918 .ssh_command_l1( 3919 "sudo /mnt/ch-remote \ 3920 --api-socket=/tmp/ch_api.sock \ 3921 resize --memory=1073741824", 3922 ) 3923 .unwrap(); 3924 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 3925 }); 3926 3927 let _ = child.kill(); 3928 let output = child.wait_with_output().unwrap(); 3929 3930 cleanup_vfio_network_interfaces(); 3931 3932 handle_child_output(r, &output); 3933 } 3934 3935 #[test] 3936 fn test_direct_kernel_boot_noacpi() { 3937 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3938 let guest = Guest::new(Box::new(focal)); 3939 3940 let kernel_path = direct_kernel_boot_path(); 3941 3942 let mut child = GuestCommand::new(&guest) 3943 .args(&["--cpus", "boot=1"]) 3944 .args(&["--memory", "size=512M"]) 3945 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3946 .args(&[ 3947 "--cmdline", 3948 format!("{} acpi=off", DIRECT_KERNEL_BOOT_CMDLINE).as_str(), 3949 ]) 3950 .default_disks() 3951 .default_net() 3952 .capture_output() 3953 .spawn() 3954 .unwrap(); 3955 3956 let r = std::panic::catch_unwind(|| { 3957 guest.wait_vm_boot(None).unwrap(); 3958 3959 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3960 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3961 }); 3962 3963 let _ = child.kill(); 3964 let output = child.wait_with_output().unwrap(); 3965 3966 handle_child_output(r, &output); 3967 } 3968 3969 #[test] 3970 fn test_reboot() { 3971 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 3972 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3973 3974 vec![Box::new(bionic), Box::new(focal)] 3975 .drain(..) 3976 .for_each(|disk_config| { 3977 let guest = Guest::new(disk_config); 3978 3979 let mut cmd = GuestCommand::new(&guest); 3980 cmd.args(&["--cpus", "boot=1"]) 3981 .args(&["--memory", "size=512M"]) 3982 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3983 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3984 .default_disks() 3985 .default_net() 3986 .capture_output(); 3987 3988 let mut child = cmd.spawn().unwrap(); 3989 3990 let r = std::panic::catch_unwind(|| { 3991 guest.wait_vm_boot(Some(120)).unwrap(); 3992 3993 let fd_count_1 = get_fd_count(child.id()); 3994 guest.reboot_linux(0, Some(120)); 3995 let fd_count_2 = get_fd_count(child.id()); 3996 assert_eq!(fd_count_1, fd_count_2); 3997 3998 guest.ssh_command("sudo shutdown -h now").unwrap(); 3999 }); 4000 4001 let _ = child.wait_timeout(std::time::Duration::from_secs(40)); 4002 let _ = child.kill(); 4003 let output = child.wait_with_output().unwrap(); 4004 handle_child_output(r, &output); 4005 4006 let r = std::panic::catch_unwind(|| { 4007 // Check that the cloud-hypervisor binary actually terminated 4008 assert!(output.status.success()); 4009 }); 4010 4011 handle_child_output(r, &output); 4012 }); 4013 } 4014 4015 #[test] 4016 fn test_virtio_vsock() { 4017 _test_virtio_vsock(false) 4018 } 4019 4020 #[test] 4021 fn test_virtio_vsock_hotplug() { 4022 _test_virtio_vsock(true); 4023 } 4024 4025 #[test] 4026 // Start cloud-hypervisor with no VM parameters, only the API server running. 4027 // From the API: Create a VM, boot it and check that it looks as expected. 4028 fn test_api_create_boot() { 4029 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4030 let guest = Guest::new(Box::new(focal)); 4031 4032 let api_socket = temp_api_path(&guest.tmp_dir); 4033 4034 let mut child = GuestCommand::new(&guest) 4035 .args(&["--api-socket", &api_socket]) 4036 .capture_output() 4037 .spawn() 4038 .unwrap(); 4039 4040 thread::sleep(std::time::Duration::new(1, 0)); 4041 4042 // Verify API server is running 4043 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4044 4045 // Create the VM first 4046 let cpu_count: u8 = 4; 4047 let http_body = guest.api_create_body( 4048 cpu_count, 4049 direct_kernel_boot_path().to_str().unwrap(), 4050 DIRECT_KERNEL_BOOT_CMDLINE, 4051 ); 4052 4053 let temp_config_path = guest.tmp_dir.as_path().join("config"); 4054 std::fs::write(&temp_config_path, http_body).unwrap(); 4055 4056 remote_command( 4057 &api_socket, 4058 "create", 4059 Some(temp_config_path.as_os_str().to_str().unwrap()), 4060 ); 4061 4062 // Then boot it 4063 remote_command(&api_socket, "boot", None); 4064 thread::sleep(std::time::Duration::new(20, 0)); 4065 4066 let r = std::panic::catch_unwind(|| { 4067 // Check that the VM booted as expected 4068 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4069 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4070 }); 4071 4072 let _ = child.kill(); 4073 let output = child.wait_with_output().unwrap(); 4074 4075 handle_child_output(r, &output); 4076 } 4077 4078 #[test] 4079 // Start cloud-hypervisor with no VM parameters, only the API server running. 4080 // From the API: Create a VM, boot it and check it can be shutdown and then 4081 // booted again 4082 fn test_api_shutdown() { 4083 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4084 let guest = Guest::new(Box::new(focal)); 4085 4086 let api_socket = temp_api_path(&guest.tmp_dir); 4087 4088 let mut child = GuestCommand::new(&guest) 4089 .args(&["--api-socket", &api_socket]) 4090 .capture_output() 4091 .spawn() 4092 .unwrap(); 4093 4094 thread::sleep(std::time::Duration::new(1, 0)); 4095 4096 // Verify API server is running 4097 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4098 4099 // Create the VM first 4100 let cpu_count: u8 = 4; 4101 let http_body = guest.api_create_body( 4102 cpu_count, 4103 direct_kernel_boot_path().to_str().unwrap(), 4104 DIRECT_KERNEL_BOOT_CMDLINE, 4105 ); 4106 4107 curl_command( 4108 &api_socket, 4109 "PUT", 4110 "http://localhost/api/v1/vm.create", 4111 Some(&http_body), 4112 ); 4113 4114 // Then boot it 4115 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4116 4117 let r = std::panic::catch_unwind(|| { 4118 guest.wait_vm_boot(None).unwrap(); 4119 4120 // Check that the VM booted as expected 4121 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4122 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4123 4124 // Shutdown without powering off to prevent filesystem corruption 4125 guest.ssh_command("sudo shutdown -H now").unwrap(); 4126 4127 // Then shut it down 4128 curl_command( 4129 &api_socket, 4130 "PUT", 4131 "http://localhost/api/v1/vm.shutdown", 4132 None, 4133 ); 4134 4135 // Then boot it again 4136 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4137 4138 guest.wait_vm_boot(None).unwrap(); 4139 4140 // Check that the VM booted as expected 4141 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4142 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4143 }); 4144 4145 let _ = child.kill(); 4146 let output = child.wait_with_output().unwrap(); 4147 4148 handle_child_output(r, &output); 4149 } 4150 4151 #[test] 4152 // Start cloud-hypervisor with no VM parameters, only the API server running. 4153 // From the API: Create a VM, boot it and check it can be deleted and then recreated 4154 // booted again. 4155 fn test_api_delete() { 4156 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4157 let guest = Guest::new(Box::new(focal)); 4158 4159 let api_socket = temp_api_path(&guest.tmp_dir); 4160 4161 let mut child = GuestCommand::new(&guest) 4162 .args(&["--api-socket", &api_socket]) 4163 .capture_output() 4164 .spawn() 4165 .unwrap(); 4166 4167 thread::sleep(std::time::Duration::new(1, 0)); 4168 4169 // Verify API server is running 4170 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4171 4172 // Create the VM first 4173 let cpu_count: u8 = 4; 4174 let http_body = guest.api_create_body( 4175 cpu_count, 4176 direct_kernel_boot_path().to_str().unwrap(), 4177 DIRECT_KERNEL_BOOT_CMDLINE, 4178 ); 4179 4180 let r = std::panic::catch_unwind(|| { 4181 curl_command( 4182 &api_socket, 4183 "PUT", 4184 "http://localhost/api/v1/vm.create", 4185 Some(&http_body), 4186 ); 4187 4188 // Then boot it 4189 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4190 4191 guest.wait_vm_boot(None).unwrap(); 4192 4193 // Check that the VM booted as expected 4194 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4195 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4196 4197 // Shutdown without powering off to prevent filesystem corruption 4198 guest.ssh_command("sudo shutdown -H now").unwrap(); 4199 4200 // Then delete it 4201 curl_command( 4202 &api_socket, 4203 "PUT", 4204 "http://localhost/api/v1/vm.delete", 4205 None, 4206 ); 4207 4208 curl_command( 4209 &api_socket, 4210 "PUT", 4211 "http://localhost/api/v1/vm.create", 4212 Some(&http_body), 4213 ); 4214 4215 // Then boot it again 4216 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4217 4218 guest.wait_vm_boot(None).unwrap(); 4219 4220 // Check that the VM booted as expected 4221 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4222 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4223 }); 4224 4225 let _ = child.kill(); 4226 let output = child.wait_with_output().unwrap(); 4227 4228 handle_child_output(r, &output); 4229 } 4230 4231 #[test] 4232 // Start cloud-hypervisor with no VM parameters, only the API server running. 4233 // From the API: Create a VM, boot it and check that it looks as expected. 4234 // Then we pause the VM, check that it's no longer available. 4235 // Finally we resume the VM and check that it's available. 4236 fn test_api_pause_resume() { 4237 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4238 let guest = Guest::new(Box::new(focal)); 4239 4240 let api_socket = temp_api_path(&guest.tmp_dir); 4241 4242 let mut child = GuestCommand::new(&guest) 4243 .args(&["--api-socket", &api_socket]) 4244 .capture_output() 4245 .spawn() 4246 .unwrap(); 4247 4248 thread::sleep(std::time::Duration::new(1, 0)); 4249 4250 // Verify API server is running 4251 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4252 4253 // Create the VM first 4254 let cpu_count: u8 = 4; 4255 let http_body = guest.api_create_body( 4256 cpu_count, 4257 direct_kernel_boot_path().to_str().unwrap(), 4258 DIRECT_KERNEL_BOOT_CMDLINE, 4259 ); 4260 curl_command( 4261 &api_socket, 4262 "PUT", 4263 "http://localhost/api/v1/vm.create", 4264 Some(&http_body), 4265 ); 4266 4267 // Then boot it 4268 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4269 thread::sleep(std::time::Duration::new(20, 0)); 4270 4271 let r = std::panic::catch_unwind(|| { 4272 // Check that the VM booted as expected 4273 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4274 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4275 4276 // We now pause the VM 4277 assert!(remote_command(&api_socket, "pause", None)); 4278 4279 // Check pausing again fails 4280 assert!(!remote_command(&api_socket, "pause", None)); 4281 4282 thread::sleep(std::time::Duration::new(2, 0)); 4283 4284 // SSH into the VM should fail 4285 assert!(ssh_command_ip( 4286 "grep -c processor /proc/cpuinfo", 4287 &guest.network.guest_ip, 4288 2, 4289 5 4290 ) 4291 .is_err()); 4292 4293 // Resume the VM 4294 assert!(remote_command(&api_socket, "resume", None)); 4295 4296 // Check resuming again fails 4297 assert!(!remote_command(&api_socket, "resume", None)); 4298 4299 thread::sleep(std::time::Duration::new(2, 0)); 4300 4301 // Now we should be able to SSH back in and get the right number of CPUs 4302 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4303 }); 4304 4305 let _ = child.kill(); 4306 let output = child.wait_with_output().unwrap(); 4307 4308 handle_child_output(r, &output); 4309 } 4310 4311 #[test] 4312 fn test_virtio_iommu() { 4313 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4314 } 4315 4316 #[test] 4317 // We cannot force the software running in the guest to reprogram the BAR 4318 // with some different addresses, but we have a reliable way of testing it 4319 // with a standard Linux kernel. 4320 // By removing a device from the PCI tree, and then rescanning the tree, 4321 // Linux consistently chooses to reorganize the PCI device BARs to other 4322 // locations in the guest address space. 4323 // This test creates a dedicated PCI network device to be checked as being 4324 // properly probed first, then removing it, and adding it again by doing a 4325 // rescan. 4326 fn test_pci_bar_reprogramming() { 4327 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4328 let guest = Guest::new(Box::new(focal)); 4329 4330 #[cfg(target_arch = "x86_64")] 4331 let kernel_path = direct_kernel_boot_path(); 4332 #[cfg(target_arch = "aarch64")] 4333 let kernel_path = edk2_path(); 4334 4335 let mut child = GuestCommand::new(&guest) 4336 .args(&["--cpus", "boot=1"]) 4337 .args(&["--memory", "size=512M"]) 4338 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4339 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4340 .default_disks() 4341 .args(&[ 4342 "--net", 4343 guest.default_net_string().as_str(), 4344 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4345 ]) 4346 .capture_output() 4347 .spawn() 4348 .unwrap(); 4349 4350 let r = std::panic::catch_unwind(|| { 4351 guest.wait_vm_boot(None).unwrap(); 4352 4353 // 2 network interfaces + default localhost ==> 3 interfaces 4354 assert_eq!( 4355 guest 4356 .ssh_command("ip -o link | wc -l") 4357 .unwrap() 4358 .trim() 4359 .parse::<u32>() 4360 .unwrap_or_default(), 4361 3 4362 ); 4363 4364 let init_bar_addr = guest 4365 .ssh_command( 4366 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4367 ) 4368 .unwrap(); 4369 4370 // Remove the PCI device 4371 guest 4372 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4373 .unwrap(); 4374 4375 // Only 1 network interface left + default localhost ==> 2 interfaces 4376 assert_eq!( 4377 guest 4378 .ssh_command("ip -o link | wc -l") 4379 .unwrap() 4380 .trim() 4381 .parse::<u32>() 4382 .unwrap_or_default(), 4383 2 4384 ); 4385 4386 // Remove the PCI device 4387 guest 4388 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4389 .unwrap(); 4390 4391 // Back to 2 network interface + default localhost ==> 3 interfaces 4392 assert_eq!( 4393 guest 4394 .ssh_command("ip -o link | wc -l") 4395 .unwrap() 4396 .trim() 4397 .parse::<u32>() 4398 .unwrap_or_default(), 4399 3 4400 ); 4401 4402 let new_bar_addr = guest 4403 .ssh_command( 4404 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4405 ) 4406 .unwrap(); 4407 4408 // Let's compare the BAR addresses for our virtio-net device. 4409 // They should be different as we expect the BAR reprogramming 4410 // to have happened. 4411 assert_ne!(init_bar_addr, new_bar_addr); 4412 }); 4413 4414 let _ = child.kill(); 4415 let output = child.wait_with_output().unwrap(); 4416 4417 handle_child_output(r, &output); 4418 } 4419 4420 #[test] 4421 fn test_memory_mergeable_off() { 4422 test_memory_mergeable(false) 4423 } 4424 4425 #[test] 4426 #[cfg(target_arch = "x86_64")] 4427 fn test_cpu_hotplug() { 4428 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4429 let guest = Guest::new(Box::new(focal)); 4430 let api_socket = temp_api_path(&guest.tmp_dir); 4431 4432 let kernel_path = direct_kernel_boot_path(); 4433 4434 let mut child = GuestCommand::new(&guest) 4435 .args(&["--cpus", "boot=2,max=4"]) 4436 .args(&["--memory", "size=512M"]) 4437 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4438 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4439 .default_disks() 4440 .default_net() 4441 .args(&["--api-socket", &api_socket]) 4442 .capture_output() 4443 .spawn() 4444 .unwrap(); 4445 4446 let r = std::panic::catch_unwind(|| { 4447 guest.wait_vm_boot(None).unwrap(); 4448 4449 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4450 4451 // Resize the VM 4452 let desired_vcpus = 4; 4453 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4454 4455 guest 4456 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4457 .unwrap(); 4458 guest 4459 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4460 .unwrap(); 4461 thread::sleep(std::time::Duration::new(10, 0)); 4462 assert_eq!( 4463 guest.get_cpu_count().unwrap_or_default(), 4464 u32::from(desired_vcpus) 4465 ); 4466 4467 guest.reboot_linux(0, None); 4468 4469 assert_eq!( 4470 guest.get_cpu_count().unwrap_or_default(), 4471 u32::from(desired_vcpus) 4472 ); 4473 4474 // Resize the VM 4475 let desired_vcpus = 2; 4476 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4477 4478 thread::sleep(std::time::Duration::new(10, 0)); 4479 assert_eq!( 4480 guest.get_cpu_count().unwrap_or_default(), 4481 u32::from(desired_vcpus) 4482 ); 4483 4484 // Resize the VM back up to 4 4485 let desired_vcpus = 4; 4486 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4487 4488 guest 4489 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4490 .unwrap(); 4491 guest 4492 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4493 .unwrap(); 4494 thread::sleep(std::time::Duration::new(10, 0)); 4495 assert_eq!( 4496 guest.get_cpu_count().unwrap_or_default(), 4497 u32::from(desired_vcpus) 4498 ); 4499 }); 4500 4501 let _ = child.kill(); 4502 let output = child.wait_with_output().unwrap(); 4503 4504 handle_child_output(r, &output); 4505 } 4506 4507 #[test] 4508 fn test_memory_hotplug() { 4509 #[cfg(target_arch = "aarch64")] 4510 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4511 #[cfg(target_arch = "x86_64")] 4512 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4513 let focal = UbuntuDiskConfig::new(focal_image); 4514 let guest = Guest::new(Box::new(focal)); 4515 let api_socket = temp_api_path(&guest.tmp_dir); 4516 4517 #[cfg(target_arch = "aarch64")] 4518 let kernel_path = edk2_path(); 4519 #[cfg(target_arch = "x86_64")] 4520 let kernel_path = direct_kernel_boot_path(); 4521 4522 let mut child = GuestCommand::new(&guest) 4523 .args(&["--cpus", "boot=2,max=4"]) 4524 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4525 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4526 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4527 .default_disks() 4528 .default_net() 4529 .args(&["--balloon", "size=0"]) 4530 .args(&["--api-socket", &api_socket]) 4531 .capture_output() 4532 .spawn() 4533 .unwrap(); 4534 4535 let r = std::panic::catch_unwind(|| { 4536 guest.wait_vm_boot(None).unwrap(); 4537 4538 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4539 4540 guest.enable_memory_hotplug(); 4541 4542 // Add RAM to the VM 4543 let desired_ram = 1024 << 20; 4544 resize_command(&api_socket, None, Some(desired_ram), None, None); 4545 4546 thread::sleep(std::time::Duration::new(10, 0)); 4547 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4548 4549 // Use balloon to remove RAM from the VM 4550 let desired_balloon = 512 << 20; 4551 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4552 4553 thread::sleep(std::time::Duration::new(10, 0)); 4554 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4555 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4556 4557 guest.reboot_linux(0, None); 4558 4559 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4560 4561 // Use balloon add RAM to the VM 4562 let desired_balloon = 0; 4563 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4564 4565 thread::sleep(std::time::Duration::new(10, 0)); 4566 4567 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4568 4569 guest.enable_memory_hotplug(); 4570 4571 // Add RAM to the VM 4572 let desired_ram = 2048 << 20; 4573 resize_command(&api_socket, None, Some(desired_ram), None, None); 4574 4575 thread::sleep(std::time::Duration::new(10, 0)); 4576 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4577 4578 // Remove RAM to the VM (only applies after reboot) 4579 let desired_ram = 1024 << 20; 4580 resize_command(&api_socket, None, Some(desired_ram), None, None); 4581 4582 guest.reboot_linux(1, None); 4583 4584 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4585 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4586 }); 4587 4588 let _ = child.kill(); 4589 let output = child.wait_with_output().unwrap(); 4590 4591 handle_child_output(r, &output); 4592 } 4593 4594 #[test] 4595 #[cfg(not(feature = "mshv"))] 4596 fn test_virtio_mem() { 4597 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4598 let guest = Guest::new(Box::new(focal)); 4599 let api_socket = temp_api_path(&guest.tmp_dir); 4600 4601 let kernel_path = direct_kernel_boot_path(); 4602 4603 let mut child = GuestCommand::new(&guest) 4604 .args(&["--cpus", "boot=2,max=4"]) 4605 .args(&[ 4606 "--memory", 4607 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4608 ]) 4609 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4610 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4611 .default_disks() 4612 .default_net() 4613 .args(&["--api-socket", &api_socket]) 4614 .capture_output() 4615 .spawn() 4616 .unwrap(); 4617 4618 let r = std::panic::catch_unwind(|| { 4619 guest.wait_vm_boot(None).unwrap(); 4620 4621 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4622 4623 guest.enable_memory_hotplug(); 4624 4625 // Add RAM to the VM 4626 let desired_ram = 1024 << 20; 4627 resize_command(&api_socket, None, Some(desired_ram), None, None); 4628 4629 thread::sleep(std::time::Duration::new(10, 0)); 4630 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4631 4632 // Add RAM to the VM 4633 let desired_ram = 2048 << 20; 4634 resize_command(&api_socket, None, Some(desired_ram), None, None); 4635 4636 thread::sleep(std::time::Duration::new(10, 0)); 4637 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4638 4639 // Remove RAM from the VM 4640 let desired_ram = 1024 << 20; 4641 resize_command(&api_socket, None, Some(desired_ram), None, None); 4642 4643 thread::sleep(std::time::Duration::new(10, 0)); 4644 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4645 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4646 4647 guest.reboot_linux(0, None); 4648 4649 // Check the amount of memory after reboot is 1GiB 4650 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4651 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4652 4653 // Check we can still resize to 512MiB 4654 let desired_ram = 512 << 20; 4655 resize_command(&api_socket, None, Some(desired_ram), None, None); 4656 thread::sleep(std::time::Duration::new(10, 0)); 4657 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4658 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4659 }); 4660 4661 let _ = child.kill(); 4662 let output = child.wait_with_output().unwrap(); 4663 4664 handle_child_output(r, &output); 4665 } 4666 4667 #[test] 4668 #[cfg(target_arch = "x86_64")] 4669 #[cfg(not(feature = "mshv"))] 4670 // Test both vCPU and memory resizing together 4671 fn test_resize() { 4672 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4673 let guest = Guest::new(Box::new(focal)); 4674 let api_socket = temp_api_path(&guest.tmp_dir); 4675 4676 let kernel_path = direct_kernel_boot_path(); 4677 4678 let mut child = GuestCommand::new(&guest) 4679 .args(&["--cpus", "boot=2,max=4"]) 4680 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4681 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4682 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4683 .default_disks() 4684 .default_net() 4685 .args(&["--api-socket", &api_socket]) 4686 .capture_output() 4687 .spawn() 4688 .unwrap(); 4689 4690 let r = std::panic::catch_unwind(|| { 4691 guest.wait_vm_boot(None).unwrap(); 4692 4693 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4694 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4695 4696 guest.enable_memory_hotplug(); 4697 4698 // Resize the VM 4699 let desired_vcpus = 4; 4700 let desired_ram = 1024 << 20; 4701 resize_command( 4702 &api_socket, 4703 Some(desired_vcpus), 4704 Some(desired_ram), 4705 None, 4706 None, 4707 ); 4708 4709 guest 4710 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4711 .unwrap(); 4712 guest 4713 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4714 .unwrap(); 4715 thread::sleep(std::time::Duration::new(10, 0)); 4716 assert_eq!( 4717 guest.get_cpu_count().unwrap_or_default(), 4718 u32::from(desired_vcpus) 4719 ); 4720 4721 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4722 }); 4723 4724 let _ = child.kill(); 4725 let output = child.wait_with_output().unwrap(); 4726 4727 handle_child_output(r, &output); 4728 } 4729 4730 #[test] 4731 fn test_memory_overhead() { 4732 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4733 let guest = Guest::new(Box::new(focal)); 4734 4735 let kernel_path = direct_kernel_boot_path(); 4736 4737 let guest_memory_size_kb = 512 * 1024; 4738 4739 let mut child = GuestCommand::new(&guest) 4740 .args(&["--cpus", "boot=1"]) 4741 .args(&[ 4742 "--memory", 4743 format!("size={}K", guest_memory_size_kb).as_str(), 4744 ]) 4745 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4746 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4747 .default_disks() 4748 .capture_output() 4749 .spawn() 4750 .unwrap(); 4751 4752 thread::sleep(std::time::Duration::new(20, 0)); 4753 4754 let r = std::panic::catch_unwind(|| { 4755 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 4756 eprintln!( 4757 "Guest memory overhead: {} vs {}", 4758 overhead, MAXIMUM_VMM_OVERHEAD_KB 4759 ); 4760 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 4761 }); 4762 4763 let _ = child.kill(); 4764 let output = child.wait_with_output().unwrap(); 4765 4766 handle_child_output(r, &output); 4767 } 4768 4769 #[test] 4770 fn test_disk_hotplug() { 4771 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4772 let guest = Guest::new(Box::new(focal)); 4773 4774 #[cfg(target_arch = "x86_64")] 4775 let kernel_path = direct_kernel_boot_path(); 4776 #[cfg(target_arch = "aarch64")] 4777 let kernel_path = edk2_path(); 4778 4779 let api_socket = temp_api_path(&guest.tmp_dir); 4780 4781 let mut child = GuestCommand::new(&guest) 4782 .args(&["--api-socket", &api_socket]) 4783 .args(&["--cpus", "boot=1"]) 4784 .args(&["--memory", "size=512M"]) 4785 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4786 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4787 .default_disks() 4788 .default_net() 4789 .capture_output() 4790 .spawn() 4791 .unwrap(); 4792 4793 let r = std::panic::catch_unwind(|| { 4794 guest.wait_vm_boot(None).unwrap(); 4795 4796 // Check /dev/vdc is not there 4797 assert_eq!( 4798 guest 4799 .ssh_command("lsblk | grep -c vdc.*16M || true") 4800 .unwrap() 4801 .trim() 4802 .parse::<u32>() 4803 .unwrap_or(1), 4804 0 4805 ); 4806 4807 // Now let's add the extra disk. 4808 let mut blk_file_path = dirs::home_dir().unwrap(); 4809 blk_file_path.push("workloads"); 4810 blk_file_path.push("blk.img"); 4811 let (cmd_success, cmd_output) = remote_command_w_output( 4812 &api_socket, 4813 "add-disk", 4814 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4815 ); 4816 assert!(cmd_success); 4817 assert!(String::from_utf8_lossy(&cmd_output) 4818 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4819 4820 thread::sleep(std::time::Duration::new(10, 0)); 4821 4822 // Check that /dev/vdc exists and the block size is 16M. 4823 assert_eq!( 4824 guest 4825 .ssh_command("lsblk | grep vdc | grep -c 16M") 4826 .unwrap() 4827 .trim() 4828 .parse::<u32>() 4829 .unwrap_or_default(), 4830 1 4831 ); 4832 // And check the block device can be read. 4833 guest 4834 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4835 .unwrap(); 4836 4837 // Let's remove it the extra disk. 4838 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4839 thread::sleep(std::time::Duration::new(5, 0)); 4840 // And check /dev/vdc is not there 4841 assert_eq!( 4842 guest 4843 .ssh_command("lsblk | grep -c vdc.*16M || true") 4844 .unwrap() 4845 .trim() 4846 .parse::<u32>() 4847 .unwrap_or(1), 4848 0 4849 ); 4850 4851 // And add it back to validate unplug did work correctly. 4852 let (cmd_success, cmd_output) = remote_command_w_output( 4853 &api_socket, 4854 "add-disk", 4855 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4856 ); 4857 assert!(cmd_success); 4858 assert!(String::from_utf8_lossy(&cmd_output) 4859 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4860 4861 thread::sleep(std::time::Duration::new(10, 0)); 4862 4863 // Check that /dev/vdc exists and the block size is 16M. 4864 assert_eq!( 4865 guest 4866 .ssh_command("lsblk | grep vdc | grep -c 16M") 4867 .unwrap() 4868 .trim() 4869 .parse::<u32>() 4870 .unwrap_or_default(), 4871 1 4872 ); 4873 // And check the block device can be read. 4874 guest 4875 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4876 .unwrap(); 4877 4878 // Reboot the VM. 4879 guest.reboot_linux(0, None); 4880 4881 // Check still there after reboot 4882 assert_eq!( 4883 guest 4884 .ssh_command("lsblk | grep vdc | grep -c 16M") 4885 .unwrap() 4886 .trim() 4887 .parse::<u32>() 4888 .unwrap_or_default(), 4889 1 4890 ); 4891 4892 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4893 4894 thread::sleep(std::time::Duration::new(20, 0)); 4895 4896 // Check device has gone away 4897 assert_eq!( 4898 guest 4899 .ssh_command("lsblk | grep -c vdc.*16M || true") 4900 .unwrap() 4901 .trim() 4902 .parse::<u32>() 4903 .unwrap_or(1), 4904 0 4905 ); 4906 4907 guest.reboot_linux(1, None); 4908 4909 // Check device still absent 4910 assert_eq!( 4911 guest 4912 .ssh_command("lsblk | grep -c vdc.*16M || true") 4913 .unwrap() 4914 .trim() 4915 .parse::<u32>() 4916 .unwrap_or(1), 4917 0 4918 ); 4919 }); 4920 4921 let _ = child.kill(); 4922 let output = child.wait_with_output().unwrap(); 4923 4924 handle_child_output(r, &output); 4925 } 4926 4927 #[test] 4928 #[cfg(target_arch = "x86_64")] 4929 fn test_virtio_block_topology() { 4930 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4931 let guest = Guest::new(Box::new(focal)); 4932 4933 let kernel_path = direct_kernel_boot_path(); 4934 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 4935 4936 Command::new("qemu-img") 4937 .args(&[ 4938 "create", 4939 "-f", 4940 "raw", 4941 test_disk_path.to_str().unwrap(), 4942 "16M", 4943 ]) 4944 .output() 4945 .expect("qemu-img command failed"); 4946 let out = Command::new("losetup") 4947 .args(&[ 4948 "--show", 4949 "--find", 4950 "--sector-size=4096", 4951 test_disk_path.to_str().unwrap(), 4952 ]) 4953 .output() 4954 .expect("failed to create loop device") 4955 .stdout; 4956 let _tmp = String::from_utf8_lossy(&out); 4957 let loop_dev = _tmp.trim(); 4958 4959 let mut child = GuestCommand::new(&guest) 4960 .args(&["--cpus", "boot=1"]) 4961 .args(&["--memory", "size=512M"]) 4962 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4963 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4964 .args(&[ 4965 "--disk", 4966 format!( 4967 "path={}", 4968 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4969 ) 4970 .as_str(), 4971 format!( 4972 "path={}", 4973 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4974 ) 4975 .as_str(), 4976 format!("path={}", loop_dev).as_str(), 4977 ]) 4978 .default_net() 4979 .capture_output() 4980 .spawn() 4981 .unwrap(); 4982 4983 let r = std::panic::catch_unwind(|| { 4984 guest.wait_vm_boot(None).unwrap(); 4985 4986 // MIN-IO column 4987 assert_eq!( 4988 guest 4989 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 4990 .unwrap() 4991 .trim() 4992 .parse::<u32>() 4993 .unwrap_or_default(), 4994 4096 4995 ); 4996 // PHY-SEC column 4997 assert_eq!( 4998 guest 4999 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5000 .unwrap() 5001 .trim() 5002 .parse::<u32>() 5003 .unwrap_or_default(), 5004 4096 5005 ); 5006 // LOG-SEC column 5007 assert_eq!( 5008 guest 5009 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5010 .unwrap() 5011 .trim() 5012 .parse::<u32>() 5013 .unwrap_or_default(), 5014 4096 5015 ); 5016 }); 5017 5018 let _ = child.kill(); 5019 let output = child.wait_with_output().unwrap(); 5020 5021 handle_child_output(r, &output); 5022 5023 Command::new("losetup") 5024 .args(&["-d", loop_dev]) 5025 .output() 5026 .expect("loop device not found"); 5027 } 5028 5029 #[test] 5030 fn test_virtio_balloon_deflate_on_oom() { 5031 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5032 let guest = Guest::new(Box::new(focal)); 5033 5034 let kernel_path = direct_kernel_boot_path(); 5035 5036 let api_socket = temp_api_path(&guest.tmp_dir); 5037 5038 //Let's start a 4G guest with balloon occupied 2G memory 5039 let mut child = GuestCommand::new(&guest) 5040 .args(&["--api-socket", &api_socket]) 5041 .args(&["--cpus", "boot=1"]) 5042 .args(&["--memory", "size=4G"]) 5043 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5044 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5045 .args(&["--balloon", "size=2G,deflate_on_oom=on"]) 5046 .default_disks() 5047 .default_net() 5048 .capture_output() 5049 .spawn() 5050 .unwrap(); 5051 5052 let r = std::panic::catch_unwind(|| { 5053 guest.wait_vm_boot(None).unwrap(); 5054 5055 // Wait for balloon memory's initialization and check its size. 5056 // The virtio-balloon driver might take a few seconds to report the 5057 // balloon effective size back to the VMM. 5058 thread::sleep(std::time::Duration::new(20, 0)); 5059 5060 let orig_balloon = balloon_size(&api_socket); 5061 println!("The original balloon memory size is {} bytes", orig_balloon); 5062 assert!(orig_balloon == 2147483648); 5063 5064 // Two steps to verify if the 'deflate_on_oom' parameter works. 5065 // 1st: run a command to trigger an OOM in the guest. 5066 guest 5067 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5068 .unwrap(); 5069 5070 // Give some time for the OOM to happen in the guest and be reported 5071 // back to the host. 5072 thread::sleep(std::time::Duration::new(20, 0)); 5073 5074 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5075 let deflated_balloon = balloon_size(&api_socket); 5076 println!( 5077 "After deflating, balloon memory size is {} bytes", 5078 deflated_balloon 5079 ); 5080 // Verify the balloon size deflated 5081 assert!(deflated_balloon < 2147483648); 5082 }); 5083 5084 let _ = child.kill(); 5085 let output = child.wait_with_output().unwrap(); 5086 5087 handle_child_output(r, &output); 5088 } 5089 5090 #[test] 5091 fn test_virtio_balloon_free_page_reporting() { 5092 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5093 let guest = Guest::new(Box::new(focal)); 5094 5095 //Let's start a 4G guest with balloon occupied 2G memory 5096 let mut child = GuestCommand::new(&guest) 5097 .args(&["--cpus", "boot=1"]) 5098 .args(&["--memory", "size=4G"]) 5099 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5100 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5101 .args(&["--balloon", "size=0,free_page_reporting=on"]) 5102 .default_disks() 5103 .default_net() 5104 .capture_output() 5105 .spawn() 5106 .unwrap(); 5107 5108 let pid = child.id(); 5109 let r = std::panic::catch_unwind(|| { 5110 guest.wait_vm_boot(None).unwrap(); 5111 5112 // Check the initial RSS is less than 1GiB 5113 let rss = process_rss_kib(pid); 5114 println!("RSS {} < 1048576", rss); 5115 assert!(rss < 1048576); 5116 5117 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5118 // seconds 5119 let guest_ip = guest.network.guest_ip.clone(); 5120 thread::spawn(move || { 5121 ssh_command_ip( 5122 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5123 &guest_ip, 5124 DEFAULT_SSH_RETRIES, 5125 DEFAULT_SSH_TIMEOUT, 5126 ) 5127 .unwrap(); 5128 }); 5129 5130 // Wait for 50 seconds to make sure the stress command is consuming 5131 // the expected amount of memory. 5132 thread::sleep(std::time::Duration::new(50, 0)); 5133 let rss = process_rss_kib(pid); 5134 println!("RSS {} >= 2097152", rss); 5135 assert!(rss >= 2097152); 5136 5137 // Wait for an extra minute to make sure the stress command has 5138 // completed and that the guest reported the free pages to the VMM 5139 // through the virtio-balloon device. We expect the RSS to be under 5140 // 2GiB. 5141 thread::sleep(std::time::Duration::new(60, 0)); 5142 let rss = process_rss_kib(pid); 5143 println!("RSS {} < 2097152", rss); 5144 assert!(rss < 2097152); 5145 }); 5146 5147 let _ = child.kill(); 5148 let output = child.wait_with_output().unwrap(); 5149 5150 handle_child_output(r, &output); 5151 } 5152 5153 #[test] 5154 fn test_pmem_hotplug() { 5155 _test_pmem_hotplug(None) 5156 } 5157 5158 #[test] 5159 fn test_pmem_multi_segment_hotplug() { 5160 _test_pmem_hotplug(Some(15)) 5161 } 5162 5163 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5164 #[cfg(target_arch = "aarch64")] 5165 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5166 #[cfg(target_arch = "x86_64")] 5167 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5168 let focal = UbuntuDiskConfig::new(focal_image); 5169 let guest = Guest::new(Box::new(focal)); 5170 5171 #[cfg(target_arch = "x86_64")] 5172 let kernel_path = direct_kernel_boot_path(); 5173 #[cfg(target_arch = "aarch64")] 5174 let kernel_path = edk2_path(); 5175 5176 let api_socket = temp_api_path(&guest.tmp_dir); 5177 5178 let mut cmd = GuestCommand::new(&guest); 5179 5180 cmd.args(&["--api-socket", &api_socket]) 5181 .args(&["--cpus", "boot=1"]) 5182 .args(&["--memory", "size=512M"]) 5183 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5184 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5185 .default_disks() 5186 .default_net() 5187 .capture_output(); 5188 5189 if pci_segment.is_some() { 5190 cmd.args(&["--platform", "num_pci_segments=16"]); 5191 } 5192 5193 let mut child = cmd.spawn().unwrap(); 5194 5195 let r = std::panic::catch_unwind(|| { 5196 guest.wait_vm_boot(None).unwrap(); 5197 5198 // Check /dev/pmem0 is not there 5199 assert_eq!( 5200 guest 5201 .ssh_command("lsblk | grep -c pmem0 || true") 5202 .unwrap() 5203 .trim() 5204 .parse::<u32>() 5205 .unwrap_or(1), 5206 0 5207 ); 5208 5209 let pmem_temp_file = TempFile::new().unwrap(); 5210 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5211 let (cmd_success, cmd_output) = remote_command_w_output( 5212 &api_socket, 5213 "add-pmem", 5214 Some(&format!( 5215 "file={},id=test0{}", 5216 pmem_temp_file.as_path().to_str().unwrap(), 5217 if let Some(pci_segment) = pci_segment { 5218 format!(",pci_segment={}", pci_segment) 5219 } else { 5220 "".to_owned() 5221 } 5222 )), 5223 ); 5224 assert!(cmd_success); 5225 if let Some(pci_segment) = pci_segment { 5226 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5227 "{{\"id\":\"test0\",\"bdf\":\"{:04x}:00:01.0\"}}", 5228 pci_segment 5229 ))); 5230 } else { 5231 assert!(String::from_utf8_lossy(&cmd_output) 5232 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5233 } 5234 5235 // Check that /dev/pmem0 exists and the block size is 128M 5236 assert_eq!( 5237 guest 5238 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5239 .unwrap() 5240 .trim() 5241 .parse::<u32>() 5242 .unwrap_or_default(), 5243 1 5244 ); 5245 5246 guest.reboot_linux(0, None); 5247 5248 // Check still there after reboot 5249 assert_eq!( 5250 guest 5251 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5252 .unwrap() 5253 .trim() 5254 .parse::<u32>() 5255 .unwrap_or_default(), 5256 1 5257 ); 5258 5259 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5260 5261 thread::sleep(std::time::Duration::new(20, 0)); 5262 5263 // Check device has gone away 5264 assert_eq!( 5265 guest 5266 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5267 .unwrap() 5268 .trim() 5269 .parse::<u32>() 5270 .unwrap_or(1), 5271 0 5272 ); 5273 5274 guest.reboot_linux(1, None); 5275 5276 // Check still absent after reboot 5277 assert_eq!( 5278 guest 5279 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5280 .unwrap() 5281 .trim() 5282 .parse::<u32>() 5283 .unwrap_or(1), 5284 0 5285 ); 5286 }); 5287 5288 let _ = child.kill(); 5289 let output = child.wait_with_output().unwrap(); 5290 5291 handle_child_output(r, &output); 5292 } 5293 5294 #[test] 5295 fn test_net_hotplug() { 5296 _test_net_hotplug(None) 5297 } 5298 5299 #[test] 5300 fn test_net_multi_segment_hotplug() { 5301 _test_net_hotplug(Some(15)) 5302 } 5303 5304 fn _test_net_hotplug(pci_segment: Option<u16>) { 5305 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5306 let guest = Guest::new(Box::new(focal)); 5307 5308 #[cfg(target_arch = "x86_64")] 5309 let kernel_path = direct_kernel_boot_path(); 5310 #[cfg(target_arch = "aarch64")] 5311 let kernel_path = edk2_path(); 5312 5313 let api_socket = temp_api_path(&guest.tmp_dir); 5314 5315 // Boot without network 5316 let mut cmd = GuestCommand::new(&guest); 5317 5318 cmd.args(&["--api-socket", &api_socket]) 5319 .args(&["--cpus", "boot=1"]) 5320 .args(&["--memory", "size=512M"]) 5321 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5322 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5323 .default_disks() 5324 .capture_output(); 5325 5326 if pci_segment.is_some() { 5327 cmd.args(&["--platform", "num_pci_segments=16"]); 5328 } 5329 5330 let mut child = cmd.spawn().unwrap(); 5331 5332 thread::sleep(std::time::Duration::new(20, 0)); 5333 5334 let r = std::panic::catch_unwind(|| { 5335 // Add network 5336 let (cmd_success, cmd_output) = remote_command_w_output( 5337 &api_socket, 5338 "add-net", 5339 Some( 5340 format!( 5341 "{}{},id=test0", 5342 guest.default_net_string(), 5343 if let Some(pci_segment) = pci_segment { 5344 format!(",pci_segment={}", pci_segment) 5345 } else { 5346 "".to_owned() 5347 } 5348 ) 5349 .as_str(), 5350 ), 5351 ); 5352 assert!(cmd_success); 5353 5354 if let Some(pci_segment) = pci_segment { 5355 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5356 "{{\"id\":\"test0\",\"bdf\":\"{:04x}:00:01.0\"}}", 5357 pci_segment 5358 ))); 5359 } else { 5360 assert!(String::from_utf8_lossy(&cmd_output) 5361 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5362 } 5363 5364 thread::sleep(std::time::Duration::new(5, 0)); 5365 5366 // 1 network interfaces + default localhost ==> 2 interfaces 5367 assert_eq!( 5368 guest 5369 .ssh_command("ip -o link | wc -l") 5370 .unwrap() 5371 .trim() 5372 .parse::<u32>() 5373 .unwrap_or_default(), 5374 2 5375 ); 5376 5377 // Remove network 5378 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5379 thread::sleep(std::time::Duration::new(5, 0)); 5380 5381 let (cmd_success, cmd_output) = remote_command_w_output( 5382 &api_socket, 5383 "add-net", 5384 Some( 5385 format!( 5386 "{}{},id=test1", 5387 guest.default_net_string(), 5388 if let Some(pci_segment) = pci_segment { 5389 format!(",pci_segment={}", pci_segment) 5390 } else { 5391 "".to_owned() 5392 } 5393 ) 5394 .as_str(), 5395 ), 5396 ); 5397 assert!(cmd_success); 5398 5399 if let Some(pci_segment) = pci_segment { 5400 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5401 "{{\"id\":\"test1\",\"bdf\":\"{:04x}:00:01.0\"}}", 5402 pci_segment 5403 ))); 5404 } else { 5405 assert!(String::from_utf8_lossy(&cmd_output) 5406 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5407 } 5408 5409 thread::sleep(std::time::Duration::new(5, 0)); 5410 5411 // 1 network interfaces + default localhost ==> 2 interfaces 5412 assert_eq!( 5413 guest 5414 .ssh_command("ip -o link | wc -l") 5415 .unwrap() 5416 .trim() 5417 .parse::<u32>() 5418 .unwrap_or_default(), 5419 2 5420 ); 5421 5422 guest.reboot_linux(0, None); 5423 5424 // Check still there after reboot 5425 // 1 network interfaces + default localhost ==> 2 interfaces 5426 assert_eq!( 5427 guest 5428 .ssh_command("ip -o link | wc -l") 5429 .unwrap() 5430 .trim() 5431 .parse::<u32>() 5432 .unwrap_or_default(), 5433 2 5434 ); 5435 }); 5436 5437 let _ = child.kill(); 5438 let output = child.wait_with_output().unwrap(); 5439 5440 handle_child_output(r, &output); 5441 } 5442 5443 #[test] 5444 fn test_initramfs() { 5445 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5446 let guest = Guest::new(Box::new(focal)); 5447 let mut workload_path = dirs::home_dir().unwrap(); 5448 workload_path.push("workloads"); 5449 5450 #[cfg(target_arch = "x86_64")] 5451 let mut kernels = vec![direct_kernel_boot_path()]; 5452 #[cfg(target_arch = "aarch64")] 5453 let kernels = vec![direct_kernel_boot_path()]; 5454 5455 #[cfg(target_arch = "x86_64")] 5456 { 5457 let mut pvh_kernel_path = workload_path.clone(); 5458 pvh_kernel_path.push("vmlinux"); 5459 kernels.push(pvh_kernel_path); 5460 } 5461 5462 let mut initramfs_path = workload_path; 5463 initramfs_path.push("alpine_initramfs.img"); 5464 5465 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5466 let cmdline = format!("console=hvc0 quiet TEST_STRING={}", test_string); 5467 5468 kernels.iter().for_each(|k_path| { 5469 let mut child = GuestCommand::new(&guest) 5470 .args(&["--kernel", k_path.to_str().unwrap()]) 5471 .args(&["--initramfs", initramfs_path.to_str().unwrap()]) 5472 .args(&["--cmdline", &cmdline]) 5473 .capture_output() 5474 .spawn() 5475 .unwrap(); 5476 5477 thread::sleep(std::time::Duration::new(20, 0)); 5478 5479 let _ = child.kill(); 5480 let output = child.wait_with_output().unwrap(); 5481 5482 let r = std::panic::catch_unwind(|| { 5483 let s = String::from_utf8_lossy(&output.stdout); 5484 5485 assert_ne!(s.lines().position(|line| line == test_string), None); 5486 }); 5487 5488 handle_child_output(r, &output); 5489 }); 5490 } 5491 5492 // One thing to note about this test. The virtio-net device is heavily used 5493 // through each ssh command. There's no need to perform a dedicated test to 5494 // verify the migration went well for virtio-net. 5495 #[test] 5496 fn test_snapshot_restore() { 5497 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5498 let guest = Guest::new(Box::new(focal)); 5499 let kernel_path = direct_kernel_boot_path(); 5500 5501 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 5502 5503 let net_id = "net123"; 5504 let net_params = format!( 5505 "id={},tap=,mac={},ip={},mask=255.255.255.0", 5506 net_id, guest.network.guest_mac, guest.network.host_ip 5507 ); 5508 5509 let cloudinit_params = format!( 5510 "path={},iommu=on", 5511 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5512 ); 5513 5514 let socket = temp_vsock_path(&guest.tmp_dir); 5515 let event_path = temp_event_monitor_path(&guest.tmp_dir); 5516 5517 let mut child = GuestCommand::new(&guest) 5518 .args(&["--api-socket", &api_socket_source]) 5519 .args(&["--event-monitor", format!("path={}", event_path).as_str()]) 5520 .args(&["--cpus", "boot=4"]) 5521 .args(&[ 5522 "--memory", 5523 "size=4G,hotplug_method=virtio-mem,hotplug_size=32G", 5524 ]) 5525 .args(&["--balloon", "size=0"]) 5526 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5527 .args(&[ 5528 "--disk", 5529 format!( 5530 "path={}", 5531 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5532 ) 5533 .as_str(), 5534 cloudinit_params.as_str(), 5535 ]) 5536 .args(&["--net", net_params.as_str()]) 5537 .args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]) 5538 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5539 .capture_output() 5540 .spawn() 5541 .unwrap(); 5542 5543 let console_text = String::from("On a branch floating down river a cricket, singing."); 5544 // Create the snapshot directory 5545 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 5546 5547 let r = std::panic::catch_unwind(|| { 5548 guest.wait_vm_boot(None).unwrap(); 5549 5550 // Check the number of vCPUs 5551 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5552 // Check the guest RAM 5553 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5554 // Increase guest RAM with virtio-mem 5555 resize_command( 5556 &api_socket_source, 5557 None, 5558 Some(6 << 30), 5559 None, 5560 Some(&event_path), 5561 ); 5562 thread::sleep(std::time::Duration::new(5, 0)); 5563 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5564 // Use balloon to remove RAM from the VM 5565 resize_command( 5566 &api_socket_source, 5567 None, 5568 None, 5569 Some(1 << 30), 5570 Some(&event_path), 5571 ); 5572 thread::sleep(std::time::Duration::new(5, 0)); 5573 let total_memory = guest.get_total_memory().unwrap_or_default(); 5574 assert!(total_memory > 4_800_000); 5575 assert!(total_memory < 5_760_000); 5576 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 5577 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5578 5579 // x86_64: We check that removing and adding back the virtio-net device 5580 // does not break the snapshot/restore support for virtio-pci. 5581 // This is an important thing to test as the hotplug will 5582 // trigger a PCI BAR reprogramming, which is a good way of 5583 // checking if the stored resources are correctly restored. 5584 // Unplug the virtio-net device 5585 // AArch64: Device hotplug is currently not supported, skipping here. 5586 #[cfg(target_arch = "x86_64")] 5587 { 5588 assert!(remote_command( 5589 &api_socket_source, 5590 "remove-device", 5591 Some(net_id), 5592 )); 5593 thread::sleep(std::time::Duration::new(10, 0)); 5594 let latest_events = [&MetaEvent { 5595 event: "device-removed".to_string(), 5596 device_id: Some(net_id.to_string()), 5597 }]; 5598 assert!(check_latest_events_exact(&latest_events, &event_path)); 5599 5600 // Plug the virtio-net device again 5601 assert!(remote_command( 5602 &api_socket_source, 5603 "add-net", 5604 Some(net_params.as_str()), 5605 )); 5606 thread::sleep(std::time::Duration::new(10, 0)); 5607 } 5608 5609 // Pause the VM 5610 assert!(remote_command(&api_socket_source, "pause", None)); 5611 let latest_events = [ 5612 &MetaEvent { 5613 event: "pausing".to_string(), 5614 device_id: None, 5615 }, 5616 &MetaEvent { 5617 event: "paused".to_string(), 5618 device_id: None, 5619 }, 5620 ]; 5621 assert!(check_latest_events_exact(&latest_events, &event_path)); 5622 5623 // Take a snapshot from the VM 5624 assert!(remote_command( 5625 &api_socket_source, 5626 "snapshot", 5627 Some(format!("file://{}", snapshot_dir).as_str()), 5628 )); 5629 5630 // Wait to make sure the snapshot is completed 5631 thread::sleep(std::time::Duration::new(10, 0)); 5632 5633 let latest_events = [ 5634 &MetaEvent { 5635 event: "snapshotting".to_string(), 5636 device_id: None, 5637 }, 5638 &MetaEvent { 5639 event: "snapshotted".to_string(), 5640 device_id: None, 5641 }, 5642 ]; 5643 assert!(check_latest_events_exact(&latest_events, &event_path)); 5644 }); 5645 5646 // Shutdown the source VM and check console output 5647 let _ = child.kill(); 5648 let output = child.wait_with_output().unwrap(); 5649 handle_child_output(r, &output); 5650 5651 let r = std::panic::catch_unwind(|| { 5652 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5653 }); 5654 5655 handle_child_output(r, &output); 5656 5657 // Remove the vsock socket file. 5658 Command::new("rm") 5659 .arg("-f") 5660 .arg(socket.as_str()) 5661 .output() 5662 .unwrap(); 5663 5664 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 5665 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 5666 5667 // Restore the VM from the snapshot 5668 let mut child = GuestCommand::new(&guest) 5669 .args(&["--api-socket", &api_socket_restored]) 5670 .args(&[ 5671 "--event-monitor", 5672 format!("path={}", event_path_restored).as_str(), 5673 ]) 5674 .args(&[ 5675 "--restore", 5676 format!("source_url=file://{}", snapshot_dir).as_str(), 5677 ]) 5678 .capture_output() 5679 .spawn() 5680 .unwrap(); 5681 5682 // Wait for the VM to be restored 5683 thread::sleep(std::time::Duration::new(10, 0)); 5684 let expected_events = [ 5685 &MetaEvent { 5686 event: "starting".to_string(), 5687 device_id: None, 5688 }, 5689 &MetaEvent { 5690 event: "restoring".to_string(), 5691 device_id: None, 5692 }, 5693 ]; 5694 assert!(check_sequential_events_exact( 5695 &expected_events, 5696 &event_path_restored 5697 )); 5698 let latest_events = [&MetaEvent { 5699 event: "restored".to_string(), 5700 device_id: None, 5701 }]; 5702 assert!(check_latest_events_exact( 5703 &latest_events, 5704 &event_path_restored 5705 )); 5706 5707 let r = std::panic::catch_unwind(|| { 5708 // Resume the VM 5709 assert!(remote_command(&api_socket_restored, "resume", None)); 5710 let latest_events = [ 5711 &MetaEvent { 5712 event: "resuming".to_string(), 5713 device_id: None, 5714 }, 5715 &MetaEvent { 5716 event: "resumed".to_string(), 5717 device_id: None, 5718 }, 5719 ]; 5720 assert!(check_latest_events_exact( 5721 &latest_events, 5722 &event_path_restored 5723 )); 5724 5725 // Perform same checks to validate VM has been properly restored 5726 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5727 let total_memory = guest.get_total_memory().unwrap_or_default(); 5728 assert!(total_memory > 4_800_000); 5729 assert!(total_memory < 5_760_000); 5730 // Deflate balloon to restore entire RAM to the VM 5731 resize_command(&api_socket_restored, None, None, Some(0), None); 5732 thread::sleep(std::time::Duration::new(5, 0)); 5733 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5734 // Decrease guest RAM with virtio-mem 5735 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 5736 thread::sleep(std::time::Duration::new(5, 0)); 5737 let total_memory = guest.get_total_memory().unwrap_or_default(); 5738 assert!(total_memory > 4_800_000); 5739 assert!(total_memory < 5_760_000); 5740 5741 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5742 }); 5743 // Shutdown the target VM and check console output 5744 let _ = child.kill(); 5745 let output = child.wait_with_output().unwrap(); 5746 handle_child_output(r, &output); 5747 5748 let r = std::panic::catch_unwind(|| { 5749 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5750 }); 5751 5752 handle_child_output(r, &output); 5753 } 5754 5755 #[test] 5756 fn test_counters() { 5757 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5758 let guest = Guest::new(Box::new(focal)); 5759 let api_socket = temp_api_path(&guest.tmp_dir); 5760 5761 let mut cmd = GuestCommand::new(&guest); 5762 cmd.args(&["--cpus", "boot=1"]) 5763 .args(&["--memory", "size=512M"]) 5764 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5765 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5766 .default_disks() 5767 .args(&["--net", guest.default_net_string().as_str()]) 5768 .args(&["--api-socket", &api_socket]) 5769 .capture_output(); 5770 5771 let mut child = cmd.spawn().unwrap(); 5772 5773 let r = std::panic::catch_unwind(|| { 5774 guest.wait_vm_boot(None).unwrap(); 5775 5776 let orig_counters = get_counters(&api_socket); 5777 guest 5778 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 5779 .unwrap(); 5780 5781 let new_counters = get_counters(&api_socket); 5782 5783 // Check that all the counters have increased 5784 assert!(new_counters > orig_counters); 5785 }); 5786 5787 let _ = child.kill(); 5788 let output = child.wait_with_output().unwrap(); 5789 5790 handle_child_output(r, &output); 5791 } 5792 5793 #[test] 5794 #[cfg(feature = "guest_debug")] 5795 fn test_coredump() { 5796 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5797 let guest = Guest::new(Box::new(focal)); 5798 let api_socket = temp_api_path(&guest.tmp_dir); 5799 5800 let mut cmd = GuestCommand::new(&guest); 5801 cmd.args(&["--cpus", "boot=4"]) 5802 .args(&["--memory", "size=4G"]) 5803 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 5804 .default_disks() 5805 .args(&["--net", guest.default_net_string().as_str()]) 5806 .args(&["--api-socket", &api_socket]) 5807 .capture_output(); 5808 5809 let mut child = cmd.spawn().unwrap(); 5810 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 5811 5812 let r = std::panic::catch_unwind(|| { 5813 guest.wait_vm_boot(None).unwrap(); 5814 5815 assert!(remote_command(&api_socket, "pause", None)); 5816 5817 assert!(remote_command( 5818 &api_socket, 5819 "coredump", 5820 Some(format!("file://{}", vmcore_file).as_str()), 5821 )); 5822 5823 // the num of CORE notes should equals to vcpu 5824 let readelf_core_num_cmd = format!( 5825 "readelf --all {} |grep CORE |grep -v Type |wc -l", 5826 vmcore_file 5827 ); 5828 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 5829 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 5830 5831 // the num of QEMU notes should equals to vcpu 5832 let readelf_vmm_num_cmd = format!("readelf --all {} |grep QEMU |wc -l", vmcore_file); 5833 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 5834 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 5835 }); 5836 5837 let _ = child.kill(); 5838 let output = child.wait_with_output().unwrap(); 5839 5840 handle_child_output(r, &output); 5841 } 5842 5843 #[test] 5844 fn test_watchdog() { 5845 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5846 let guest = Guest::new(Box::new(focal)); 5847 let api_socket = temp_api_path(&guest.tmp_dir); 5848 5849 let kernel_path = direct_kernel_boot_path(); 5850 5851 let mut cmd = GuestCommand::new(&guest); 5852 cmd.args(&["--cpus", "boot=1"]) 5853 .args(&["--memory", "size=512M"]) 5854 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5855 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5856 .default_disks() 5857 .args(&["--net", guest.default_net_string().as_str()]) 5858 .args(&["--watchdog"]) 5859 .args(&["--api-socket", &api_socket]) 5860 .capture_output(); 5861 5862 let mut child = cmd.spawn().unwrap(); 5863 5864 let r = std::panic::catch_unwind(|| { 5865 guest.wait_vm_boot(None).unwrap(); 5866 let mut current_reboot_count = 1; 5867 enable_guest_watchdog(&guest, &mut current_reboot_count); 5868 check_guest_watchdog_no_reboot(&guest, ¤t_reboot_count); 5869 check_guest_watchdog_one_reboot(&guest, &api_socket, &mut current_reboot_count); 5870 }); 5871 5872 let _ = child.kill(); 5873 let output = child.wait_with_output().unwrap(); 5874 5875 handle_child_output(r, &output); 5876 } 5877 5878 #[test] 5879 fn test_tap_from_fd() { 5880 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5881 let guest = Guest::new(Box::new(focal)); 5882 let kernel_path = direct_kernel_boot_path(); 5883 5884 // Create a TAP interface with multi-queue enabled 5885 let num_queue_pairs: usize = 2; 5886 5887 use std::str::FromStr; 5888 let taps = net_util::open_tap( 5889 Some("chtap0"), 5890 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 5891 None, 5892 &mut None, 5893 num_queue_pairs, 5894 Some(libc::O_RDWR | libc::O_NONBLOCK), 5895 ) 5896 .unwrap(); 5897 5898 let mut child = GuestCommand::new(&guest) 5899 .args(&["--cpus", &format!("boot={}", num_queue_pairs)]) 5900 .args(&["--memory", "size=512M"]) 5901 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5902 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5903 .default_disks() 5904 .args(&[ 5905 "--net", 5906 &format!( 5907 "fd=[{},{}],mac={},num_queues={}", 5908 taps[0].as_raw_fd(), 5909 taps[1].as_raw_fd(), 5910 guest.network.guest_mac, 5911 num_queue_pairs * 2 5912 ), 5913 ]) 5914 .capture_output() 5915 .spawn() 5916 .unwrap(); 5917 5918 let r = std::panic::catch_unwind(|| { 5919 guest.wait_vm_boot(None).unwrap(); 5920 5921 assert_eq!( 5922 guest 5923 .ssh_command("ip -o link | wc -l") 5924 .unwrap() 5925 .trim() 5926 .parse::<u32>() 5927 .unwrap_or_default(), 5928 2 5929 ); 5930 5931 guest.reboot_linux(0, None); 5932 5933 assert_eq!( 5934 guest 5935 .ssh_command("ip -o link | wc -l") 5936 .unwrap() 5937 .trim() 5938 .parse::<u32>() 5939 .unwrap_or_default(), 5940 2 5941 ); 5942 }); 5943 5944 let _ = child.kill(); 5945 let output = child.wait_with_output().unwrap(); 5946 5947 handle_child_output(r, &output); 5948 } 5949 5950 // By design, a guest VM won't be able to connect to the host 5951 // machine when using a macvtap network interface (while it can 5952 // communicate externally). As a workaround, this integration 5953 // test creates two macvtap interfaces in 'bridge' mode on the 5954 // same physical net interface, one for the guest and one for 5955 // the host. With additional setup on the IP address and the 5956 // routing table, it enables the communications between the 5957 // guest VM and the host machine. 5958 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 5959 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 5960 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5961 let guest = Guest::new(Box::new(focal)); 5962 let api_socket = temp_api_path(&guest.tmp_dir); 5963 5964 #[cfg(target_arch = "x86_64")] 5965 let kernel_path = direct_kernel_boot_path(); 5966 #[cfg(target_arch = "aarch64")] 5967 let kernel_path = edk2_path(); 5968 5969 let phy_net = "eth0"; 5970 5971 // Create a macvtap interface for the guest VM to use 5972 assert!(exec_host_command_status(&format!( 5973 "sudo ip link add link {} name {} type macvtap mod bridge", 5974 phy_net, guest_macvtap_name 5975 )) 5976 .success()); 5977 assert!(exec_host_command_status(&format!( 5978 "sudo ip link set {} address {} up", 5979 guest_macvtap_name, guest.network.guest_mac 5980 )) 5981 .success()); 5982 assert!( 5983 exec_host_command_status(&format!("sudo ip link show {}", guest_macvtap_name)) 5984 .success() 5985 ); 5986 5987 let tap_index = 5988 fs::read_to_string(&format!("/sys/class/net/{}/ifindex", guest_macvtap_name)).unwrap(); 5989 let tap_device = format!("/dev/tap{}", tap_index.trim()); 5990 5991 assert!( 5992 exec_host_command_status(&format!("sudo chown $UID.$UID {}", tap_device)).success() 5993 ); 5994 5995 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 5996 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 5997 assert!(tap_fd1 > 0); 5998 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 5999 assert!(tap_fd2 > 0); 6000 6001 // Create a macvtap on the same physical net interface for 6002 // the host machine to use 6003 assert!(exec_host_command_status(&format!( 6004 "sudo ip link add link {} name {} type macvtap mod bridge", 6005 phy_net, host_macvtap_name 6006 )) 6007 .success()); 6008 // Use default mask "255.255.255.0" 6009 assert!(exec_host_command_status(&format!( 6010 "sudo ip address add {}/24 dev {}", 6011 guest.network.host_ip, host_macvtap_name 6012 )) 6013 .success()); 6014 assert!(exec_host_command_status(&format!( 6015 "sudo ip link set dev {} up", 6016 host_macvtap_name 6017 )) 6018 .success()); 6019 6020 let mut guest_command = GuestCommand::new(&guest); 6021 guest_command 6022 .args(&["--cpus", "boot=2"]) 6023 .args(&["--memory", "size=512M"]) 6024 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6025 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6026 .default_disks() 6027 .args(&["--api-socket", &api_socket]); 6028 6029 let net_params = format!( 6030 "fd=[{},{}],mac={},num_queues=4", 6031 tap_fd1, tap_fd2, guest.network.guest_mac 6032 ); 6033 6034 if !hotplug { 6035 guest_command.args(&["--net", &net_params]); 6036 } 6037 6038 let mut child = guest_command.capture_output().spawn().unwrap(); 6039 6040 if hotplug { 6041 // Give some time to the VMM process to listen to the API 6042 // socket. This is the only requirement to avoid the following 6043 // call to ch-remote from failing. 6044 thread::sleep(std::time::Duration::new(10, 0)); 6045 // Hotplug the virtio-net device 6046 let (cmd_success, cmd_output) = 6047 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6048 assert!(cmd_success); 6049 #[cfg(target_arch = "x86_64")] 6050 assert!(String::from_utf8_lossy(&cmd_output) 6051 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6052 #[cfg(target_arch = "aarch64")] 6053 assert!(String::from_utf8_lossy(&cmd_output) 6054 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6055 } 6056 6057 // The functional connectivity provided by the virtio-net device 6058 // gets tested through wait_vm_boot() as it expects to receive a 6059 // HTTP request, and through the SSH command as well. 6060 let r = std::panic::catch_unwind(|| { 6061 guest.wait_vm_boot(None).unwrap(); 6062 6063 assert_eq!( 6064 guest 6065 .ssh_command("ip -o link | wc -l") 6066 .unwrap() 6067 .trim() 6068 .parse::<u32>() 6069 .unwrap_or_default(), 6070 2 6071 ); 6072 }); 6073 6074 let _ = child.kill(); 6075 6076 exec_host_command_status(&format!("sudo ip link del {}", guest_macvtap_name)); 6077 exec_host_command_status(&format!("sudo ip link del {}", host_macvtap_name)); 6078 6079 let output = child.wait_with_output().unwrap(); 6080 6081 handle_child_output(r, &output); 6082 } 6083 6084 #[test] 6085 fn test_macvtap() { 6086 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6087 } 6088 6089 #[test] 6090 fn test_macvtap_hotplug() { 6091 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6092 } 6093 6094 #[test] 6095 #[cfg(not(feature = "mshv"))] 6096 fn test_ovs_dpdk() { 6097 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6098 let guest1 = Guest::new(Box::new(focal1)); 6099 6100 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6101 let guest2 = Guest::new(Box::new(focal2)); 6102 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6103 6104 let (mut child1, mut child2) = 6105 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6106 6107 // Create the snapshot directory 6108 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6109 6110 let r = std::panic::catch_unwind(|| { 6111 // Remove one of the two ports from the OVS bridge 6112 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6113 6114 // Spawn a new netcat listener in the first VM 6115 let guest_ip = guest1.network.guest_ip.clone(); 6116 thread::spawn(move || { 6117 ssh_command_ip( 6118 "nc -l 12345", 6119 &guest_ip, 6120 DEFAULT_SSH_RETRIES, 6121 DEFAULT_SSH_TIMEOUT, 6122 ) 6123 .unwrap(); 6124 }); 6125 6126 // Wait for the server to be listening 6127 thread::sleep(std::time::Duration::new(5, 0)); 6128 6129 // Check the connection fails this time 6130 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6131 6132 // Add the OVS port back 6133 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6134 6135 // And finally check the connection is functional again 6136 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6137 6138 // Pause the VM 6139 assert!(remote_command(&api_socket_source, "pause", None)); 6140 6141 // Take a snapshot from the VM 6142 assert!(remote_command( 6143 &api_socket_source, 6144 "snapshot", 6145 Some(format!("file://{}", snapshot_dir).as_str()), 6146 )); 6147 6148 // Wait to make sure the snapshot is completed 6149 thread::sleep(std::time::Duration::new(10, 0)); 6150 }); 6151 6152 // Shutdown the source VM 6153 let _ = child2.kill(); 6154 let output = child2.wait_with_output().unwrap(); 6155 handle_child_output(r, &output); 6156 6157 // Remove the vhost-user socket file. 6158 Command::new("rm") 6159 .arg("-f") 6160 .arg("/tmp/dpdkvhostclient2") 6161 .output() 6162 .unwrap(); 6163 6164 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6165 // Restore the VM from the snapshot 6166 let mut child2 = GuestCommand::new(&guest2) 6167 .args(&["--api-socket", &api_socket_restored]) 6168 .args(&[ 6169 "--restore", 6170 format!("source_url=file://{}", snapshot_dir).as_str(), 6171 ]) 6172 .capture_output() 6173 .spawn() 6174 .unwrap(); 6175 6176 // Wait for the VM to be restored 6177 thread::sleep(std::time::Duration::new(10, 0)); 6178 6179 let r = std::panic::catch_unwind(|| { 6180 // Resume the VM 6181 assert!(remote_command(&api_socket_restored, "resume", None)); 6182 6183 // Spawn a new netcat listener in the first VM 6184 let guest_ip = guest1.network.guest_ip.clone(); 6185 thread::spawn(move || { 6186 ssh_command_ip( 6187 "nc -l 12345", 6188 &guest_ip, 6189 DEFAULT_SSH_RETRIES, 6190 DEFAULT_SSH_TIMEOUT, 6191 ) 6192 .unwrap(); 6193 }); 6194 6195 // Wait for the server to be listening 6196 thread::sleep(std::time::Duration::new(5, 0)); 6197 6198 // And check the connection is still functional after restore 6199 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6200 }); 6201 6202 cleanup_ovs_dpdk(); 6203 6204 let _ = child1.kill(); 6205 let _ = child2.kill(); 6206 6207 let output = child1.wait_with_output().unwrap(); 6208 child2.wait().unwrap(); 6209 6210 handle_child_output(r, &output); 6211 } 6212 6213 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6214 cleanup_spdk_nvme(); 6215 6216 assert!(exec_host_command_status(&format!( 6217 "mkdir -p {}", 6218 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6219 )) 6220 .success()); 6221 assert!(exec_host_command_status(&format!( 6222 "truncate {} -s 128M", 6223 nvme_dir.join("test-disk.raw").to_str().unwrap() 6224 )) 6225 .success()); 6226 assert!(exec_host_command_status(&format!( 6227 "mkfs.ext4 {}", 6228 nvme_dir.join("test-disk.raw").to_str().unwrap() 6229 )) 6230 .success()); 6231 6232 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6233 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6234 .args(&["-i", "0", "-m", "0x1"]) 6235 .spawn() 6236 .unwrap(); 6237 thread::sleep(std::time::Duration::new(2, 0)); 6238 6239 assert!(exec_host_command_status( 6240 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER" 6241 ) 6242 .success()); 6243 assert!(exec_host_command_status(&format!( 6244 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6245 nvme_dir.join("test-disk.raw").to_str().unwrap() 6246 )) 6247 .success()); 6248 assert!(exec_host_command_status( 6249 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6250 ) 6251 .success()); 6252 assert!(exec_host_command_status( 6253 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6254 ) 6255 .success()); 6256 assert!(exec_host_command_status(&format!( 6257 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6258 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6259 )) 6260 .success()); 6261 } 6262 6263 fn cleanup_spdk_nvme() { 6264 exec_host_command_status("pkill -f nvmf_tgt"); 6265 } 6266 6267 #[test] 6268 fn test_vfio_user() { 6269 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6270 let jammy = UbuntuDiskConfig::new(jammy_image); 6271 let guest = Guest::new(Box::new(jammy)); 6272 6273 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6274 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6275 6276 let api_socket = temp_api_path(&guest.tmp_dir); 6277 let mut child = GuestCommand::new(&guest) 6278 .args(&["--api-socket", &api_socket]) 6279 .args(&["--cpus", "boot=1"]) 6280 .args(&["--memory", "size=512M,shared=on,hugepages=on"]) 6281 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6282 .args(&["--serial", "tty", "--console", "off"]) 6283 .default_disks() 6284 .default_net() 6285 .capture_output() 6286 .spawn() 6287 .unwrap(); 6288 6289 let r = std::panic::catch_unwind(|| { 6290 guest.wait_vm_boot(None).unwrap(); 6291 6292 // Hotplug the SPDK-NVMe device to the VM 6293 let (cmd_success, cmd_output) = remote_command_w_output( 6294 &api_socket, 6295 "add-user-device", 6296 Some(&format!( 6297 "socket={},id=vfio_user0", 6298 spdk_nvme_dir 6299 .as_path() 6300 .join("nvme-vfio-user/cntrl") 6301 .to_str() 6302 .unwrap(), 6303 )), 6304 ); 6305 assert!(cmd_success); 6306 assert!(String::from_utf8_lossy(&cmd_output) 6307 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6308 6309 thread::sleep(std::time::Duration::new(10, 0)); 6310 6311 // Check both if /dev/nvme exists and if the block size is 128M. 6312 assert_eq!( 6313 guest 6314 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6315 .unwrap() 6316 .trim() 6317 .parse::<u32>() 6318 .unwrap_or_default(), 6319 1 6320 ); 6321 6322 // Check changes persist after reboot 6323 assert_eq!( 6324 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6325 "" 6326 ); 6327 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6328 guest 6329 .ssh_command("echo test123 | sudo tee /mnt/test") 6330 .unwrap(); 6331 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6332 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6333 6334 guest.reboot_linux(0, None); 6335 assert_eq!( 6336 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6337 "" 6338 ); 6339 assert_eq!( 6340 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6341 "test123" 6342 ); 6343 }); 6344 6345 cleanup_spdk_nvme(); 6346 6347 let _ = child.kill(); 6348 let output = child.wait_with_output().unwrap(); 6349 6350 handle_child_output(r, &output); 6351 } 6352 6353 #[test] 6354 #[cfg(target_arch = "x86_64")] 6355 fn test_vdpa_block() { 6356 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6357 if !exec_host_command_status("lsmod | grep vdpa_sim_blk").success() { 6358 return; 6359 } 6360 6361 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6362 let guest = Guest::new(Box::new(focal)); 6363 let api_socket = temp_api_path(&guest.tmp_dir); 6364 6365 let kernel_path = direct_kernel_boot_path(); 6366 6367 let mut child = GuestCommand::new(&guest) 6368 .args(&["--cpus", "boot=2"]) 6369 .args(&["--memory", "size=512M,hugepages=on"]) 6370 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6371 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6372 .default_disks() 6373 .default_net() 6374 .args(&["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6375 .args(&["--platform", "num_pci_segments=2,iommu_segments=1"]) 6376 .args(&["--api-socket", &api_socket]) 6377 .capture_output() 6378 .spawn() 6379 .unwrap(); 6380 6381 let r = std::panic::catch_unwind(|| { 6382 guest.wait_vm_boot(None).unwrap(); 6383 6384 // Check both if /dev/vdc exists and if the block size is 128M. 6385 assert_eq!( 6386 guest 6387 .ssh_command("lsblk | grep vdc | grep -c 128M") 6388 .unwrap() 6389 .trim() 6390 .parse::<u32>() 6391 .unwrap_or_default(), 6392 1 6393 ); 6394 6395 // Check the content of the block device after we wrote to it. 6396 // The vpda-sim-blk should let us read what we previously wrote. 6397 guest 6398 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6399 .unwrap(); 6400 assert_eq!( 6401 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6402 "foobar" 6403 ); 6404 6405 // Hotplug an extra vDPA block device behind the vIOMMU 6406 // Add a new vDPA device to the VM 6407 let (cmd_success, cmd_output) = remote_command_w_output( 6408 &api_socket, 6409 "add-vdpa", 6410 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6411 ); 6412 assert!(cmd_success); 6413 assert!(String::from_utf8_lossy(&cmd_output) 6414 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6415 6416 thread::sleep(std::time::Duration::new(10, 0)); 6417 6418 // Check IOMMU setup 6419 assert!(guest 6420 .does_device_vendor_pair_match("0x1057", "0x1af4") 6421 .unwrap_or_default()); 6422 assert_eq!( 6423 guest 6424 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6425 .unwrap() 6426 .trim(), 6427 "0001:00:01.0" 6428 ); 6429 6430 // Check both if /dev/vdd exists and if the block size is 128M. 6431 assert_eq!( 6432 guest 6433 .ssh_command("lsblk | grep vdd | grep -c 128M") 6434 .unwrap() 6435 .trim() 6436 .parse::<u32>() 6437 .unwrap_or_default(), 6438 1 6439 ); 6440 6441 // Write some content to the block device we've just plugged. 6442 guest 6443 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6444 .unwrap(); 6445 6446 // Check we can read the content back. 6447 assert_eq!( 6448 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6449 "foobar" 6450 ); 6451 6452 // Unplug the device 6453 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6454 assert!(cmd_success); 6455 thread::sleep(std::time::Duration::new(10, 0)); 6456 6457 // Check /dev/vdd doesn't exist anymore 6458 assert_eq!( 6459 guest 6460 .ssh_command("lsblk | grep -c vdd || true") 6461 .unwrap() 6462 .trim() 6463 .parse::<u32>() 6464 .unwrap_or(1), 6465 0 6466 ); 6467 }); 6468 6469 let _ = child.kill(); 6470 let output = child.wait_with_output().unwrap(); 6471 6472 handle_child_output(r, &output); 6473 } 6474 6475 #[test] 6476 #[cfg(target_arch = "x86_64")] 6477 fn test_vdpa_net() { 6478 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6479 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6480 return; 6481 } 6482 6483 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6484 let guest = Guest::new(Box::new(focal)); 6485 6486 let kernel_path = direct_kernel_boot_path(); 6487 6488 let mut child = GuestCommand::new(&guest) 6489 .args(&["--cpus", "boot=2"]) 6490 .args(&["--memory", "size=512M,hugepages=on"]) 6491 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6492 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6493 .default_disks() 6494 .default_net() 6495 .args(&["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 6496 .capture_output() 6497 .spawn() 6498 .unwrap(); 6499 6500 let r = std::panic::catch_unwind(|| { 6501 guest.wait_vm_boot(None).unwrap(); 6502 6503 // Check we can find network interface related to vDPA device 6504 assert_eq!( 6505 guest 6506 .ssh_command("ip -o link | grep -c ens6") 6507 .unwrap() 6508 .trim() 6509 .parse::<u32>() 6510 .unwrap_or(0), 6511 1 6512 ); 6513 6514 guest 6515 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 6516 .unwrap(); 6517 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 6518 6519 // Check there is no packet yet on both TX/RX of the network interface 6520 assert_eq!( 6521 guest 6522 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 6523 .unwrap() 6524 .trim() 6525 .parse::<u32>() 6526 .unwrap_or(0), 6527 2 6528 ); 6529 6530 // Send 6 packets with ping command 6531 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 6532 6533 // Check we can find 6 packets on both TX/RX of the network interface 6534 assert_eq!( 6535 guest 6536 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 6537 .unwrap() 6538 .trim() 6539 .parse::<u32>() 6540 .unwrap_or(0), 6541 2 6542 ); 6543 6544 // No need to check for hotplug as we already tested it through 6545 // test_vdpa_block() 6546 }); 6547 6548 let _ = child.kill(); 6549 let output = child.wait_with_output().unwrap(); 6550 6551 handle_child_output(r, &output); 6552 } 6553 } 6554 6555 mod sequential { 6556 use crate::*; 6557 6558 #[test] 6559 fn test_memory_mergeable_on() { 6560 test_memory_mergeable(true) 6561 } 6562 } 6563 6564 #[cfg(target_arch = "x86_64")] 6565 mod windows { 6566 use crate::*; 6567 use once_cell::sync::Lazy; 6568 6569 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 6570 6571 struct WindowsGuest { 6572 guest: Guest, 6573 auth: PasswordAuth, 6574 } 6575 6576 trait FsType { 6577 const FS_FAT: u8; 6578 const FS_NTFS: u8; 6579 } 6580 impl FsType for WindowsGuest { 6581 const FS_FAT: u8 = 0; 6582 const FS_NTFS: u8 = 1; 6583 } 6584 6585 impl WindowsGuest { 6586 fn new() -> Self { 6587 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 6588 let guest = Guest::new(Box::new(disk)); 6589 let auth = PasswordAuth { 6590 username: String::from("administrator"), 6591 password: String::from("Admin123"), 6592 }; 6593 6594 WindowsGuest { guest, auth } 6595 } 6596 6597 fn guest(&self) -> &Guest { 6598 &self.guest 6599 } 6600 6601 fn ssh_cmd(&self, cmd: &str) -> String { 6602 ssh_command_ip_with_auth( 6603 cmd, 6604 &self.auth, 6605 &self.guest.network.guest_ip, 6606 DEFAULT_SSH_RETRIES, 6607 DEFAULT_SSH_TIMEOUT, 6608 ) 6609 .unwrap() 6610 } 6611 6612 fn cpu_count(&self) -> u8 { 6613 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 6614 .trim() 6615 .parse::<u8>() 6616 .unwrap_or(0) 6617 } 6618 6619 fn ram_size(&self) -> usize { 6620 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 6621 .trim() 6622 .parse::<usize>() 6623 .unwrap_or(0) 6624 } 6625 6626 fn netdev_count(&self) -> u8 { 6627 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6628 .trim() 6629 .parse::<u8>() 6630 .unwrap_or(0) 6631 } 6632 6633 fn disk_count(&self) -> u8 { 6634 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6635 .trim() 6636 .parse::<u8>() 6637 .unwrap_or(0) 6638 } 6639 6640 fn reboot(&self) { 6641 let _ = self.ssh_cmd("shutdown /r /t 0"); 6642 } 6643 6644 fn shutdown(&self) { 6645 let _ = self.ssh_cmd("shutdown /s /t 0"); 6646 } 6647 6648 fn run_dnsmasq(&self) -> std::process::Child { 6649 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 6650 let dhcp_host = format!( 6651 "--dhcp-host={},{}", 6652 self.guest.network.guest_mac, self.guest.network.guest_ip 6653 ); 6654 let dhcp_range = format!( 6655 "--dhcp-range=eth,{},{}", 6656 self.guest.network.guest_ip, self.guest.network.guest_ip 6657 ); 6658 6659 Command::new("dnsmasq") 6660 .arg("--no-daemon") 6661 .arg("--log-queries") 6662 .arg(listen_address.as_str()) 6663 .arg("--except-interface=lo") 6664 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 6665 .arg("--conf-file=/dev/null") 6666 .arg(dhcp_host.as_str()) 6667 .arg(dhcp_range.as_str()) 6668 .spawn() 6669 .unwrap() 6670 } 6671 6672 // TODO Cleanup image file explicitly after test, if there's some space issues. 6673 fn disk_new(&self, fs: u8, sz: usize) -> String { 6674 let mut guard = NEXT_DISK_ID.lock().unwrap(); 6675 let id = *guard; 6676 *guard = id + 1; 6677 6678 let img = PathBuf::from(format!("/tmp/test-hotplug-{}.raw", id)); 6679 let _ = fs::remove_file(&img); 6680 6681 // Create an image file 6682 let out = Command::new("qemu-img") 6683 .args(&[ 6684 "create", 6685 "-f", 6686 "raw", 6687 img.to_str().unwrap(), 6688 format!("{}m", sz).as_str(), 6689 ]) 6690 .output() 6691 .expect("qemu-img command failed") 6692 .stdout; 6693 println!("{:?}", out); 6694 6695 // Associate image to a loop device 6696 let out = Command::new("losetup") 6697 .args(&["--show", "-f", img.to_str().unwrap()]) 6698 .output() 6699 .expect("failed to create loop device") 6700 .stdout; 6701 let _tmp = String::from_utf8_lossy(&out); 6702 let loop_dev = _tmp.trim(); 6703 println!("{:?}", out); 6704 6705 // Create a partition table 6706 // echo 'type=7' | sudo sfdisk "${LOOP}" 6707 let mut child = Command::new("sfdisk") 6708 .args(&[loop_dev]) 6709 .stdin(Stdio::piped()) 6710 .spawn() 6711 .unwrap(); 6712 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 6713 stdin 6714 .write_all("type=7".as_bytes()) 6715 .expect("failed to write stdin"); 6716 let out = child.wait_with_output().expect("sfdisk failed").stdout; 6717 println!("{:?}", out); 6718 6719 // Disengage the loop device 6720 let out = Command::new("losetup") 6721 .args(&["-d", loop_dev]) 6722 .output() 6723 .expect("loop device not found") 6724 .stdout; 6725 println!("{:?}", out); 6726 6727 // Re-associate loop device pointing to the partition only 6728 let out = Command::new("losetup") 6729 .args(&[ 6730 "--show", 6731 "--offset", 6732 (512 * 2048).to_string().as_str(), 6733 "-f", 6734 img.to_str().unwrap(), 6735 ]) 6736 .output() 6737 .expect("failed to create loop device") 6738 .stdout; 6739 let _tmp = String::from_utf8_lossy(&out); 6740 let loop_dev = _tmp.trim(); 6741 println!("{:?}", out); 6742 6743 // Create filesystem. 6744 let fs_cmd = match fs { 6745 WindowsGuest::FS_FAT => "mkfs.msdos", 6746 WindowsGuest::FS_NTFS => "mkfs.ntfs", 6747 _ => panic!("Unknown filesystem type '{}'", fs), 6748 }; 6749 let out = Command::new(fs_cmd) 6750 .args(&[&loop_dev]) 6751 .output() 6752 .unwrap_or_else(|_| panic!("{} failed", fs_cmd)) 6753 .stdout; 6754 println!("{:?}", out); 6755 6756 // Disengage the loop device 6757 let out = Command::new("losetup") 6758 .args(&["-d", loop_dev]) 6759 .output() 6760 .unwrap_or_else(|_| panic!("loop device '{}' not found", loop_dev)) 6761 .stdout; 6762 println!("{:?}", out); 6763 6764 img.to_str().unwrap().to_string() 6765 } 6766 6767 fn disks_set_rw(&self) { 6768 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 6769 } 6770 6771 fn disks_online(&self) { 6772 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 6773 } 6774 6775 fn disk_file_put(&self, fname: &str, data: &str) { 6776 let _ = self.ssh_cmd(&format!( 6777 "powershell -Command \"'{}' | Set-Content -Path {}\"", 6778 data, fname 6779 )); 6780 } 6781 6782 fn disk_file_read(&self, fname: &str) -> String { 6783 self.ssh_cmd(&format!( 6784 "powershell -Command \"Get-Content -Path {}\"", 6785 fname 6786 )) 6787 } 6788 6789 fn wait_for_boot(&self) -> bool { 6790 let cmd = "dir /b c:\\ | find \"Windows\""; 6791 let tmo_max = 180; 6792 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 6793 // interval must be small. 6794 let tmo_int = 2; 6795 let out = ssh_command_ip_with_auth( 6796 cmd, 6797 &self.auth, 6798 &self.guest.network.guest_ip, 6799 { 6800 let mut ret = 1; 6801 let mut tmo_acc = 0; 6802 loop { 6803 tmo_acc += tmo_int * ret; 6804 if tmo_acc >= tmo_max { 6805 break; 6806 } 6807 ret += 1; 6808 } 6809 ret 6810 }, 6811 tmo_int, 6812 ) 6813 .unwrap(); 6814 6815 if "Windows" == out.trim() { 6816 return true; 6817 } 6818 6819 false 6820 } 6821 } 6822 6823 fn vcpu_threads_count(pid: u32) -> u8 { 6824 // ps -T -p 12345 | grep vcpu | wc -l 6825 let out = Command::new("ps") 6826 .args(&["-T", "-p", format!("{}", pid).as_str()]) 6827 .output() 6828 .expect("ps command failed") 6829 .stdout; 6830 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 6831 } 6832 6833 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 6834 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 6835 let out = Command::new("ps") 6836 .args(&["-T", "-p", format!("{}", pid).as_str()]) 6837 .output() 6838 .expect("ps command failed") 6839 .stdout; 6840 let mut n = 0; 6841 String::from_utf8_lossy(&out) 6842 .split_whitespace() 6843 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 6844 n 6845 } 6846 6847 fn disk_ctrl_threads_count(pid: u32) -> u8 { 6848 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 6849 let out = Command::new("ps") 6850 .args(&["-T", "-p", format!("{}", pid).as_str()]) 6851 .output() 6852 .expect("ps command failed") 6853 .stdout; 6854 let mut n = 0; 6855 String::from_utf8_lossy(&out) 6856 .split_whitespace() 6857 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 6858 n 6859 } 6860 6861 #[test] 6862 fn test_windows_guest() { 6863 let windows_guest = WindowsGuest::new(); 6864 6865 let mut ovmf_path = dirs::home_dir().unwrap(); 6866 ovmf_path.push("workloads"); 6867 ovmf_path.push(OVMF_NAME); 6868 6869 let mut child = GuestCommand::new(windows_guest.guest()) 6870 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6871 .args(&["--memory", "size=4G"]) 6872 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6873 .args(&["--serial", "tty"]) 6874 .args(&["--console", "off"]) 6875 .default_disks() 6876 .default_net() 6877 .capture_output() 6878 .spawn() 6879 .unwrap(); 6880 6881 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 6882 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6883 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 6884 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6885 6886 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 6887 6888 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6889 6890 let r = std::panic::catch_unwind(|| { 6891 // Wait to make sure Windows boots up 6892 assert!(windows_guest.wait_for_boot()); 6893 6894 windows_guest.shutdown(); 6895 }); 6896 6897 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6898 let _ = child.kill(); 6899 let output = child.wait_with_output().unwrap(); 6900 6901 let _ = child_dnsmasq.kill(); 6902 let _ = child_dnsmasq.wait(); 6903 6904 handle_child_output(r, &output); 6905 } 6906 6907 #[test] 6908 fn test_windows_guest_multiple_queues() { 6909 let windows_guest = WindowsGuest::new(); 6910 6911 let mut ovmf_path = dirs::home_dir().unwrap(); 6912 ovmf_path.push("workloads"); 6913 ovmf_path.push(OVMF_NAME); 6914 6915 let mut child = GuestCommand::new(windows_guest.guest()) 6916 .args(&["--cpus", "boot=4,kvm_hyperv=on"]) 6917 .args(&["--memory", "size=4G"]) 6918 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6919 .args(&["--serial", "tty"]) 6920 .args(&["--console", "off"]) 6921 .args(&[ 6922 "--disk", 6923 format!( 6924 "path={},num_queues=4", 6925 windows_guest 6926 .guest() 6927 .disk_config 6928 .disk(DiskType::OperatingSystem) 6929 .unwrap() 6930 ) 6931 .as_str(), 6932 ]) 6933 .args(&[ 6934 "--net", 6935 format!( 6936 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 6937 windows_guest.guest().network.guest_mac, 6938 windows_guest.guest().network.host_ip 6939 ) 6940 .as_str(), 6941 ]) 6942 .capture_output() 6943 .spawn() 6944 .unwrap(); 6945 6946 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 6947 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6948 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 6949 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6950 6951 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 6952 6953 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6954 6955 let r = std::panic::catch_unwind(|| { 6956 // Wait to make sure Windows boots up 6957 assert!(windows_guest.wait_for_boot()); 6958 6959 windows_guest.shutdown(); 6960 }); 6961 6962 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6963 let _ = child.kill(); 6964 let output = child.wait_with_output().unwrap(); 6965 6966 let _ = child_dnsmasq.kill(); 6967 let _ = child_dnsmasq.wait(); 6968 6969 handle_child_output(r, &output); 6970 } 6971 6972 #[test] 6973 #[cfg(not(feature = "mshv"))] 6974 #[ignore = "See #4327"] 6975 fn test_windows_guest_snapshot_restore() { 6976 let windows_guest = WindowsGuest::new(); 6977 6978 let mut ovmf_path = dirs::home_dir().unwrap(); 6979 ovmf_path.push("workloads"); 6980 ovmf_path.push(OVMF_NAME); 6981 6982 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6983 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 6984 6985 let mut child = GuestCommand::new(windows_guest.guest()) 6986 .args(&["--api-socket", &api_socket_source]) 6987 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6988 .args(&["--memory", "size=4G"]) 6989 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6990 .args(&["--serial", "tty"]) 6991 .args(&["--console", "off"]) 6992 .default_disks() 6993 .default_net() 6994 .capture_output() 6995 .spawn() 6996 .unwrap(); 6997 6998 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 6999 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7000 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7001 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7002 7003 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7004 7005 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7006 7007 // Wait to make sure Windows boots up 7008 assert!(windows_guest.wait_for_boot()); 7009 7010 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 7011 7012 // Pause the VM 7013 assert!(remote_command(&api_socket_source, "pause", None)); 7014 7015 // Take a snapshot from the VM 7016 assert!(remote_command( 7017 &api_socket_source, 7018 "snapshot", 7019 Some(format!("file://{}", snapshot_dir).as_str()), 7020 )); 7021 7022 // Wait to make sure the snapshot is completed 7023 thread::sleep(std::time::Duration::new(30, 0)); 7024 7025 let _ = child.kill(); 7026 child.wait().unwrap(); 7027 7028 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 7029 7030 // Restore the VM from the snapshot 7031 let mut child = GuestCommand::new(windows_guest.guest()) 7032 .args(&["--api-socket", &api_socket_restored]) 7033 .args(&[ 7034 "--restore", 7035 format!("source_url=file://{}", snapshot_dir).as_str(), 7036 ]) 7037 .capture_output() 7038 .spawn() 7039 .unwrap(); 7040 7041 // Wait for the VM to be restored 7042 thread::sleep(std::time::Duration::new(20, 0)); 7043 7044 let r = std::panic::catch_unwind(|| { 7045 // Resume the VM 7046 assert!(remote_command(&api_socket_restored, "resume", None)); 7047 7048 windows_guest.shutdown(); 7049 }); 7050 7051 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7052 let _ = child.kill(); 7053 let output = child.wait_with_output().unwrap(); 7054 7055 let _ = child_dnsmasq.kill(); 7056 let _ = child_dnsmasq.wait(); 7057 7058 handle_child_output(r, &output); 7059 } 7060 7061 #[test] 7062 #[cfg(not(feature = "mshv"))] 7063 fn test_windows_guest_cpu_hotplug() { 7064 let windows_guest = WindowsGuest::new(); 7065 7066 let mut ovmf_path = dirs::home_dir().unwrap(); 7067 ovmf_path.push("workloads"); 7068 ovmf_path.push(OVMF_NAME); 7069 7070 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7071 let api_socket = temp_api_path(&tmp_dir); 7072 7073 let mut child = GuestCommand::new(windows_guest.guest()) 7074 .args(&["--api-socket", &api_socket]) 7075 .args(&["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 7076 .args(&["--memory", "size=4G"]) 7077 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7078 .args(&["--serial", "tty"]) 7079 .args(&["--console", "off"]) 7080 .default_disks() 7081 .default_net() 7082 .capture_output() 7083 .spawn() 7084 .unwrap(); 7085 7086 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7087 7088 let r = std::panic::catch_unwind(|| { 7089 // Wait to make sure Windows boots up 7090 assert!(windows_guest.wait_for_boot()); 7091 7092 let vcpu_num = 2; 7093 // Check the initial number of CPUs the guest sees 7094 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7095 // Check the initial number of vcpu threads in the CH process 7096 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7097 7098 let vcpu_num = 6; 7099 // Hotplug some CPUs 7100 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7101 // Wait to make sure CPUs are added 7102 thread::sleep(std::time::Duration::new(10, 0)); 7103 // Check the guest sees the correct number 7104 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7105 // Check the CH process has the correct number of vcpu threads 7106 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7107 7108 let vcpu_num = 4; 7109 // Remove some CPUs. Note that Windows doesn't support hot-remove. 7110 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7111 // Wait to make sure CPUs are removed 7112 thread::sleep(std::time::Duration::new(10, 0)); 7113 // Reboot to let Windows catch up 7114 windows_guest.reboot(); 7115 // Wait to make sure Windows completely rebooted 7116 thread::sleep(std::time::Duration::new(60, 0)); 7117 // Check the guest sees the correct number 7118 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7119 // Check the CH process has the correct number of vcpu threads 7120 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7121 7122 windows_guest.shutdown(); 7123 }); 7124 7125 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7126 let _ = child.kill(); 7127 let output = child.wait_with_output().unwrap(); 7128 7129 let _ = child_dnsmasq.kill(); 7130 let _ = child_dnsmasq.wait(); 7131 7132 handle_child_output(r, &output); 7133 } 7134 7135 #[test] 7136 #[cfg(not(feature = "mshv"))] 7137 fn test_windows_guest_ram_hotplug() { 7138 let windows_guest = WindowsGuest::new(); 7139 7140 let mut ovmf_path = dirs::home_dir().unwrap(); 7141 ovmf_path.push("workloads"); 7142 ovmf_path.push(OVMF_NAME); 7143 7144 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7145 let api_socket = temp_api_path(&tmp_dir); 7146 7147 let mut child = GuestCommand::new(windows_guest.guest()) 7148 .args(&["--api-socket", &api_socket]) 7149 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7150 .args(&["--memory", "size=2G,hotplug_size=5G"]) 7151 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7152 .args(&["--serial", "tty"]) 7153 .args(&["--console", "off"]) 7154 .default_disks() 7155 .default_net() 7156 .capture_output() 7157 .spawn() 7158 .unwrap(); 7159 7160 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7161 7162 let r = std::panic::catch_unwind(|| { 7163 // Wait to make sure Windows boots up 7164 assert!(windows_guest.wait_for_boot()); 7165 7166 let ram_size = 2 * 1024 * 1024 * 1024; 7167 // Check the initial number of RAM the guest sees 7168 let current_ram_size = windows_guest.ram_size(); 7169 // This size seems to be reserved by the system and thus the 7170 // reported amount differs by this constant value. 7171 let reserved_ram_size = ram_size - current_ram_size; 7172 // Verify that there's not more than 4mb constant diff wasted 7173 // by the reserved ram. 7174 assert!(reserved_ram_size < 4 * 1024 * 1024); 7175 7176 let ram_size = 4 * 1024 * 1024 * 1024; 7177 // Hotplug some RAM 7178 resize_command(&api_socket, None, Some(ram_size), None, None); 7179 // Wait to make sure RAM has been added 7180 thread::sleep(std::time::Duration::new(10, 0)); 7181 // Check the guest sees the correct number 7182 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7183 7184 let ram_size = 3 * 1024 * 1024 * 1024; 7185 // Unplug some RAM. Note that hot-remove most likely won't work. 7186 resize_command(&api_socket, None, Some(ram_size), None, None); 7187 // Wait to make sure RAM has been added 7188 thread::sleep(std::time::Duration::new(10, 0)); 7189 // Reboot to let Windows catch up 7190 windows_guest.reboot(); 7191 // Wait to make sure guest completely rebooted 7192 thread::sleep(std::time::Duration::new(60, 0)); 7193 // Check the guest sees the correct number 7194 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7195 7196 windows_guest.shutdown(); 7197 }); 7198 7199 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7200 let _ = child.kill(); 7201 let output = child.wait_with_output().unwrap(); 7202 7203 let _ = child_dnsmasq.kill(); 7204 let _ = child_dnsmasq.wait(); 7205 7206 handle_child_output(r, &output); 7207 } 7208 7209 #[test] 7210 #[cfg(not(feature = "mshv"))] 7211 fn test_windows_guest_netdev_hotplug() { 7212 let windows_guest = WindowsGuest::new(); 7213 7214 let mut ovmf_path = dirs::home_dir().unwrap(); 7215 ovmf_path.push("workloads"); 7216 ovmf_path.push(OVMF_NAME); 7217 7218 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7219 let api_socket = temp_api_path(&tmp_dir); 7220 7221 let mut child = GuestCommand::new(windows_guest.guest()) 7222 .args(&["--api-socket", &api_socket]) 7223 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7224 .args(&["--memory", "size=4G"]) 7225 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7226 .args(&["--serial", "tty"]) 7227 .args(&["--console", "off"]) 7228 .default_disks() 7229 .default_net() 7230 .capture_output() 7231 .spawn() 7232 .unwrap(); 7233 7234 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7235 7236 let r = std::panic::catch_unwind(|| { 7237 // Wait to make sure Windows boots up 7238 assert!(windows_guest.wait_for_boot()); 7239 7240 // Initially present network device 7241 let netdev_num = 1; 7242 assert_eq!(windows_guest.netdev_count(), netdev_num); 7243 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7244 7245 // Hotplug network device 7246 let (cmd_success, cmd_output) = remote_command_w_output( 7247 &api_socket, 7248 "add-net", 7249 Some(windows_guest.guest().default_net_string().as_str()), 7250 ); 7251 assert!(cmd_success); 7252 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 7253 thread::sleep(std::time::Duration::new(5, 0)); 7254 // Verify the device is on the system 7255 let netdev_num = 2; 7256 assert_eq!(windows_guest.netdev_count(), netdev_num); 7257 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7258 7259 // Remove network device 7260 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 7261 assert!(cmd_success); 7262 thread::sleep(std::time::Duration::new(5, 0)); 7263 // Verify the device has been removed 7264 let netdev_num = 1; 7265 assert_eq!(windows_guest.netdev_count(), netdev_num); 7266 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7267 7268 windows_guest.shutdown(); 7269 }); 7270 7271 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7272 let _ = child.kill(); 7273 let output = child.wait_with_output().unwrap(); 7274 7275 let _ = child_dnsmasq.kill(); 7276 let _ = child_dnsmasq.wait(); 7277 7278 handle_child_output(r, &output); 7279 } 7280 7281 #[test] 7282 #[cfg(not(feature = "mshv"))] 7283 fn test_windows_guest_disk_hotplug() { 7284 let windows_guest = WindowsGuest::new(); 7285 7286 let mut ovmf_path = dirs::home_dir().unwrap(); 7287 ovmf_path.push("workloads"); 7288 ovmf_path.push(OVMF_NAME); 7289 7290 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7291 let api_socket = temp_api_path(&tmp_dir); 7292 7293 let mut child = GuestCommand::new(windows_guest.guest()) 7294 .args(&["--api-socket", &api_socket]) 7295 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7296 .args(&["--memory", "size=4G"]) 7297 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7298 .args(&["--serial", "tty"]) 7299 .args(&["--console", "off"]) 7300 .default_disks() 7301 .default_net() 7302 .capture_output() 7303 .spawn() 7304 .unwrap(); 7305 7306 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7307 7308 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 7309 7310 let r = std::panic::catch_unwind(|| { 7311 // Wait to make sure Windows boots up 7312 assert!(windows_guest.wait_for_boot()); 7313 7314 // Initially present disk device 7315 let disk_num = 1; 7316 assert_eq!(windows_guest.disk_count(), disk_num); 7317 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7318 7319 // Hotplug disk device 7320 let (cmd_success, cmd_output) = remote_command_w_output( 7321 &api_socket, 7322 "add-disk", 7323 Some(format!("path={},readonly=off", disk).as_str()), 7324 ); 7325 assert!(cmd_success); 7326 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 7327 thread::sleep(std::time::Duration::new(5, 0)); 7328 // Online disk device 7329 windows_guest.disks_set_rw(); 7330 windows_guest.disks_online(); 7331 // Verify the device is on the system 7332 let disk_num = 2; 7333 assert_eq!(windows_guest.disk_count(), disk_num); 7334 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7335 7336 let data = "hello"; 7337 let fname = "d:\\world"; 7338 windows_guest.disk_file_put(fname, data); 7339 7340 // Unmount disk device 7341 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 7342 assert!(cmd_success); 7343 thread::sleep(std::time::Duration::new(5, 0)); 7344 // Verify the device has been removed 7345 let disk_num = 1; 7346 assert_eq!(windows_guest.disk_count(), disk_num); 7347 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7348 7349 // Remount and check the file exists with the expected contents 7350 let (cmd_success, _cmd_output) = remote_command_w_output( 7351 &api_socket, 7352 "add-disk", 7353 Some(format!("path={},readonly=off", disk).as_str()), 7354 ); 7355 assert!(cmd_success); 7356 thread::sleep(std::time::Duration::new(5, 0)); 7357 let out = windows_guest.disk_file_read(fname); 7358 assert_eq!(data, out.trim()); 7359 7360 // Intentionally no unmount, it'll happen at shutdown. 7361 7362 windows_guest.shutdown(); 7363 }); 7364 7365 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7366 let _ = child.kill(); 7367 let output = child.wait_with_output().unwrap(); 7368 7369 let _ = child_dnsmasq.kill(); 7370 let _ = child_dnsmasq.wait(); 7371 7372 handle_child_output(r, &output); 7373 } 7374 7375 #[test] 7376 #[cfg(not(feature = "mshv"))] 7377 fn test_windows_guest_disk_hotplug_multi() { 7378 let windows_guest = WindowsGuest::new(); 7379 7380 let mut ovmf_path = dirs::home_dir().unwrap(); 7381 ovmf_path.push("workloads"); 7382 ovmf_path.push(OVMF_NAME); 7383 7384 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7385 let api_socket = temp_api_path(&tmp_dir); 7386 7387 let mut child = GuestCommand::new(windows_guest.guest()) 7388 .args(&["--api-socket", &api_socket]) 7389 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7390 .args(&["--memory", "size=2G"]) 7391 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7392 .args(&["--serial", "tty"]) 7393 .args(&["--console", "off"]) 7394 .default_disks() 7395 .default_net() 7396 .capture_output() 7397 .spawn() 7398 .unwrap(); 7399 7400 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7401 7402 // Predefined data to used at various test stages 7403 let disk_test_data: [[String; 4]; 2] = [ 7404 [ 7405 "_disk2".to_string(), 7406 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 7407 "d:\\world".to_string(), 7408 "hello".to_string(), 7409 ], 7410 [ 7411 "_disk3".to_string(), 7412 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 7413 "e:\\hello".to_string(), 7414 "world".to_string(), 7415 ], 7416 ]; 7417 7418 let r = std::panic::catch_unwind(|| { 7419 // Wait to make sure Windows boots up 7420 assert!(windows_guest.wait_for_boot()); 7421 7422 // Initially present disk device 7423 let disk_num = 1; 7424 assert_eq!(windows_guest.disk_count(), disk_num); 7425 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7426 7427 for it in &disk_test_data { 7428 let disk_id = it[0].as_str(); 7429 let disk = it[1].as_str(); 7430 // Hotplug disk device 7431 let (cmd_success, cmd_output) = remote_command_w_output( 7432 &api_socket, 7433 "add-disk", 7434 Some(format!("path={},readonly=off", disk).as_str()), 7435 ); 7436 assert!(cmd_success); 7437 assert!(String::from_utf8_lossy(&cmd_output) 7438 .contains(format!("\"id\":\"{}\"", disk_id).as_str())); 7439 thread::sleep(std::time::Duration::new(5, 0)); 7440 // Online disk devices 7441 windows_guest.disks_set_rw(); 7442 windows_guest.disks_online(); 7443 } 7444 // Verify the devices are on the system 7445 let disk_num = (disk_test_data.len() + 1) as u8; 7446 assert_eq!(windows_guest.disk_count(), disk_num); 7447 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7448 7449 // Put test data 7450 for it in &disk_test_data { 7451 let fname = it[2].as_str(); 7452 let data = it[3].as_str(); 7453 windows_guest.disk_file_put(fname, data); 7454 } 7455 7456 // Unmount disk devices 7457 for it in &disk_test_data { 7458 let disk_id = it[0].as_str(); 7459 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 7460 assert!(cmd_success); 7461 thread::sleep(std::time::Duration::new(5, 0)); 7462 } 7463 7464 // Verify the devices have been removed 7465 let disk_num = 1; 7466 assert_eq!(windows_guest.disk_count(), disk_num); 7467 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7468 7469 // Remount 7470 for it in &disk_test_data { 7471 let disk = it[1].as_str(); 7472 let (cmd_success, _cmd_output) = remote_command_w_output( 7473 &api_socket, 7474 "add-disk", 7475 Some(format!("path={},readonly=off", disk).as_str()), 7476 ); 7477 assert!(cmd_success); 7478 thread::sleep(std::time::Duration::new(5, 0)); 7479 } 7480 7481 // Check the files exists with the expected contents 7482 for it in &disk_test_data { 7483 let fname = it[2].as_str(); 7484 let data = it[3].as_str(); 7485 let out = windows_guest.disk_file_read(fname); 7486 assert_eq!(data, out.trim()); 7487 } 7488 7489 // Intentionally no unmount, it'll happen at shutdown. 7490 7491 windows_guest.shutdown(); 7492 }); 7493 7494 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7495 let _ = child.kill(); 7496 let output = child.wait_with_output().unwrap(); 7497 7498 let _ = child_dnsmasq.kill(); 7499 let _ = child_dnsmasq.wait(); 7500 7501 handle_child_output(r, &output); 7502 } 7503 7504 #[test] 7505 #[cfg(not(feature = "mshv"))] 7506 fn test_windows_guest_netdev_multi() { 7507 let windows_guest = WindowsGuest::new(); 7508 7509 let mut ovmf_path = dirs::home_dir().unwrap(); 7510 ovmf_path.push("workloads"); 7511 ovmf_path.push(OVMF_NAME); 7512 7513 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7514 let api_socket = temp_api_path(&tmp_dir); 7515 7516 let mut child = GuestCommand::new(windows_guest.guest()) 7517 .args(&["--api-socket", &api_socket]) 7518 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7519 .args(&["--memory", "size=4G"]) 7520 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7521 .args(&["--serial", "tty"]) 7522 .args(&["--console", "off"]) 7523 .default_disks() 7524 // The multi net dev config is borrowed from test_multiple_network_interfaces 7525 .args(&[ 7526 "--net", 7527 windows_guest.guest().default_net_string().as_str(), 7528 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 7529 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 7530 ]) 7531 .capture_output() 7532 .spawn() 7533 .unwrap(); 7534 7535 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7536 7537 let r = std::panic::catch_unwind(|| { 7538 // Wait to make sure Windows boots up 7539 assert!(windows_guest.wait_for_boot()); 7540 7541 let netdev_num = 3; 7542 assert_eq!(windows_guest.netdev_count(), netdev_num); 7543 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7544 7545 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 7546 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 7547 7548 windows_guest.shutdown(); 7549 }); 7550 7551 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7552 let _ = child.kill(); 7553 let output = child.wait_with_output().unwrap(); 7554 7555 let _ = child_dnsmasq.kill(); 7556 let _ = child_dnsmasq.wait(); 7557 7558 handle_child_output(r, &output); 7559 } 7560 } 7561 7562 #[cfg(target_arch = "x86_64")] 7563 mod sgx { 7564 use crate::*; 7565 7566 #[test] 7567 fn test_sgx() { 7568 let focal = UbuntuDiskConfig::new(FOCAL_SGX_IMAGE_NAME.to_string()); 7569 let guest = Guest::new(Box::new(focal)); 7570 let mut workload_path = dirs::home_dir().unwrap(); 7571 workload_path.push("workloads"); 7572 7573 let mut kernel_path = workload_path; 7574 kernel_path.push("vmlinux_w_sgx"); 7575 7576 let mut child = GuestCommand::new(&guest) 7577 .args(&["--cpus", "boot=1"]) 7578 .args(&["--memory", "size=512M"]) 7579 .args(&["--kernel", kernel_path.to_str().unwrap()]) 7580 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7581 .default_disks() 7582 .default_net() 7583 .args(&["--sgx-epc", "id=epc0,size=64M"]) 7584 .capture_output() 7585 .spawn() 7586 .unwrap(); 7587 7588 let r = std::panic::catch_unwind(|| { 7589 guest.wait_vm_boot(None).unwrap(); 7590 7591 // Check if SGX is correctly detected in the guest. 7592 guest.check_sgx_support().unwrap(); 7593 7594 // Validate the SGX EPC section is 64MiB. 7595 assert_eq!( 7596 guest 7597 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 7598 .unwrap() 7599 .trim(), 7600 "0x0000000004000000" 7601 ); 7602 7603 // Run a test relying on SGX enclaves and check if it runs 7604 // successfully. 7605 assert!(guest 7606 .ssh_command("cd /linux-sgx/SampleCode/LocalAttestation/bin/ && sudo ./app") 7607 .unwrap() 7608 .trim() 7609 .contains( 7610 "succeed to load enclaves.\nsucceed to \ 7611 establish secure channel.\nSucceed to exchange \ 7612 secure message...\nSucceed to close Session..." 7613 )); 7614 }); 7615 7616 let _ = child.kill(); 7617 let output = child.wait_with_output().unwrap(); 7618 7619 handle_child_output(r, &output); 7620 } 7621 } 7622 7623 #[cfg(target_arch = "x86_64")] 7624 mod vfio { 7625 use crate::*; 7626 7627 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 7628 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 7629 let guest = Guest::new(Box::new(hirsute)); 7630 let api_socket = temp_api_path(&guest.tmp_dir); 7631 7632 let mut child = GuestCommand::new(&guest) 7633 .args(&["--cpus", "boot=4"]) 7634 .args(&[ 7635 "--memory", 7636 format!("size=4G,hotplug_size=4G,hotplug_method={}", hotplug_method).as_str(), 7637 ]) 7638 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7639 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 7640 .args(&["--api-socket", &api_socket]) 7641 .default_disks() 7642 .default_net() 7643 .capture_output() 7644 .spawn() 7645 .unwrap(); 7646 7647 let r = std::panic::catch_unwind(|| { 7648 guest.wait_vm_boot(None).unwrap(); 7649 7650 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 7651 7652 guest.enable_memory_hotplug(); 7653 7654 // Add RAM to the VM 7655 let desired_ram = 6 << 30; 7656 resize_command(&api_socket, None, Some(desired_ram), None, None); 7657 thread::sleep(std::time::Duration::new(30, 0)); 7658 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7659 7660 // Check the VFIO device works when RAM is increased to 6GiB 7661 guest.check_nvidia_gpu(); 7662 }); 7663 7664 let _ = child.kill(); 7665 let output = child.wait_with_output().unwrap(); 7666 7667 handle_child_output(r, &output); 7668 } 7669 7670 #[test] 7671 fn test_nvidia_card_memory_hotplug_acpi() { 7672 test_nvidia_card_memory_hotplug("acpi") 7673 } 7674 7675 #[test] 7676 fn test_nvidia_card_memory_hotplug_virtio_mem() { 7677 test_nvidia_card_memory_hotplug("virtio-mem") 7678 } 7679 7680 #[test] 7681 fn test_nvidia_card_pci_hotplug() { 7682 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 7683 let guest = Guest::new(Box::new(hirsute)); 7684 let api_socket = temp_api_path(&guest.tmp_dir); 7685 7686 let mut child = GuestCommand::new(&guest) 7687 .args(&["--cpus", "boot=4"]) 7688 .args(&["--memory", "size=4G"]) 7689 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7690 .args(&["--api-socket", &api_socket]) 7691 .default_disks() 7692 .default_net() 7693 .capture_output() 7694 .spawn() 7695 .unwrap(); 7696 7697 let r = std::panic::catch_unwind(|| { 7698 guest.wait_vm_boot(None).unwrap(); 7699 7700 // Hotplug the card to the VM 7701 let (cmd_success, cmd_output) = remote_command_w_output( 7702 &api_socket, 7703 "add-device", 7704 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 7705 ); 7706 assert!(cmd_success); 7707 assert!(String::from_utf8_lossy(&cmd_output) 7708 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 7709 7710 thread::sleep(std::time::Duration::new(10, 0)); 7711 7712 // Check the VFIO device works after hotplug 7713 guest.check_nvidia_gpu(); 7714 }); 7715 7716 let _ = child.kill(); 7717 let output = child.wait_with_output().unwrap(); 7718 7719 handle_child_output(r, &output); 7720 } 7721 7722 #[test] 7723 fn test_nvidia_card_reboot() { 7724 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 7725 let guest = Guest::new(Box::new(hirsute)); 7726 let api_socket = temp_api_path(&guest.tmp_dir); 7727 7728 let mut child = GuestCommand::new(&guest) 7729 .args(&["--cpus", "boot=4"]) 7730 .args(&["--memory", "size=4G"]) 7731 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7732 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 7733 .args(&["--api-socket", &api_socket]) 7734 .default_disks() 7735 .default_net() 7736 .capture_output() 7737 .spawn() 7738 .unwrap(); 7739 7740 let r = std::panic::catch_unwind(|| { 7741 guest.wait_vm_boot(None).unwrap(); 7742 7743 // Check the VFIO device works after boot 7744 guest.check_nvidia_gpu(); 7745 7746 guest.reboot_linux(0, None); 7747 7748 // Check the VFIO device works after reboot 7749 guest.check_nvidia_gpu(); 7750 }); 7751 7752 let _ = child.kill(); 7753 let output = child.wait_with_output().unwrap(); 7754 7755 handle_child_output(r, &output); 7756 } 7757 } 7758 7759 mod live_migration { 7760 use crate::*; 7761 7762 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 7763 // same host. It ensures the following behaviors: 7764 // 1. The source VM is up and functional (including various virtio-devices are working properly); 7765 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 7766 // 3. The source VM terminated gracefully after live migration; 7767 // 4. The destination VM is functional (including various virtio-devices are working properly) after 7768 // live migration; 7769 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 7770 fn _test_live_migration( 7771 upgrade_test: bool, 7772 numa: bool, 7773 local: bool, 7774 watchdog: bool, 7775 balloon: bool, 7776 ) { 7777 assert!( 7778 !(numa && balloon), 7779 "Invalid inputs: 'numa' and 'balloon' cannot be tested at the same time." 7780 ); 7781 7782 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7783 let guest = Guest::new(Box::new(focal)); 7784 let kernel_path = direct_kernel_boot_path(); 7785 let console_text = String::from("On a branch floating down river a cricket, singing."); 7786 let net_id = "net123"; 7787 let net_params = format!( 7788 "id={},tap=,mac={},ip={},mask=255.255.255.0", 7789 net_id, guest.network.guest_mac, guest.network.host_ip 7790 ); 7791 7792 let memory_param: &[&str] = match (balloon, local, numa) { 7793 (false, false, false) => &["--memory", "size=4G"], 7794 (false, false, true) => &[ 7795 "--memory", 7796 "size=0,hotplug_method=virtio-mem", 7797 "--memory-zone", 7798 "id=mem0,size=1G,hotplug_size=4G", 7799 "id=mem1,size=1G,hotplug_size=4G", 7800 "id=mem2,size=2G,hotplug_size=4G", 7801 "--numa", 7802 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 7803 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 7804 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 7805 ], 7806 (false, true, false) => &["--memory", "size=4G,shared=on"], 7807 (false, true, true) => &[ 7808 "--memory", 7809 "size=0,hotplug_method=virtio-mem,shared=on", 7810 "--memory-zone", 7811 "id=mem0,size=1G,hotplug_size=4G,shared=on", 7812 "id=mem1,size=1G,hotplug_size=4G,shared=on", 7813 "id=mem2,size=2G,hotplug_size=4G,shared=on", 7814 "--numa", 7815 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 7816 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 7817 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 7818 ], 7819 (true, false, _) => &[ 7820 "--memory", 7821 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 7822 "--balloon", 7823 "size=0", 7824 ], 7825 (true, true, _) => &[ 7826 "--memory", 7827 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 7828 "--balloon", 7829 "size=0", 7830 ], 7831 }; 7832 7833 let pmem_temp_file = TempFile::new().unwrap(); 7834 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 7835 std::process::Command::new("mkfs.ext4") 7836 .arg(pmem_temp_file.as_path()) 7837 .output() 7838 .expect("Expect creating disk image to succeed"); 7839 let pmem_path = String::from("/dev/pmem0"); 7840 7841 // Start the source VM 7842 let src_vm_path = if !upgrade_test { 7843 clh_command("cloud-hypervisor") 7844 } else { 7845 cloud_hypervisor_release_path() 7846 }; 7847 let src_api_socket = temp_api_path(&guest.tmp_dir); 7848 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 7849 src_vm_cmd 7850 .args(&["--cpus", "boot=6,max=12"]) 7851 .args(memory_param) 7852 .args(&["--kernel", kernel_path.to_str().unwrap()]) 7853 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7854 .default_disks() 7855 .args(&["--net", net_params.as_str()]) 7856 .args(&["--api-socket", &src_api_socket]) 7857 .args(&[ 7858 "--pmem", 7859 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 7860 ]); 7861 if watchdog { 7862 src_vm_cmd.args(&["--watchdog"]); 7863 } 7864 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 7865 7866 // Start the destination VM 7867 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 7868 dest_api_socket.push_str(".dest"); 7869 let mut dest_child = GuestCommand::new(&guest) 7870 .args(&["--api-socket", &dest_api_socket]) 7871 .capture_output() 7872 .spawn() 7873 .unwrap(); 7874 7875 let r = std::panic::catch_unwind(|| { 7876 guest.wait_vm_boot(None).unwrap(); 7877 7878 // Make sure the source VM is functaionl 7879 // Check the number of vCPUs 7880 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 6); 7881 7882 // Check the guest RAM 7883 if balloon { 7884 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 7885 // Increase the guest RAM 7886 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 7887 thread::sleep(std::time::Duration::new(5, 0)); 7888 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7889 // Use balloon to remove RAM from the VM 7890 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 7891 thread::sleep(std::time::Duration::new(5, 0)); 7892 let total_memory = guest.get_total_memory().unwrap_or_default(); 7893 assert!(total_memory > 4_800_000); 7894 assert!(total_memory < 5_760_000); 7895 } else if numa { 7896 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 7897 } else { 7898 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 7899 } 7900 7901 // Check the guest virtio-devices, e.g. block, rng, console, and net 7902 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 7903 7904 // Check the NUMA parameters are applied correctly and resize 7905 // each zone to test the case where we migrate a VM with the 7906 // virtio-mem regions being used. 7907 if numa { 7908 guest.check_numa_common( 7909 Some(&[960_000, 960_000, 1_920_000]), 7910 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 7911 Some(&["10 15 20", "20 10 25", "25 30 10"]), 7912 ); 7913 7914 // AArch64 currently does not support hotplug, and therefore we only 7915 // test hotplug-related function on x86_64 here. 7916 #[cfg(target_arch = "x86_64")] 7917 { 7918 guest.enable_memory_hotplug(); 7919 7920 // Resize every memory zone and check each associated NUMA node 7921 // has been assigned the right amount of memory. 7922 resize_zone_command(&src_api_socket, "mem0", "2G"); 7923 resize_zone_command(&src_api_socket, "mem1", "2G"); 7924 resize_zone_command(&src_api_socket, "mem2", "3G"); 7925 thread::sleep(std::time::Duration::new(5, 0)); 7926 7927 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 7928 } 7929 } 7930 7931 // x86_64: Following what's done in the `test_snapshot_restore`, we need 7932 // to make sure that removing and adding back the virtio-net device does 7933 // not break the live-migration support for virtio-pci. 7934 #[cfg(target_arch = "x86_64")] 7935 { 7936 assert!(remote_command( 7937 &src_api_socket, 7938 "remove-device", 7939 Some(net_id), 7940 )); 7941 thread::sleep(std::time::Duration::new(10, 0)); 7942 7943 // Plug the virtio-net device again 7944 assert!(remote_command( 7945 &src_api_socket, 7946 "add-net", 7947 Some(net_params.as_str()), 7948 )); 7949 thread::sleep(std::time::Duration::new(10, 0)); 7950 } 7951 7952 if watchdog { 7953 let mut current_reboot_count = 1; 7954 enable_guest_watchdog(&guest, &mut current_reboot_count); 7955 check_guest_watchdog_no_reboot(&guest, ¤t_reboot_count); 7956 } 7957 7958 // Start the live-migration 7959 let migration_socket = String::from( 7960 guest 7961 .tmp_dir 7962 .as_path() 7963 .join("live-migration.sock") 7964 .to_str() 7965 .unwrap(), 7966 ); 7967 // Start to receive migration from the destintion VM 7968 let mut receive_migration = Command::new(clh_command("ch-remote")) 7969 .args(&[ 7970 &format!("--api-socket={}", &dest_api_socket), 7971 "receive-migration", 7972 &format! {"unix:{}", migration_socket}, 7973 ]) 7974 .stderr(Stdio::piped()) 7975 .stdout(Stdio::piped()) 7976 .spawn() 7977 .unwrap(); 7978 // Give it '1s' to make sure the 'migration_socket' file is properly created 7979 thread::sleep(std::time::Duration::new(1, 0)); 7980 // Start to send migration from the source VM 7981 7982 let mut args = [ 7983 format!("--api-socket={}", &src_api_socket), 7984 "send-migration".to_string(), 7985 format! {"unix:{}", migration_socket}, 7986 ] 7987 .to_vec(); 7988 7989 if local { 7990 args.insert(2, "--local".to_string()); 7991 } 7992 7993 let mut send_migration = Command::new(clh_command("ch-remote")) 7994 .args(&args) 7995 .stderr(Stdio::piped()) 7996 .stdout(Stdio::piped()) 7997 .spawn() 7998 .unwrap(); 7999 8000 // The 'send-migration' command should be executed successfully within the given timeout 8001 let success = if let Some(status) = send_migration 8002 .wait_timeout(std::time::Duration::from_secs(30)) 8003 .unwrap() 8004 { 8005 status.success() 8006 } else { 8007 false 8008 }; 8009 8010 if !success { 8011 let _ = send_migration.kill(); 8012 let output = send_migration.wait_with_output().unwrap(); 8013 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 8014 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8015 } 8016 8017 // The 'receive-migration' command should be executed successfully within the given timeout 8018 let success = if let Some(status) = receive_migration 8019 .wait_timeout(std::time::Duration::from_secs(30)) 8020 .unwrap() 8021 { 8022 status.success() 8023 } else { 8024 false 8025 }; 8026 8027 if !success { 8028 let _ = receive_migration.kill(); 8029 let output = receive_migration.wait_with_output().unwrap(); 8030 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 8031 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8032 } 8033 assert!( 8034 success, 8035 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8036 ); 8037 }); 8038 8039 let print_and_panic = |src_vm: Child, dest_vm: Child, message: &str| -> ! { 8040 let mut src_vm = src_vm; 8041 let mut dest_vm = dest_vm; 8042 8043 let _ = src_vm.kill(); 8044 let src_output = src_vm.wait_with_output().unwrap(); 8045 eprintln!( 8046 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8047 String::from_utf8_lossy(&src_output.stdout) 8048 ); 8049 eprintln!( 8050 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8051 String::from_utf8_lossy(&src_output.stderr) 8052 ); 8053 let _ = dest_vm.kill(); 8054 let dest_output = dest_vm.wait_with_output().unwrap(); 8055 eprintln!( 8056 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8057 String::from_utf8_lossy(&dest_output.stdout) 8058 ); 8059 eprintln!( 8060 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8061 String::from_utf8_lossy(&dest_output.stderr) 8062 ); 8063 8064 panic!("Test failed: {}", message) 8065 }; 8066 8067 // Check and report any errors occured during the live-migration 8068 if r.is_err() { 8069 print_and_panic(src_child, dest_child, "Error occured during live-migration"); 8070 } 8071 8072 // Check the source vm has been terminated successful (give it '3s' to settle) 8073 thread::sleep(std::time::Duration::new(3, 0)); 8074 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8075 print_and_panic( 8076 src_child, 8077 dest_child, 8078 "source VM was not terminated successfully.", 8079 ); 8080 }; 8081 8082 // Post live-migration check to make sure the destination VM is funcational 8083 let r = std::panic::catch_unwind(|| { 8084 // Perform same checks to validate VM has been properly migrated 8085 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 6); 8086 #[cfg(target_arch = "x86_64")] 8087 if numa { 8088 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 8089 } else { 8090 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8091 } 8092 #[cfg(target_arch = "aarch64")] 8093 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8094 8095 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8096 8097 // Perform NUMA related checks 8098 if numa { 8099 #[cfg(target_arch = "aarch64")] 8100 { 8101 guest.check_numa_common( 8102 Some(&[960_000, 960_000, 1_920_000]), 8103 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8104 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8105 ); 8106 } 8107 8108 // AArch64 currently does not support hotplug, and therefore we only 8109 // test hotplug-related function on x86_64 here. 8110 #[cfg(target_arch = "x86_64")] 8111 { 8112 guest.check_numa_common( 8113 Some(&[1_920_000, 1_920_000, 2_880_000]), 8114 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8115 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8116 ); 8117 8118 guest.enable_memory_hotplug(); 8119 8120 // Resize every memory zone and check each associated NUMA node 8121 // has been assigned the right amount of memory. 8122 resize_zone_command(&dest_api_socket, "mem0", "4G"); 8123 resize_zone_command(&dest_api_socket, "mem1", "4G"); 8124 resize_zone_command(&dest_api_socket, "mem2", "4G"); 8125 // Resize to the maximum amount of CPUs and check each NUMA 8126 // node has been assigned the right CPUs set. 8127 resize_command(&dest_api_socket, Some(12), None, None, None); 8128 thread::sleep(std::time::Duration::new(5, 0)); 8129 8130 guest.check_numa_common( 8131 Some(&[3_840_000, 3_840_000, 3_840_000]), 8132 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 8133 None, 8134 ); 8135 } 8136 } 8137 8138 if watchdog { 8139 let mut current_reboot_count = 2; 8140 check_guest_watchdog_no_reboot(&guest, ¤t_reboot_count); 8141 check_guest_watchdog_one_reboot( 8142 &guest, 8143 &dest_api_socket, 8144 &mut current_reboot_count, 8145 ); 8146 } 8147 8148 if balloon { 8149 let total_memory = guest.get_total_memory().unwrap_or_default(); 8150 assert!(total_memory > 4_800_000); 8151 assert!(total_memory < 5_760_000); 8152 // Deflate balloon to restore entire RAM to the VM 8153 resize_command(&dest_api_socket, None, None, Some(0), None); 8154 thread::sleep(std::time::Duration::new(5, 0)); 8155 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8156 // Decrease guest RAM with virtio-mem 8157 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 8158 thread::sleep(std::time::Duration::new(5, 0)); 8159 let total_memory = guest.get_total_memory().unwrap_or_default(); 8160 assert!(total_memory > 4_800_000); 8161 assert!(total_memory < 5_760_000); 8162 } 8163 }); 8164 8165 // Clean-up the destination VM and make sure it terminated correctly 8166 let _ = dest_child.kill(); 8167 let dest_output = dest_child.wait_with_output().unwrap(); 8168 handle_child_output(r, &dest_output); 8169 8170 // Check the destination VM has the expected 'concole_text' from its output 8171 let r = std::panic::catch_unwind(|| { 8172 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8173 }); 8174 handle_child_output(r, &dest_output); 8175 } 8176 8177 #[test] 8178 fn test_live_migration_basic() { 8179 _test_live_migration(false, false, false, false, false) 8180 } 8181 8182 #[test] 8183 fn test_live_migration_local() { 8184 _test_live_migration(false, false, true, false, false) 8185 } 8186 8187 #[test] 8188 #[cfg(not(feature = "mshv"))] 8189 fn test_live_migration_numa() { 8190 _test_live_migration(false, true, false, false, false) 8191 } 8192 8193 #[test] 8194 #[cfg(not(feature = "mshv"))] 8195 fn test_live_migration_numa_local() { 8196 _test_live_migration(false, true, true, false, false) 8197 } 8198 8199 #[test] 8200 fn test_live_migration_watchdog() { 8201 _test_live_migration(false, false, false, true, false) 8202 } 8203 8204 #[test] 8205 fn test_live_migration_watchdog_local() { 8206 _test_live_migration(false, false, true, true, false) 8207 } 8208 8209 #[test] 8210 fn test_live_migration_balloon() { 8211 _test_live_migration(false, false, false, false, true) 8212 } 8213 8214 #[test] 8215 fn test_live_migration_balloon_local() { 8216 _test_live_migration(false, false, true, false, true) 8217 } 8218 8219 #[test] 8220 #[ignore] 8221 fn test_live_upgrade_basic() { 8222 _test_live_migration(true, false, false, false, false) 8223 } 8224 8225 #[test] 8226 #[ignore] 8227 fn test_live_upgrade_local() { 8228 _test_live_migration(true, false, true, false, false) 8229 } 8230 8231 #[test] 8232 #[ignore] 8233 #[cfg(not(feature = "mshv"))] 8234 fn test_live_upgrade_numa() { 8235 _test_live_migration(true, true, false, false, false) 8236 } 8237 8238 #[test] 8239 #[ignore] 8240 #[cfg(not(feature = "mshv"))] 8241 fn test_live_upgrade_numa_local() { 8242 _test_live_migration(true, true, true, false, false) 8243 } 8244 8245 #[test] 8246 #[ignore] 8247 fn test_live_upgrade_watchdog() { 8248 _test_live_migration(true, false, false, true, false) 8249 } 8250 8251 #[test] 8252 #[ignore] 8253 fn test_live_upgrade_watchdog_local() { 8254 _test_live_migration(true, false, true, true, false) 8255 } 8256 8257 #[test] 8258 #[ignore] 8259 fn test_live_upgrade_balloon() { 8260 _test_live_migration(true, false, false, false, true) 8261 } 8262 8263 #[test] 8264 #[ignore] 8265 fn test_live_upgrade_balloon_local() { 8266 _test_live_migration(true, false, true, false, true) 8267 } 8268 8269 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 8270 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8271 let ovs_guest = Guest::new(Box::new(ovs_focal)); 8272 8273 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8274 let migration_guest = Guest::new(Box::new(migration_focal)); 8275 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 8276 8277 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 8278 let (mut ovs_child, mut src_child) = 8279 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 8280 8281 // Start the destination VM 8282 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 8283 dest_api_socket.push_str(".dest"); 8284 let mut dest_child = GuestCommand::new(&migration_guest) 8285 .args(&["--api-socket", &dest_api_socket]) 8286 .capture_output() 8287 .spawn() 8288 .unwrap(); 8289 8290 let r = std::panic::catch_unwind(|| { 8291 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 8292 thread::sleep(std::time::Duration::new(1, 0)); 8293 8294 // Start the live-migration 8295 let migration_socket = String::from( 8296 migration_guest 8297 .tmp_dir 8298 .as_path() 8299 .join("live-migration.sock") 8300 .to_str() 8301 .unwrap(), 8302 ); 8303 // Start to receive migration from the destintion VM 8304 let mut receive_migration = Command::new(clh_command("ch-remote")) 8305 .args(&[ 8306 &format!("--api-socket={}", &dest_api_socket), 8307 "receive-migration", 8308 &format! {"unix:{}", migration_socket}, 8309 ]) 8310 .stderr(Stdio::piped()) 8311 .stdout(Stdio::piped()) 8312 .spawn() 8313 .unwrap(); 8314 // Give it '1s' to make sure the 'migration_socket' file is properly created 8315 thread::sleep(std::time::Duration::new(1, 0)); 8316 // Start to send migration from the source VM 8317 let mut args = [ 8318 format!("--api-socket={}", &src_api_socket), 8319 "send-migration".to_string(), 8320 format! {"unix:{}", migration_socket}, 8321 ] 8322 .to_vec(); 8323 8324 if local { 8325 args.insert(2, "--local".to_string()); 8326 } 8327 8328 let mut send_migration = Command::new(clh_command("ch-remote")) 8329 .args(&args) 8330 .stderr(Stdio::piped()) 8331 .stdout(Stdio::piped()) 8332 .spawn() 8333 .unwrap(); 8334 8335 // The 'send-migration' command should be executed successfully within the given timeout 8336 let success = if let Some(status) = send_migration 8337 .wait_timeout(std::time::Duration::from_secs(30)) 8338 .unwrap() 8339 { 8340 status.success() 8341 } else { 8342 false 8343 }; 8344 8345 if !success { 8346 let _ = send_migration.kill(); 8347 let output = send_migration.wait_with_output().unwrap(); 8348 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 8349 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8350 } 8351 8352 // The 'receive-migration' command should be executed successfully within the given timeout 8353 let success = if let Some(status) = receive_migration 8354 .wait_timeout(std::time::Duration::from_secs(30)) 8355 .unwrap() 8356 { 8357 status.success() 8358 } else { 8359 false 8360 }; 8361 8362 if !success { 8363 let _ = receive_migration.kill(); 8364 let output = receive_migration.wait_with_output().unwrap(); 8365 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 8366 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8367 } 8368 assert!( 8369 success, 8370 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8371 ); 8372 }); 8373 8374 let print_and_panic = |src_vm: Child, dest_vm: Child, ovs_vm: Child, message: &str| -> ! { 8375 let mut src_vm = src_vm; 8376 let mut dest_vm = dest_vm; 8377 let mut ovs_vm = ovs_vm; 8378 8379 let _ = src_vm.kill(); 8380 let src_output = src_vm.wait_with_output().unwrap(); 8381 eprintln!( 8382 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8383 String::from_utf8_lossy(&src_output.stdout) 8384 ); 8385 eprintln!( 8386 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8387 String::from_utf8_lossy(&src_output.stderr) 8388 ); 8389 let _ = dest_vm.kill(); 8390 let dest_output = dest_vm.wait_with_output().unwrap(); 8391 eprintln!( 8392 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8393 String::from_utf8_lossy(&dest_output.stdout) 8394 ); 8395 eprintln!( 8396 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8397 String::from_utf8_lossy(&dest_output.stderr) 8398 ); 8399 let _ = ovs_vm.kill(); 8400 let ovs_output = ovs_vm.wait_with_output().unwrap(); 8401 eprintln!( 8402 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 8403 String::from_utf8_lossy(&ovs_output.stdout) 8404 ); 8405 eprintln!( 8406 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 8407 String::from_utf8_lossy(&ovs_output.stderr) 8408 ); 8409 8410 cleanup_ovs_dpdk(); 8411 8412 panic!("Test failed: {}", message) 8413 }; 8414 8415 // Check and report any errors occured during the live-migration 8416 if r.is_err() { 8417 print_and_panic( 8418 src_child, 8419 dest_child, 8420 ovs_child, 8421 "Error occured during live-migration", 8422 ); 8423 } 8424 8425 // Check the source vm has been terminated successful (give it '3s' to settle) 8426 thread::sleep(std::time::Duration::new(3, 0)); 8427 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8428 print_and_panic( 8429 src_child, 8430 dest_child, 8431 ovs_child, 8432 "source VM was not terminated successfully.", 8433 ); 8434 }; 8435 8436 // Post live-migration check to make sure the destination VM is funcational 8437 let r = std::panic::catch_unwind(|| { 8438 // Perform same checks to validate VM has been properly migrated 8439 // Spawn a new netcat listener in the OVS VM 8440 let guest_ip = ovs_guest.network.guest_ip.clone(); 8441 thread::spawn(move || { 8442 ssh_command_ip( 8443 "nc -l 12345", 8444 &guest_ip, 8445 DEFAULT_SSH_RETRIES, 8446 DEFAULT_SSH_TIMEOUT, 8447 ) 8448 .unwrap(); 8449 }); 8450 8451 // Wait for the server to be listening 8452 thread::sleep(std::time::Duration::new(5, 0)); 8453 8454 // And check the connection is still functional after live-migration 8455 migration_guest 8456 .ssh_command("nc -vz 172.100.0.1 12345") 8457 .unwrap(); 8458 8459 cleanup_ovs_dpdk(); 8460 }); 8461 8462 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 8463 let _ = dest_child.kill(); 8464 let _ = ovs_child.kill(); 8465 let dest_output = dest_child.wait_with_output().unwrap(); 8466 handle_child_output(r, &dest_output); 8467 let ovs_output = ovs_child.wait_with_output().unwrap(); 8468 handle_child_output(Ok(()), &ovs_output); 8469 } 8470 8471 #[test] 8472 #[cfg(not(feature = "mshv"))] 8473 fn test_live_migration_ovs_dpdk() { 8474 _test_live_migration_ovs_dpdk(false, false); 8475 } 8476 8477 #[test] 8478 #[cfg(not(feature = "mshv"))] 8479 fn test_live_migration_ovs_dpdk_local() { 8480 _test_live_migration_ovs_dpdk(false, true); 8481 } 8482 8483 #[test] 8484 #[ignore] 8485 #[cfg(not(feature = "mshv"))] 8486 fn test_live_upgrade_ovs_dpdk() { 8487 _test_live_migration_ovs_dpdk(true, false); 8488 } 8489 8490 #[test] 8491 #[ignore] 8492 #[cfg(not(feature = "mshv"))] 8493 fn test_live_upgrade_ovs_dpdk_local() { 8494 _test_live_migration_ovs_dpdk(true, true); 8495 } 8496 } 8497 8498 #[cfg(target_arch = "aarch64")] 8499 mod aarch64_acpi { 8500 use crate::*; 8501 8502 #[test] 8503 fn test_simple_launch_acpi() { 8504 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8505 8506 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 8507 let guest = Guest::new(disk_config); 8508 8509 let mut child = GuestCommand::new(&guest) 8510 .args(&["--cpus", "boot=1"]) 8511 .args(&["--memory", "size=512M"]) 8512 .args(&["--kernel", edk2_path().to_str().unwrap()]) 8513 .default_disks() 8514 .default_net() 8515 .args(&["--serial", "tty", "--console", "off"]) 8516 .capture_output() 8517 .spawn() 8518 .unwrap(); 8519 8520 let r = std::panic::catch_unwind(|| { 8521 guest.wait_vm_boot(Some(120)).unwrap(); 8522 8523 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 8524 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 8525 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 8526 }); 8527 8528 let _ = child.kill(); 8529 let output = child.wait_with_output().unwrap(); 8530 8531 handle_child_output(r, &output); 8532 }); 8533 } 8534 8535 #[test] 8536 fn test_guest_numa_nodes_acpi() { 8537 _test_guest_numa_nodes(true); 8538 } 8539 8540 #[test] 8541 fn test_cpu_topology_421_acpi() { 8542 test_cpu_topology(4, 2, 1, true); 8543 } 8544 8545 #[test] 8546 fn test_cpu_topology_142_acpi() { 8547 test_cpu_topology(1, 4, 2, true); 8548 } 8549 8550 #[test] 8551 fn test_cpu_topology_262_acpi() { 8552 test_cpu_topology(2, 6, 2, true); 8553 } 8554 8555 #[test] 8556 fn test_power_button_acpi() { 8557 _test_power_button(true); 8558 } 8559 8560 #[test] 8561 fn test_virtio_iommu() { 8562 _test_virtio_iommu(true) 8563 } 8564 } 8565