1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 #[cfg(target_arch = "x86_64")] 28 use std::sync::Mutex; 29 use std::thread; 30 use test_infra::*; 31 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 32 use wait_timeout::ChildExt; 33 34 #[cfg(target_arch = "x86_64")] 35 mod x86_64 { 36 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw"; 37 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 38 pub const FOCAL_SGX_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-sgx.raw"; 39 pub const HIRSUTE_NVIDIA_IMAGE_NAME: &str = "hirsute-server-cloudimg-amd64-nvidia.raw"; 40 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 41 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 42 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 43 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20220329-0.raw"; 44 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2019.raw"; 45 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 46 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 47 } 48 49 #[cfg(target_arch = "x86_64")] 50 use x86_64::*; 51 52 #[cfg(target_arch = "aarch64")] 53 mod aarch64 { 54 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 55 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 56 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 57 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 58 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 59 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 60 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 61 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 62 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 63 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 64 } 65 66 #[cfg(target_arch = "aarch64")] 67 use aarch64::*; 68 69 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 70 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 71 72 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 73 74 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 75 let mut workload_path = dirs::home_dir().unwrap(); 76 workload_path.push("workloads"); 77 78 let mut virtiofsd_path = workload_path; 79 virtiofsd_path.push("virtiofsd"); 80 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 81 82 let virtiofsd_socket_path = 83 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 84 85 // Start the daemon 86 let child = Command::new(virtiofsd_path.as_str()) 87 .args(&["--shared-dir", shared_dir]) 88 .args(&["--socket-path", virtiofsd_socket_path.as_str()]) 89 .args(&["--cache", "never"]) 90 .spawn() 91 .unwrap(); 92 93 thread::sleep(std::time::Duration::new(10, 0)); 94 95 (child, virtiofsd_socket_path) 96 } 97 98 fn prepare_vubd( 99 tmp_dir: &TempDir, 100 blk_img: &str, 101 num_queues: usize, 102 rdonly: bool, 103 direct: bool, 104 ) -> (std::process::Child, String) { 105 let mut workload_path = dirs::home_dir().unwrap(); 106 workload_path.push("workloads"); 107 108 let mut blk_file_path = workload_path; 109 blk_file_path.push(blk_img); 110 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 111 112 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 113 114 // Start the daemon 115 let child = Command::new(clh_command("vhost_user_block")) 116 .args(&[ 117 "--block-backend", 118 format!( 119 "path={},socket={},num_queues={},readonly={},direct={}", 120 blk_file_path, vubd_socket_path, num_queues, rdonly, direct 121 ) 122 .as_str(), 123 ]) 124 .spawn() 125 .unwrap(); 126 127 thread::sleep(std::time::Duration::new(10, 0)); 128 129 (child, vubd_socket_path) 130 } 131 132 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 133 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 134 } 135 136 fn temp_api_path(tmp_dir: &TempDir) -> String { 137 String::from( 138 tmp_dir 139 .as_path() 140 .join("cloud-hypervisor.sock") 141 .to_str() 142 .unwrap(), 143 ) 144 } 145 146 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 147 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 148 } 149 150 // Creates the directory and returns the path. 151 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 152 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 153 std::fs::create_dir(&snapshot_dir).unwrap(); 154 snapshot_dir 155 } 156 157 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 158 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 159 vmcore_file 160 } 161 162 // Creates the path for direct kernel boot and return the path. 163 // For x86_64, this function returns the vmlinux kernel path. 164 // For AArch64, this function returns the PE kernel path. 165 fn direct_kernel_boot_path() -> PathBuf { 166 let mut workload_path = dirs::home_dir().unwrap(); 167 workload_path.push("workloads"); 168 169 let mut kernel_path = workload_path; 170 #[cfg(target_arch = "x86_64")] 171 kernel_path.push("vmlinux"); 172 #[cfg(target_arch = "aarch64")] 173 kernel_path.push("Image"); 174 175 kernel_path 176 } 177 178 #[cfg(target_arch = "aarch64")] 179 fn edk2_path() -> PathBuf { 180 let mut workload_path = dirs::home_dir().unwrap(); 181 workload_path.push("workloads"); 182 let mut edk2_path = workload_path; 183 edk2_path.push("CLOUDHV_EFI.fd"); 184 185 edk2_path 186 } 187 188 fn cloud_hypervisor_release_path() -> String { 189 let mut workload_path = dirs::home_dir().unwrap(); 190 workload_path.push("workloads"); 191 192 let mut ch_release_path = workload_path; 193 #[cfg(target_arch = "x86_64")] 194 ch_release_path.push("cloud-hypervisor-static"); 195 #[cfg(target_arch = "aarch64")] 196 ch_release_path.push("cloud-hypervisor-static-aarch64"); 197 198 ch_release_path.into_os_string().into_string().unwrap() 199 } 200 201 fn prepare_vhost_user_net_daemon( 202 tmp_dir: &TempDir, 203 ip: &str, 204 tap: Option<&str>, 205 num_queues: usize, 206 client_mode: bool, 207 ) -> (std::process::Command, String) { 208 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 209 210 // Start the daemon 211 let net_params = if let Some(tap_str) = tap { 212 format!( 213 "tap={},ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 214 tap_str, ip, vunet_socket_path, num_queues, client_mode 215 ) 216 } else { 217 format!( 218 "ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 219 ip, vunet_socket_path, num_queues, client_mode 220 ) 221 }; 222 223 let mut command = Command::new(clh_command("vhost_user_net")); 224 command.args(&["--net-backend", net_params.as_str()]); 225 226 (command, vunet_socket_path) 227 } 228 229 fn curl_command(api_socket: &str, method: &str, url: &str, http_body: Option<&str>) { 230 let mut curl_args: Vec<&str> = ["--unix-socket", api_socket, "-i", "-X", method, url].to_vec(); 231 232 if let Some(body) = http_body { 233 curl_args.push("-H"); 234 curl_args.push("Accept: application/json"); 235 curl_args.push("-H"); 236 curl_args.push("Content-Type: application/json"); 237 curl_args.push("-d"); 238 curl_args.push(body); 239 } 240 241 let status = Command::new("curl") 242 .args(curl_args) 243 .status() 244 .expect("Failed to launch curl command"); 245 246 assert!(status.success()); 247 } 248 249 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 250 let mut cmd = Command::new(clh_command("ch-remote")); 251 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 252 253 if let Some(arg) = arg { 254 cmd.arg(arg); 255 } 256 let output = cmd.output().unwrap(); 257 if output.status.success() { 258 true 259 } else { 260 eprintln!("Error running ch-remote command: {:?}", &cmd); 261 let stderr = String::from_utf8_lossy(&output.stderr); 262 eprintln!("stderr: {}", stderr); 263 false 264 } 265 } 266 267 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 268 let mut cmd = Command::new(clh_command("ch-remote")); 269 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 270 271 if let Some(arg) = arg { 272 cmd.arg(arg); 273 } 274 275 let output = cmd.output().expect("Failed to launch ch-remote"); 276 277 (output.status.success(), output.stdout) 278 } 279 280 fn resize_command( 281 api_socket: &str, 282 desired_vcpus: Option<u8>, 283 desired_ram: Option<usize>, 284 desired_balloon: Option<usize>, 285 event_file: Option<&str>, 286 ) -> bool { 287 let mut cmd = Command::new(clh_command("ch-remote")); 288 cmd.args(&[&format!("--api-socket={}", api_socket), "resize"]); 289 290 if let Some(desired_vcpus) = desired_vcpus { 291 cmd.arg(format!("--cpus={}", desired_vcpus)); 292 } 293 294 if let Some(desired_ram) = desired_ram { 295 cmd.arg(format!("--memory={}", desired_ram)); 296 } 297 298 if let Some(desired_balloon) = desired_balloon { 299 cmd.arg(format!("--balloon={}", desired_balloon)); 300 } 301 302 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 303 304 if let Some(event_path) = event_file { 305 let latest_events = [ 306 &MetaEvent { 307 event: "resizing".to_string(), 308 device_id: None, 309 }, 310 &MetaEvent { 311 event: "resized".to_string(), 312 device_id: None, 313 }, 314 ]; 315 assert!(check_latest_events_exact(&latest_events, event_path)); 316 } 317 318 ret 319 } 320 321 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 322 let mut cmd = Command::new(clh_command("ch-remote")); 323 cmd.args(&[ 324 &format!("--api-socket={}", api_socket), 325 "resize-zone", 326 &format!("--id={}", id), 327 &format!("--size={}", desired_size), 328 ]); 329 330 cmd.status().expect("Failed to launch ch-remote").success() 331 } 332 333 // setup OVS-DPDK bridge and ports 334 fn setup_ovs_dpdk() { 335 // setup OVS-DPDK 336 assert!(exec_host_command_status("service openvswitch-switch start").success()); 337 assert!(exec_host_command_status("ovs-vsctl init").success()); 338 assert!( 339 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 340 .success() 341 ); 342 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 343 344 // Create OVS-DPDK bridge and ports 345 assert!(exec_host_command_status( 346 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 347 ) 348 .success()); 349 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 350 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 351 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 352 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 353 } 354 fn cleanup_ovs_dpdk() { 355 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 356 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 357 } 358 // Setup two guests and ensure they are connected through ovs-dpdk 359 fn setup_ovs_dpdk_guests( 360 guest1: &Guest, 361 guest2: &Guest, 362 api_socket: &str, 363 release_binary: bool, 364 ) -> (Child, Child) { 365 setup_ovs_dpdk(); 366 367 let clh_path = if !release_binary { 368 clh_command("cloud-hypervisor") 369 } else { 370 cloud_hypervisor_release_path() 371 }; 372 373 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 374 .args(&["--cpus", "boot=2"]) 375 .args(&["--memory", "size=0,shared=on"]) 376 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 377 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 378 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 379 .default_disks() 380 .args(&["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 381 .capture_output() 382 .spawn() 383 .unwrap(); 384 385 #[cfg(target_arch = "x86_64")] 386 let guest_net_iface = "ens5"; 387 #[cfg(target_arch = "aarch64")] 388 let guest_net_iface = "enp0s5"; 389 390 let r = std::panic::catch_unwind(|| { 391 guest1.wait_vm_boot(None).unwrap(); 392 393 guest1 394 .ssh_command(&format!( 395 "sudo ip addr add 172.100.0.1/24 dev {}", 396 guest_net_iface 397 )) 398 .unwrap(); 399 guest1 400 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 401 .unwrap(); 402 403 let guest_ip = guest1.network.guest_ip.clone(); 404 thread::spawn(move || { 405 ssh_command_ip( 406 "nc -l 12345", 407 &guest_ip, 408 DEFAULT_SSH_RETRIES, 409 DEFAULT_SSH_TIMEOUT, 410 ) 411 .unwrap(); 412 }); 413 }); 414 if r.is_err() { 415 cleanup_ovs_dpdk(); 416 417 let _ = child1.kill(); 418 let output = child1.wait_with_output().unwrap(); 419 handle_child_output(r, &output); 420 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 421 } 422 423 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 424 .args(&["--api-socket", api_socket]) 425 .args(&["--cpus", "boot=2"]) 426 .args(&["--memory", "size=0,shared=on"]) 427 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 428 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 429 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 430 .default_disks() 431 .args(&["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 432 .capture_output() 433 .spawn() 434 .unwrap(); 435 436 let r = std::panic::catch_unwind(|| { 437 guest2.wait_vm_boot(None).unwrap(); 438 439 guest2 440 .ssh_command(&format!( 441 "sudo ip addr add 172.100.0.2/24 dev {}", 442 guest_net_iface 443 )) 444 .unwrap(); 445 guest2 446 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 447 .unwrap(); 448 449 // Check the connection works properly between the two VMs 450 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 451 }); 452 if r.is_err() { 453 cleanup_ovs_dpdk(); 454 455 let _ = child1.kill(); 456 let _ = child2.kill(); 457 let output = child2.wait_with_output().unwrap(); 458 handle_child_output(r, &output); 459 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 460 } 461 462 (child1, child2) 463 } 464 465 enum FwType { 466 Ovmf, 467 RustHypervisorFirmware, 468 } 469 470 fn fw_path(_fw_type: FwType) -> String { 471 let mut workload_path = dirs::home_dir().unwrap(); 472 workload_path.push("workloads"); 473 474 let mut fw_path = workload_path; 475 #[cfg(target_arch = "aarch64")] 476 fw_path.push("CLOUDHV_EFI.fd"); 477 #[cfg(target_arch = "x86_64")] 478 { 479 match _fw_type { 480 FwType::Ovmf => fw_path.push(OVMF_NAME), 481 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 482 } 483 } 484 485 fw_path.to_str().unwrap().to_string() 486 } 487 488 struct MetaEvent { 489 event: String, 490 device_id: Option<String>, 491 } 492 493 impl MetaEvent { 494 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 495 let mut matched = false; 496 if v["event"].as_str().unwrap() == self.event { 497 if let Some(device_id) = &self.device_id { 498 if v["properties"]["id"].as_str().unwrap() == device_id { 499 matched = true 500 } 501 } else { 502 matched = true; 503 } 504 } 505 matched 506 } 507 } 508 509 // Parse the event_monitor file based on the format that each event 510 // is followed by a double newline 511 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 512 let content = fs::read(event_file).unwrap(); 513 let mut ret = Vec::new(); 514 for entry in String::from_utf8_lossy(&content) 515 .trim() 516 .split("\n\n") 517 .collect::<Vec<&str>>() 518 { 519 ret.push(serde_json::from_str(entry).unwrap()); 520 } 521 522 ret 523 } 524 525 // Return true if all events from the input 'expected_events' are matched sequentially 526 // with events from the 'event_file' 527 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 528 let json_events = parse_event_file(event_file); 529 let len = expected_events.len(); 530 let mut idx = 0; 531 for e in &json_events { 532 if idx == len { 533 break; 534 } 535 if expected_events[idx].match_with_json_event(e) { 536 idx += 1; 537 } 538 } 539 540 idx == len 541 } 542 543 // Return true if all events from the input 'expected_events' are matched exactly 544 // with events from the 'event_file' 545 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 546 let json_events = parse_event_file(event_file); 547 assert!(expected_events.len() <= json_events.len()); 548 let json_events = &json_events[..expected_events.len()]; 549 550 for (idx, e) in json_events.iter().enumerate() { 551 if !expected_events[idx].match_with_json_event(e) { 552 return false; 553 } 554 } 555 556 true 557 } 558 559 // Return true if events from the input 'expected_events' are matched exactly 560 // with the most recent events from the 'event_file' 561 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 562 let json_events = parse_event_file(event_file); 563 assert!(latest_events.len() <= json_events.len()); 564 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 565 566 for (idx, e) in json_events.iter().enumerate() { 567 if !latest_events[idx].match_with_json_event(e) { 568 return false; 569 } 570 } 571 572 true 573 } 574 575 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 576 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 577 let guest = Guest::new(Box::new(focal)); 578 let total_vcpus = threads_per_core * cores_per_package * packages; 579 let direct_kernel_boot_path = direct_kernel_boot_path(); 580 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 581 let fw_path = fw_path(FwType::RustHypervisorFirmware); 582 if use_fw { 583 kernel_path = fw_path.as_str(); 584 } 585 586 let mut child = GuestCommand::new(&guest) 587 .args(&[ 588 "--cpus", 589 &format!( 590 "boot={},topology={}:{}:1:{}", 591 total_vcpus, threads_per_core, cores_per_package, packages 592 ), 593 ]) 594 .args(&["--memory", "size=512M"]) 595 .args(&["--kernel", kernel_path]) 596 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 597 .default_disks() 598 .default_net() 599 .capture_output() 600 .spawn() 601 .unwrap(); 602 603 let r = std::panic::catch_unwind(|| { 604 guest.wait_vm_boot(None).unwrap(); 605 assert_eq!( 606 guest.get_cpu_count().unwrap_or_default(), 607 u32::from(total_vcpus) 608 ); 609 assert_eq!( 610 guest 611 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 612 .unwrap() 613 .trim() 614 .parse::<u8>() 615 .unwrap_or(0), 616 threads_per_core 617 ); 618 619 assert_eq!( 620 guest 621 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 622 .unwrap() 623 .trim() 624 .parse::<u8>() 625 .unwrap_or(0), 626 cores_per_package 627 ); 628 629 assert_eq!( 630 guest 631 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 632 .unwrap() 633 .trim() 634 .parse::<u8>() 635 .unwrap_or(0), 636 packages 637 ); 638 }); 639 640 let _ = child.kill(); 641 let output = child.wait_with_output().unwrap(); 642 643 handle_child_output(r, &output); 644 } 645 646 #[allow(unused_variables)] 647 fn _test_guest_numa_nodes(acpi: bool) { 648 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 649 let guest = Guest::new(Box::new(focal)); 650 let api_socket = temp_api_path(&guest.tmp_dir); 651 #[cfg(target_arch = "x86_64")] 652 let kernel_path = direct_kernel_boot_path(); 653 #[cfg(target_arch = "aarch64")] 654 let kernel_path = if acpi { 655 edk2_path() 656 } else { 657 direct_kernel_boot_path() 658 }; 659 660 let mut child = GuestCommand::new(&guest) 661 .args(&["--cpus", "boot=6,max=12"]) 662 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 663 .args(&[ 664 "--memory-zone", 665 "id=mem0,size=1G,hotplug_size=3G", 666 "id=mem1,size=2G,hotplug_size=3G", 667 "id=mem2,size=3G,hotplug_size=3G", 668 ]) 669 .args(&[ 670 "--numa", 671 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 672 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 673 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 674 ]) 675 .args(&["--kernel", kernel_path.to_str().unwrap()]) 676 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 677 .args(&["--api-socket", &api_socket]) 678 .capture_output() 679 .default_disks() 680 .default_net() 681 .spawn() 682 .unwrap(); 683 684 let r = std::panic::catch_unwind(|| { 685 guest.wait_vm_boot(None).unwrap(); 686 687 guest.check_numa_common( 688 Some(&[960_000, 1_920_000, 2_880_000]), 689 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 690 Some(&["10 15 20", "20 10 25", "25 30 10"]), 691 ); 692 693 // AArch64 currently does not support hotplug, and therefore we only 694 // test hotplug-related function on x86_64 here. 695 #[cfg(target_arch = "x86_64")] 696 { 697 guest.enable_memory_hotplug(); 698 699 // Resize every memory zone and check each associated NUMA node 700 // has been assigned the right amount of memory. 701 resize_zone_command(&api_socket, "mem0", "4G"); 702 resize_zone_command(&api_socket, "mem1", "4G"); 703 resize_zone_command(&api_socket, "mem2", "4G"); 704 // Resize to the maximum amount of CPUs and check each NUMA 705 // node has been assigned the right CPUs set. 706 resize_command(&api_socket, Some(12), None, None, None); 707 thread::sleep(std::time::Duration::new(5, 0)); 708 709 guest.check_numa_common( 710 Some(&[3_840_000, 3_840_000, 3_840_000]), 711 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 712 None, 713 ); 714 } 715 }); 716 717 let _ = child.kill(); 718 let output = child.wait_with_output().unwrap(); 719 720 handle_child_output(r, &output); 721 } 722 723 #[allow(unused_variables)] 724 fn _test_power_button(acpi: bool) { 725 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 726 let guest = Guest::new(Box::new(focal)); 727 let mut cmd = GuestCommand::new(&guest); 728 let api_socket = temp_api_path(&guest.tmp_dir); 729 730 #[cfg(target_arch = "x86_64")] 731 let kernel_path = direct_kernel_boot_path(); 732 #[cfg(target_arch = "aarch64")] 733 let kernel_path = if acpi { 734 edk2_path() 735 } else { 736 direct_kernel_boot_path() 737 }; 738 739 cmd.args(&["--cpus", "boot=1"]) 740 .args(&["--memory", "size=512M"]) 741 .args(&["--kernel", kernel_path.to_str().unwrap()]) 742 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 743 .capture_output() 744 .default_disks() 745 .default_net() 746 .args(&["--api-socket", &api_socket]); 747 748 let child = cmd.spawn().unwrap(); 749 750 let r = std::panic::catch_unwind(|| { 751 guest.wait_vm_boot(None).unwrap(); 752 assert!(remote_command(&api_socket, "power-button", None)); 753 }); 754 755 let output = child.wait_with_output().unwrap(); 756 assert!(output.status.success()); 757 handle_child_output(r, &output); 758 } 759 760 type PrepareNetDaemon = 761 dyn Fn(&TempDir, &str, Option<&str>, usize, bool) -> (std::process::Command, String); 762 763 fn test_vhost_user_net( 764 tap: Option<&str>, 765 num_queues: usize, 766 prepare_daemon: &PrepareNetDaemon, 767 generate_host_mac: bool, 768 client_mode_daemon: bool, 769 ) { 770 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 771 let guest = Guest::new(Box::new(focal)); 772 let api_socket = temp_api_path(&guest.tmp_dir); 773 774 let kernel_path = direct_kernel_boot_path(); 775 776 let host_mac = if generate_host_mac { 777 Some(MacAddr::local_random()) 778 } else { 779 None 780 }; 781 782 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 783 &guest.tmp_dir, 784 &guest.network.host_ip, 785 tap, 786 num_queues, 787 client_mode_daemon, 788 ); 789 790 let net_params = format!( 791 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={}", 792 guest.network.guest_mac, 793 vunet_socket_path, 794 num_queues, 795 if let Some(host_mac) = host_mac { 796 format!(",host_mac={}", host_mac) 797 } else { 798 "".to_owned() 799 }, 800 if client_mode_daemon { 801 "server" 802 } else { 803 "client" 804 }, 805 ); 806 807 let mut ch_command = GuestCommand::new(&guest); 808 ch_command 809 .args(&["--cpus", format!("boot={}", num_queues / 2).as_str()]) 810 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 811 .args(&["--kernel", kernel_path.to_str().unwrap()]) 812 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 813 .default_disks() 814 .args(&["--net", net_params.as_str()]) 815 .args(&["--api-socket", &api_socket]) 816 .capture_output(); 817 818 let mut daemon_child: std::process::Child; 819 let mut child: std::process::Child; 820 821 if client_mode_daemon { 822 child = ch_command.spawn().unwrap(); 823 // Make sure the VMM is waiting for the backend to connect 824 thread::sleep(std::time::Duration::new(10, 0)); 825 daemon_child = daemon_command.spawn().unwrap(); 826 } else { 827 daemon_child = daemon_command.spawn().unwrap(); 828 // Make sure the backend is waiting for the VMM to connect 829 thread::sleep(std::time::Duration::new(10, 0)); 830 child = ch_command.spawn().unwrap(); 831 } 832 833 let r = std::panic::catch_unwind(|| { 834 guest.wait_vm_boot(None).unwrap(); 835 836 if let Some(tap_name) = tap { 837 let tap_count = exec_host_command_output(&format!("ip link | grep -c {}", tap_name)); 838 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 839 } 840 841 if let Some(host_mac) = tap { 842 let mac_count = exec_host_command_output(&format!("ip link | grep -c {}", host_mac)); 843 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 844 } 845 846 // 1 network interface + default localhost ==> 2 interfaces 847 // It's important to note that this test is fully exercising the 848 // vhost-user-net implementation and the associated backend since 849 // it does not define any --net network interface. That means all 850 // the ssh communication in that test happens through the network 851 // interface backed by vhost-user-net. 852 assert_eq!( 853 guest 854 .ssh_command("ip -o link | wc -l") 855 .unwrap() 856 .trim() 857 .parse::<u32>() 858 .unwrap_or_default(), 859 2 860 ); 861 862 // The following pci devices will appear on guest with PCI-MSI 863 // interrupt vectors assigned. 864 // 1 virtio-console with 3 vectors: config, Rx, Tx 865 // 1 virtio-blk with 2 vectors: config, Request 866 // 1 virtio-blk with 2 vectors: config, Request 867 // 1 virtio-rng with 2 vectors: config, Request 868 // Since virtio-net has 2 queue pairs, its vectors is as follows: 869 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 870 // Based on the above, the total vectors should 14. 871 #[cfg(target_arch = "x86_64")] 872 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 873 #[cfg(target_arch = "aarch64")] 874 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 875 assert_eq!( 876 guest 877 .ssh_command(grep_cmd) 878 .unwrap() 879 .trim() 880 .parse::<u32>() 881 .unwrap_or_default(), 882 10 + (num_queues as u32) 883 ); 884 885 // ACPI feature is needed. 886 #[cfg(target_arch = "x86_64")] 887 { 888 guest.enable_memory_hotplug(); 889 890 // Add RAM to the VM 891 let desired_ram = 1024 << 20; 892 resize_command(&api_socket, None, Some(desired_ram), None, None); 893 894 thread::sleep(std::time::Duration::new(10, 0)); 895 896 // Here by simply checking the size (through ssh), we validate 897 // the connection is still working, which means vhost-user-net 898 // keeps working after the resize. 899 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 900 } 901 }); 902 903 let _ = child.kill(); 904 let output = child.wait_with_output().unwrap(); 905 906 thread::sleep(std::time::Duration::new(5, 0)); 907 let _ = daemon_child.kill(); 908 let _ = daemon_child.wait(); 909 910 handle_child_output(r, &output); 911 } 912 913 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 914 915 fn test_vhost_user_blk( 916 num_queues: usize, 917 readonly: bool, 918 direct: bool, 919 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 920 ) { 921 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 922 let guest = Guest::new(Box::new(focal)); 923 let api_socket = temp_api_path(&guest.tmp_dir); 924 925 let kernel_path = direct_kernel_boot_path(); 926 927 let (blk_params, daemon_child) = { 928 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 929 // Start the daemon 930 let (daemon_child, vubd_socket_path) = 931 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 932 933 ( 934 format!( 935 "vhost_user=true,socket={},num_queues={},queue_size=128", 936 vubd_socket_path, num_queues, 937 ), 938 Some(daemon_child), 939 ) 940 }; 941 942 let mut child = GuestCommand::new(&guest) 943 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 944 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 945 .args(&["--kernel", kernel_path.to_str().unwrap()]) 946 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 947 .args(&[ 948 "--disk", 949 format!( 950 "path={}", 951 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 952 ) 953 .as_str(), 954 format!( 955 "path={}", 956 guest.disk_config.disk(DiskType::CloudInit).unwrap() 957 ) 958 .as_str(), 959 blk_params.as_str(), 960 ]) 961 .default_net() 962 .args(&["--api-socket", &api_socket]) 963 .capture_output() 964 .spawn() 965 .unwrap(); 966 967 let r = std::panic::catch_unwind(|| { 968 guest.wait_vm_boot(None).unwrap(); 969 970 // Check both if /dev/vdc exists and if the block size is 16M. 971 assert_eq!( 972 guest 973 .ssh_command("lsblk | grep vdc | grep -c 16M") 974 .unwrap() 975 .trim() 976 .parse::<u32>() 977 .unwrap_or_default(), 978 1 979 ); 980 981 // Check if this block is RO or RW. 982 assert_eq!( 983 guest 984 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 985 .unwrap() 986 .trim() 987 .parse::<u32>() 988 .unwrap_or_default(), 989 readonly as u32 990 ); 991 992 // Check if the number of queues in /sys/block/vdc/mq matches the 993 // expected num_queues. 994 assert_eq!( 995 guest 996 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 997 .unwrap() 998 .trim() 999 .parse::<u32>() 1000 .unwrap_or_default(), 1001 num_queues as u32 1002 ); 1003 1004 // Mount the device 1005 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1006 guest.ssh_command("mkdir mount_image").unwrap(); 1007 guest 1008 .ssh_command( 1009 format!( 1010 "sudo mount -o {} -t ext4 /dev/vdc mount_image/", 1011 mount_ro_rw_flag 1012 ) 1013 .as_str(), 1014 ) 1015 .unwrap(); 1016 1017 // Check the content of the block device. The file "foo" should 1018 // contain "bar". 1019 assert_eq!( 1020 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1021 "bar" 1022 ); 1023 1024 // ACPI feature is needed. 1025 #[cfg(target_arch = "x86_64")] 1026 { 1027 guest.enable_memory_hotplug(); 1028 1029 // Add RAM to the VM 1030 let desired_ram = 1024 << 20; 1031 resize_command(&api_socket, None, Some(desired_ram), None, None); 1032 1033 thread::sleep(std::time::Duration::new(10, 0)); 1034 1035 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1036 1037 // Check again the content of the block device after the resize 1038 // has been performed. 1039 assert_eq!( 1040 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1041 "bar" 1042 ); 1043 } 1044 1045 // Unmount the device 1046 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1047 guest.ssh_command("rm -r mount_image").unwrap(); 1048 }); 1049 1050 let _ = child.kill(); 1051 let output = child.wait_with_output().unwrap(); 1052 1053 if let Some(mut daemon_child) = daemon_child { 1054 thread::sleep(std::time::Duration::new(5, 0)); 1055 let _ = daemon_child.kill(); 1056 let _ = daemon_child.wait(); 1057 } 1058 1059 handle_child_output(r, &output); 1060 } 1061 1062 fn test_boot_from_vhost_user_blk( 1063 num_queues: usize, 1064 readonly: bool, 1065 direct: bool, 1066 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1067 ) { 1068 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1069 let guest = Guest::new(Box::new(focal)); 1070 1071 let kernel_path = direct_kernel_boot_path(); 1072 1073 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1074 1075 let (blk_boot_params, daemon_child) = { 1076 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1077 // Start the daemon 1078 let (daemon_child, vubd_socket_path) = prepare_daemon( 1079 &guest.tmp_dir, 1080 disk_path.as_str(), 1081 num_queues, 1082 readonly, 1083 direct, 1084 ); 1085 1086 ( 1087 format!( 1088 "vhost_user=true,socket={},num_queues={},queue_size=128", 1089 vubd_socket_path, num_queues, 1090 ), 1091 Some(daemon_child), 1092 ) 1093 }; 1094 1095 let mut child = GuestCommand::new(&guest) 1096 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 1097 .args(&["--memory", "size=512M,shared=on"]) 1098 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1099 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1100 .args(&[ 1101 "--disk", 1102 blk_boot_params.as_str(), 1103 format!( 1104 "path={}", 1105 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1106 ) 1107 .as_str(), 1108 ]) 1109 .default_net() 1110 .capture_output() 1111 .spawn() 1112 .unwrap(); 1113 1114 let r = std::panic::catch_unwind(|| { 1115 guest.wait_vm_boot(None).unwrap(); 1116 1117 // Just check the VM booted correctly. 1118 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1119 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1120 }); 1121 let _ = child.kill(); 1122 let output = child.wait_with_output().unwrap(); 1123 1124 if let Some(mut daemon_child) = daemon_child { 1125 thread::sleep(std::time::Duration::new(5, 0)); 1126 let _ = daemon_child.kill(); 1127 let _ = daemon_child.wait(); 1128 } 1129 1130 handle_child_output(r, &output); 1131 } 1132 1133 fn _test_virtio_fs( 1134 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1135 hotplug: bool, 1136 pci_segment: Option<u16>, 1137 ) { 1138 #[cfg(target_arch = "aarch64")] 1139 let focal_image = if hotplug { 1140 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1141 } else { 1142 FOCAL_IMAGE_NAME.to_string() 1143 }; 1144 #[cfg(target_arch = "x86_64")] 1145 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1146 let focal = UbuntuDiskConfig::new(focal_image); 1147 let guest = Guest::new(Box::new(focal)); 1148 let api_socket = temp_api_path(&guest.tmp_dir); 1149 1150 let mut workload_path = dirs::home_dir().unwrap(); 1151 workload_path.push("workloads"); 1152 1153 let mut shared_dir = workload_path; 1154 shared_dir.push("shared_dir"); 1155 1156 #[cfg(target_arch = "x86_64")] 1157 let kernel_path = direct_kernel_boot_path(); 1158 #[cfg(target_arch = "aarch64")] 1159 let kernel_path = if hotplug { 1160 edk2_path() 1161 } else { 1162 direct_kernel_boot_path() 1163 }; 1164 1165 let (mut daemon_child, virtiofsd_socket_path) = 1166 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1167 1168 let mut guest_command = GuestCommand::new(&guest); 1169 guest_command 1170 .args(&["--cpus", "boot=1"]) 1171 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1172 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1173 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1174 .default_disks() 1175 .default_net() 1176 .args(&["--api-socket", &api_socket]); 1177 if pci_segment.is_some() { 1178 guest_command.args(&["--platform", "num_pci_segments=16"]); 1179 } 1180 1181 let fs_params = format!( 1182 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1183 virtiofsd_socket_path, 1184 if let Some(pci_segment) = pci_segment { 1185 format!(",pci_segment={}", pci_segment) 1186 } else { 1187 "".to_owned() 1188 } 1189 ); 1190 1191 if !hotplug { 1192 guest_command.args(&["--fs", fs_params.as_str()]); 1193 } 1194 1195 let mut child = guest_command.capture_output().spawn().unwrap(); 1196 1197 let r = std::panic::catch_unwind(|| { 1198 guest.wait_vm_boot(None).unwrap(); 1199 1200 if hotplug { 1201 // Add fs to the VM 1202 let (cmd_success, cmd_output) = 1203 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1204 assert!(cmd_success); 1205 1206 if let Some(pci_segment) = pci_segment { 1207 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1208 "{{\"id\":\"myfs0\",\"bdf\":\"{:04x}:00:01.0\"}}", 1209 pci_segment 1210 ))); 1211 } else { 1212 assert!(String::from_utf8_lossy(&cmd_output) 1213 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1214 } 1215 1216 thread::sleep(std::time::Duration::new(10, 0)); 1217 } 1218 1219 // Mount shared directory through virtio_fs filesystem 1220 guest 1221 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1222 .unwrap(); 1223 1224 // Check file1 exists and its content is "foo" 1225 assert_eq!( 1226 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1227 "foo" 1228 ); 1229 // Check file2 does not exist 1230 guest 1231 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1232 .unwrap(); 1233 1234 // Check file3 exists and its content is "bar" 1235 assert_eq!( 1236 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1237 "bar" 1238 ); 1239 1240 // ACPI feature is needed. 1241 #[cfg(target_arch = "x86_64")] 1242 { 1243 guest.enable_memory_hotplug(); 1244 1245 // Add RAM to the VM 1246 let desired_ram = 1024 << 20; 1247 resize_command(&api_socket, None, Some(desired_ram), None, None); 1248 1249 thread::sleep(std::time::Duration::new(30, 0)); 1250 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1251 1252 // After the resize, check again that file1 exists and its 1253 // content is "foo". 1254 assert_eq!( 1255 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1256 "foo" 1257 ); 1258 } 1259 1260 if hotplug { 1261 // Remove from VM 1262 guest.ssh_command("sudo umount mount_dir").unwrap(); 1263 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1264 } 1265 }); 1266 1267 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1268 thread::sleep(std::time::Duration::new(10, 0)); 1269 let (daemon_child, virtiofsd_socket_path) = 1270 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1271 1272 let r = std::panic::catch_unwind(|| { 1273 thread::sleep(std::time::Duration::new(10, 0)); 1274 let fs_params = format!( 1275 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1276 virtiofsd_socket_path, 1277 if let Some(pci_segment) = pci_segment { 1278 format!(",pci_segment={}", pci_segment) 1279 } else { 1280 "".to_owned() 1281 } 1282 ); 1283 1284 // Add back and check it works 1285 let (cmd_success, cmd_output) = 1286 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1287 assert!(cmd_success); 1288 if let Some(pci_segment) = pci_segment { 1289 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1290 "{{\"id\":\"myfs0\",\"bdf\":\"{:04x}:00:01.0\"}}", 1291 pci_segment 1292 ))); 1293 } else { 1294 assert!(String::from_utf8_lossy(&cmd_output) 1295 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1296 } 1297 1298 thread::sleep(std::time::Duration::new(10, 0)); 1299 // Mount shared directory through virtio_fs filesystem 1300 guest 1301 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1302 .unwrap(); 1303 1304 // Check file1 exists and its content is "foo" 1305 assert_eq!( 1306 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1307 "foo" 1308 ); 1309 }); 1310 1311 (r, Some(daemon_child)) 1312 } else { 1313 (r, None) 1314 }; 1315 1316 let _ = child.kill(); 1317 let output = child.wait_with_output().unwrap(); 1318 1319 let _ = daemon_child.kill(); 1320 let _ = daemon_child.wait(); 1321 1322 if let Some(mut daemon_child) = hotplug_daemon_child { 1323 let _ = daemon_child.kill(); 1324 let _ = daemon_child.wait(); 1325 } 1326 1327 handle_child_output(r, &output); 1328 } 1329 1330 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1331 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1332 let guest = Guest::new(Box::new(focal)); 1333 1334 let kernel_path = direct_kernel_boot_path(); 1335 1336 let pmem_temp_file = TempFile::new().unwrap(); 1337 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1338 1339 std::process::Command::new("mkfs.ext4") 1340 .arg(pmem_temp_file.as_path()) 1341 .output() 1342 .expect("Expect creating disk image to succeed"); 1343 1344 let mut child = GuestCommand::new(&guest) 1345 .args(&["--cpus", "boot=1"]) 1346 .args(&["--memory", "size=512M"]) 1347 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1348 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1349 .default_disks() 1350 .default_net() 1351 .args(&[ 1352 "--pmem", 1353 format!( 1354 "file={}{}{}", 1355 pmem_temp_file.as_path().to_str().unwrap(), 1356 if specify_size { ",size=128M" } else { "" }, 1357 if discard_writes { 1358 ",discard_writes=on" 1359 } else { 1360 "" 1361 } 1362 ) 1363 .as_str(), 1364 ]) 1365 .capture_output() 1366 .spawn() 1367 .unwrap(); 1368 1369 let r = std::panic::catch_unwind(|| { 1370 guest.wait_vm_boot(None).unwrap(); 1371 1372 // Check for the presence of /dev/pmem0 1373 assert_eq!( 1374 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1375 "/dev/pmem0" 1376 ); 1377 1378 // Check changes persist after reboot 1379 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1380 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1381 guest 1382 .ssh_command("echo test123 | sudo tee /mnt/test") 1383 .unwrap(); 1384 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1385 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1386 1387 guest.reboot_linux(0, None); 1388 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1389 assert_eq!( 1390 guest 1391 .ssh_command("sudo cat /mnt/test || true") 1392 .unwrap() 1393 .trim(), 1394 if discard_writes { "" } else { "test123" } 1395 ); 1396 }); 1397 1398 let _ = child.kill(); 1399 let output = child.wait_with_output().unwrap(); 1400 1401 handle_child_output(r, &output); 1402 } 1403 1404 fn get_fd_count(pid: u32) -> usize { 1405 fs::read_dir(format!("/proc/{}/fd", pid)).unwrap().count() 1406 } 1407 1408 fn _test_virtio_vsock(hotplug: bool) { 1409 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1410 let guest = Guest::new(Box::new(focal)); 1411 1412 #[cfg(target_arch = "x86_64")] 1413 let kernel_path = direct_kernel_boot_path(); 1414 #[cfg(target_arch = "aarch64")] 1415 let kernel_path = if hotplug { 1416 edk2_path() 1417 } else { 1418 direct_kernel_boot_path() 1419 }; 1420 1421 let socket = temp_vsock_path(&guest.tmp_dir); 1422 let api_socket = temp_api_path(&guest.tmp_dir); 1423 1424 let mut cmd = GuestCommand::new(&guest); 1425 cmd.args(&["--api-socket", &api_socket]); 1426 cmd.args(&["--cpus", "boot=1"]); 1427 cmd.args(&["--memory", "size=512M"]); 1428 cmd.args(&["--kernel", kernel_path.to_str().unwrap()]); 1429 cmd.args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1430 cmd.default_disks(); 1431 cmd.default_net(); 1432 1433 if !hotplug { 1434 cmd.args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]); 1435 } 1436 1437 let mut child = cmd.capture_output().spawn().unwrap(); 1438 1439 let r = std::panic::catch_unwind(|| { 1440 guest.wait_vm_boot(None).unwrap(); 1441 1442 if hotplug { 1443 let (cmd_success, cmd_output) = remote_command_w_output( 1444 &api_socket, 1445 "add-vsock", 1446 Some(format!("cid=3,socket={},id=test0", socket).as_str()), 1447 ); 1448 assert!(cmd_success); 1449 assert!(String::from_utf8_lossy(&cmd_output) 1450 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1451 thread::sleep(std::time::Duration::new(10, 0)); 1452 // Check adding a second one fails 1453 assert!(!remote_command( 1454 &api_socket, 1455 "add-vsock", 1456 Some("cid=1234,socket=/tmp/fail") 1457 )); 1458 } 1459 1460 // Validate vsock works as expected. 1461 guest.check_vsock(socket.as_str()); 1462 guest.reboot_linux(0, None); 1463 // Validate vsock still works after a reboot. 1464 guest.check_vsock(socket.as_str()); 1465 1466 if hotplug { 1467 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1468 } 1469 }); 1470 1471 let _ = child.kill(); 1472 let output = child.wait_with_output().unwrap(); 1473 1474 handle_child_output(r, &output); 1475 } 1476 1477 fn get_ksm_pages_shared() -> u32 { 1478 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1479 .unwrap() 1480 .trim() 1481 .parse::<u32>() 1482 .unwrap() 1483 } 1484 1485 fn test_memory_mergeable(mergeable: bool) { 1486 let memory_param = if mergeable { 1487 "mergeable=on" 1488 } else { 1489 "mergeable=off" 1490 }; 1491 1492 // We are assuming the rest of the system in our CI is not using mergeable memeory 1493 let ksm_ps_init = get_ksm_pages_shared(); 1494 assert!(ksm_ps_init == 0); 1495 1496 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1497 let guest1 = Guest::new(Box::new(focal1)); 1498 let mut child1 = GuestCommand::new(&guest1) 1499 .args(&["--cpus", "boot=1"]) 1500 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1501 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1502 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1503 .default_disks() 1504 .args(&["--net", guest1.default_net_string().as_str()]) 1505 .args(&["--serial", "tty", "--console", "off"]) 1506 .capture_output() 1507 .spawn() 1508 .unwrap(); 1509 1510 let r = std::panic::catch_unwind(|| { 1511 guest1.wait_vm_boot(None).unwrap(); 1512 }); 1513 if r.is_err() { 1514 let _ = child1.kill(); 1515 let output = child1.wait_with_output().unwrap(); 1516 handle_child_output(r, &output); 1517 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1518 } 1519 1520 let ksm_ps_guest1 = get_ksm_pages_shared(); 1521 1522 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1523 let guest2 = Guest::new(Box::new(focal2)); 1524 let mut child2 = GuestCommand::new(&guest2) 1525 .args(&["--cpus", "boot=1"]) 1526 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1527 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1528 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1529 .default_disks() 1530 .args(&["--net", guest2.default_net_string().as_str()]) 1531 .args(&["--serial", "tty", "--console", "off"]) 1532 .capture_output() 1533 .spawn() 1534 .unwrap(); 1535 1536 let r = std::panic::catch_unwind(|| { 1537 guest2.wait_vm_boot(None).unwrap(); 1538 let ksm_ps_guest2 = get_ksm_pages_shared(); 1539 1540 if mergeable { 1541 println!( 1542 "ksm pages_shared after vm1 booted '{}', ksm pages_shared after vm2 booted '{}'", 1543 ksm_ps_guest1, ksm_ps_guest2 1544 ); 1545 // We are expecting the number of shared pages to increase as the number of VM increases 1546 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1547 } else { 1548 assert!(ksm_ps_guest1 == 0); 1549 assert!(ksm_ps_guest2 == 0); 1550 } 1551 }); 1552 1553 let _ = child1.kill(); 1554 let _ = child2.kill(); 1555 1556 let output = child1.wait_with_output().unwrap(); 1557 child2.wait().unwrap(); 1558 1559 handle_child_output(r, &output); 1560 } 1561 1562 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 1563 let smaps = fs::File::open(format!("/proc/{}/smaps", pid)).unwrap(); 1564 let reader = io::BufReader::new(smaps); 1565 1566 let mut skip_map: bool = false; 1567 let mut region_name: String = "".to_string(); 1568 let mut region_maps = HashMap::new(); 1569 for line in reader.lines() { 1570 let l = line.unwrap(); 1571 1572 if l.contains('-') { 1573 let values: Vec<&str> = l.split_whitespace().collect(); 1574 region_name = values.last().unwrap().trim().to_string(); 1575 if region_name == "0" { 1576 region_name = "anonymous".to_string() 1577 } 1578 } 1579 1580 // Each section begins with something that looks like: 1581 // Size: 2184 kB 1582 if l.starts_with("Size:") { 1583 let values: Vec<&str> = l.split_whitespace().collect(); 1584 let map_size = values[1].parse::<u32>().unwrap(); 1585 // We skip the assigned guest RAM map, its RSS is only 1586 // dependent on the guest actual memory usage. 1587 // Everything else can be added to the VMM overhead. 1588 skip_map = map_size >= guest_memory_size; 1589 continue; 1590 } 1591 1592 // If this is a map we're taking into account, then we only 1593 // count the RSS. The sum of all counted RSS is the VMM overhead. 1594 if !skip_map && l.starts_with("Rss:") { 1595 let values: Vec<&str> = l.split_whitespace().collect(); 1596 let value = values[1].trim().parse::<u32>().unwrap(); 1597 *region_maps.entry(region_name.clone()).or_insert(0) += value; 1598 } 1599 } 1600 1601 region_maps 1602 } 1603 1604 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 1605 let mut total = 0; 1606 1607 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 1608 eprintln!("{}: {}", region_name, value); 1609 total += value; 1610 } 1611 1612 total 1613 } 1614 1615 fn process_rss_kib(pid: u32) -> usize { 1616 let command = format!("ps -q {} -o rss=", pid); 1617 let rss = exec_host_command_output(&command); 1618 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 1619 } 1620 1621 // 10MB is our maximum accepted overhead. 1622 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 1623 1624 #[derive(PartialEq, Eq, PartialOrd)] 1625 struct Counters { 1626 rx_bytes: u64, 1627 rx_frames: u64, 1628 tx_bytes: u64, 1629 tx_frames: u64, 1630 read_bytes: u64, 1631 write_bytes: u64, 1632 read_ops: u64, 1633 write_ops: u64, 1634 } 1635 1636 fn get_counters(api_socket: &str) -> Counters { 1637 // Get counters 1638 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 1639 assert!(cmd_success); 1640 1641 let counters: HashMap<&str, HashMap<&str, u64>> = 1642 serde_json::from_slice(&cmd_output).unwrap_or_default(); 1643 1644 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 1645 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 1646 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 1647 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 1648 1649 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 1650 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 1651 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 1652 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 1653 1654 Counters { 1655 rx_bytes, 1656 rx_frames, 1657 tx_bytes, 1658 tx_frames, 1659 read_bytes, 1660 write_bytes, 1661 read_ops, 1662 write_ops, 1663 } 1664 } 1665 1666 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 1667 let (tx, rx) = mpsc::channel::<String>(); 1668 thread::spawn(move || loop { 1669 thread::sleep(std::time::Duration::new(1, 0)); 1670 let mut buf = [0; 512]; 1671 match pty.read(&mut buf) { 1672 Ok(_) => { 1673 let output = std::str::from_utf8(&buf).unwrap().to_string(); 1674 match tx.send(output) { 1675 Ok(_) => (), 1676 Err(_) => break, 1677 } 1678 } 1679 Err(_) => break, 1680 } 1681 }); 1682 rx 1683 } 1684 1685 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 1686 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1687 assert!(cmd_success); 1688 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1689 assert_eq!("Pty", info["config"][pty_type]["mode"]); 1690 PathBuf::from( 1691 info["config"][pty_type]["file"] 1692 .as_str() 1693 .expect("Missing pty path"), 1694 ) 1695 } 1696 1697 // VFIO test network setup. 1698 // We reserve a different IP class for it: 172.18.0.0/24. 1699 #[cfg(target_arch = "x86_64")] 1700 fn setup_vfio_network_interfaces() { 1701 // 'vfio-br0' 1702 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 1703 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 1704 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 1705 // 'vfio-tap0' 1706 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 1707 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 1708 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 1709 // 'vfio-tap1' 1710 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 1711 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 1712 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 1713 // 'vfio-tap2' 1714 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 1715 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 1716 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 1717 // 'vfio-tap3' 1718 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 1719 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 1720 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 1721 } 1722 1723 // Tear VFIO test network down 1724 #[cfg(target_arch = "x86_64")] 1725 fn cleanup_vfio_network_interfaces() { 1726 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 1727 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 1728 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 1729 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 1730 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 1731 } 1732 1733 fn balloon_size(api_socket: &str) -> u64 { 1734 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1735 assert!(cmd_success); 1736 1737 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1738 let total_mem = &info["config"]["memory"]["size"] 1739 .to_string() 1740 .parse::<u64>() 1741 .unwrap(); 1742 let actual_mem = &info["memory_actual_size"] 1743 .to_string() 1744 .parse::<u64>() 1745 .unwrap(); 1746 total_mem - actual_mem 1747 } 1748 1749 // This test validates that it can find the virtio-iommu device at first. 1750 // It also verifies that both disks and the network card are attached to 1751 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 1752 // The last interesting part of this test is that it exercises the network 1753 // interface attached to the virtual IOMMU since this is the one used to 1754 // send all commands through SSH. 1755 fn _test_virtio_iommu(acpi: bool) { 1756 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 1757 // Focal image is still old. 1758 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 1759 // the kernel binary has been updated. 1760 #[cfg(target_arch = "aarch64")] 1761 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 1762 #[cfg(target_arch = "x86_64")] 1763 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1764 let focal = UbuntuDiskConfig::new(focal_image); 1765 let guest = Guest::new(Box::new(focal)); 1766 1767 #[cfg(target_arch = "x86_64")] 1768 let kernel_path = direct_kernel_boot_path(); 1769 #[cfg(target_arch = "aarch64")] 1770 let kernel_path = if acpi { 1771 edk2_path() 1772 } else { 1773 direct_kernel_boot_path() 1774 }; 1775 1776 let mut child = GuestCommand::new(&guest) 1777 .args(&["--cpus", "boot=1"]) 1778 .args(&["--memory", "size=512M"]) 1779 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1780 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1781 .args(&[ 1782 "--disk", 1783 format!( 1784 "path={},iommu=on", 1785 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1786 ) 1787 .as_str(), 1788 format!( 1789 "path={},iommu=on", 1790 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1791 ) 1792 .as_str(), 1793 ]) 1794 .args(&["--net", guest.default_net_string_w_iommu().as_str()]) 1795 .capture_output() 1796 .spawn() 1797 .unwrap(); 1798 1799 let r = std::panic::catch_unwind(|| { 1800 guest.wait_vm_boot(None).unwrap(); 1801 1802 // Verify the virtio-iommu device is present. 1803 assert!(guest 1804 .does_device_vendor_pair_match("0x1057", "0x1af4") 1805 .unwrap_or_default()); 1806 1807 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 1808 // different with ACPI. 1809 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 1810 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 1811 // and appear under folder '/sys/kernel/iommu_groups/'. 1812 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 1813 // which is the console. The first disk "0000:00:02.0" is in group '1'. 1814 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 1815 // contains "0000:00:02.0" which is the first disk. 1816 // 1817 // Verify the iommu group of the first disk. 1818 let iommu_group = if acpi { 0 } else { 1 }; 1819 assert_eq!( 1820 guest 1821 .ssh_command( 1822 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1823 ) 1824 .unwrap() 1825 .trim(), 1826 "0000:00:02.0" 1827 ); 1828 1829 // Verify the iommu group of the second disk. 1830 let iommu_group = if acpi { 1 } else { 2 }; 1831 assert_eq!( 1832 guest 1833 .ssh_command( 1834 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1835 ) 1836 .unwrap() 1837 .trim(), 1838 "0000:00:03.0" 1839 ); 1840 1841 // Verify the iommu group of the network card. 1842 let iommu_group = if acpi { 2 } else { 3 }; 1843 assert_eq!( 1844 guest 1845 .ssh_command( 1846 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1847 ) 1848 .unwrap() 1849 .trim(), 1850 "0000:00:04.0" 1851 ); 1852 }); 1853 1854 let _ = child.kill(); 1855 let output = child.wait_with_output().unwrap(); 1856 1857 handle_child_output(r, &output); 1858 } 1859 1860 fn enable_guest_watchdog(guest: &Guest, current_reboot_count: &mut u32) { 1861 // Check for PCI device 1862 assert!(guest 1863 .does_device_vendor_pair_match("0x1063", "0x1af4") 1864 .unwrap_or_default()); 1865 1866 // Enable systemd watchdog 1867 guest 1868 .ssh_command("echo RuntimeWatchdogSec=15s | sudo tee -a /etc/systemd/system.conf") 1869 .unwrap(); 1870 1871 guest.ssh_command("sudo reboot").unwrap(); 1872 1873 guest.wait_vm_boot(None).unwrap(); 1874 1875 // Check that systemd has activated the watchdog 1876 assert_eq!( 1877 guest 1878 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 1879 .unwrap() 1880 .trim() 1881 .parse::<u32>() 1882 .unwrap_or_default(), 1883 2 1884 ); 1885 1886 // Ensure that the current boot journal is written so reboot counts are valid 1887 guest.ssh_command("sudo journalctl --sync").unwrap(); 1888 1889 *current_reboot_count += 1; 1890 1891 let boot_count = guest 1892 .ssh_command("sudo journalctl --list-boots | wc -l") 1893 .unwrap() 1894 .trim() 1895 .parse::<u32>() 1896 .unwrap_or_default(); 1897 assert_eq!(boot_count, *current_reboot_count); 1898 } 1899 1900 fn check_guest_watchdog_no_reboot(guest: &Guest, current_reboot_count: &u32) { 1901 // Allow some normal time to elapse to check we don't get spurious reboots 1902 thread::sleep(std::time::Duration::new(40, 0)); 1903 1904 // Check no reboot 1905 let boot_count = guest 1906 .ssh_command("sudo journalctl --list-boots | wc -l") 1907 .unwrap() 1908 .trim() 1909 .parse::<u32>() 1910 .unwrap_or_default(); 1911 assert_eq!(boot_count, *current_reboot_count); 1912 } 1913 1914 fn check_guest_watchdog_one_reboot( 1915 guest: &Guest, 1916 _api_socket: &str, 1917 current_reboot_count: &mut u32, 1918 ) { 1919 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 1920 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 1921 1922 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 1923 guest.wait_vm_boot(Some(50)).unwrap(); 1924 1925 // Check that watchdog triggered reboot 1926 *current_reboot_count += 1; 1927 let boot_count = guest 1928 .ssh_command("sudo journalctl --list-boots | wc -l") 1929 .unwrap() 1930 .trim() 1931 .parse::<u32>() 1932 .unwrap_or_default(); 1933 assert_eq!(boot_count, *current_reboot_count); 1934 1935 #[cfg(target_arch = "x86_64")] 1936 { 1937 // Now pause the VM and remain offline for 30s 1938 assert!(remote_command(_api_socket, "pause", None)); 1939 thread::sleep(std::time::Duration::new(30, 0)); 1940 assert!(remote_command(_api_socket, "resume", None)); 1941 1942 // Check no reboot 1943 let boot_count = guest 1944 .ssh_command("sudo journalctl --list-boots | wc -l") 1945 .unwrap() 1946 .trim() 1947 .parse::<u32>() 1948 .unwrap_or_default(); 1949 assert_eq!(boot_count, *current_reboot_count); 1950 } 1951 } 1952 1953 mod parallel { 1954 use std::io::SeekFrom; 1955 1956 use crate::*; 1957 1958 #[test] 1959 #[cfg(target_arch = "x86_64")] 1960 fn test_bionic_hypervisor_fw() { 1961 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), BIONIC_IMAGE_NAME) 1962 } 1963 1964 #[test] 1965 #[cfg(target_arch = "x86_64")] 1966 fn test_focal_hypervisor_fw() { 1967 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 1968 } 1969 1970 #[test] 1971 #[cfg(target_arch = "x86_64")] 1972 fn test_bionic_ovmf() { 1973 test_simple_launch(fw_path(FwType::Ovmf), BIONIC_IMAGE_NAME) 1974 } 1975 1976 #[test] 1977 #[cfg(target_arch = "x86_64")] 1978 fn test_focal_ovmf() { 1979 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 1980 } 1981 1982 #[cfg(target_arch = "x86_64")] 1983 fn test_simple_launch(fw_path: String, disk_path: &str) { 1984 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 1985 let guest = Guest::new(disk_config); 1986 let event_path = temp_event_monitor_path(&guest.tmp_dir); 1987 1988 let mut child = GuestCommand::new(&guest) 1989 .args(&["--cpus", "boot=1"]) 1990 .args(&["--memory", "size=512M"]) 1991 .args(&["--kernel", fw_path.as_str()]) 1992 .default_disks() 1993 .default_net() 1994 .args(&["--serial", "tty", "--console", "off"]) 1995 .args(&["--event-monitor", format!("path={}", event_path).as_str()]) 1996 .capture_output() 1997 .spawn() 1998 .unwrap(); 1999 2000 let r = std::panic::catch_unwind(|| { 2001 guest.wait_vm_boot(Some(120)).unwrap(); 2002 2003 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2004 assert_eq!(guest.get_initial_apicid().unwrap_or(1), 0); 2005 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2006 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2007 2008 let expected_sequential_events = [ 2009 &MetaEvent { 2010 event: "starting".to_string(), 2011 device_id: None, 2012 }, 2013 &MetaEvent { 2014 event: "booting".to_string(), 2015 device_id: None, 2016 }, 2017 &MetaEvent { 2018 event: "booted".to_string(), 2019 device_id: None, 2020 }, 2021 &MetaEvent { 2022 event: "activated".to_string(), 2023 device_id: Some("_disk0".to_string()), 2024 }, 2025 &MetaEvent { 2026 event: "reset".to_string(), 2027 device_id: Some("_disk0".to_string()), 2028 }, 2029 ]; 2030 assert!(check_sequential_events( 2031 &expected_sequential_events, 2032 &event_path 2033 )); 2034 2035 guest.ssh_command("sudo poweroff").unwrap(); 2036 thread::sleep(std::time::Duration::new(5, 0)); 2037 let latest_events = [ 2038 &MetaEvent { 2039 event: "shutdown".to_string(), 2040 device_id: None, 2041 }, 2042 &MetaEvent { 2043 event: "deleted".to_string(), 2044 device_id: None, 2045 }, 2046 &MetaEvent { 2047 event: "shutdown".to_string(), 2048 device_id: None, 2049 }, 2050 ]; 2051 assert!(check_latest_events_exact(&latest_events, &event_path)); 2052 }); 2053 2054 let _ = child.kill(); 2055 let output = child.wait_with_output().unwrap(); 2056 2057 handle_child_output(r, &output); 2058 } 2059 2060 #[test] 2061 fn test_multi_cpu() { 2062 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 2063 let guest = Guest::new(Box::new(bionic)); 2064 let mut cmd = GuestCommand::new(&guest); 2065 cmd.args(&["--cpus", "boot=2,max=4"]) 2066 .args(&["--memory", "size=512M"]) 2067 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2068 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2069 .capture_output() 2070 .default_disks() 2071 .default_net(); 2072 2073 let mut child = cmd.spawn().unwrap(); 2074 2075 let r = std::panic::catch_unwind(|| { 2076 guest.wait_vm_boot(Some(120)).unwrap(); 2077 2078 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2079 2080 #[cfg(target_arch = "x86_64")] 2081 assert_eq!( 2082 guest 2083 .ssh_command(r#"dmesg | grep "smpboot: Allowing" | sed "s/\[\ *[0-9.]*\] //""#) 2084 .unwrap() 2085 .trim(), 2086 "smpboot: Allowing 4 CPUs, 2 hotplug CPUs" 2087 ); 2088 #[cfg(target_arch = "aarch64")] 2089 assert_eq!( 2090 guest 2091 .ssh_command(r#"dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""#) 2092 .unwrap() 2093 .trim(), 2094 "smp: Brought up 1 node, 2 CPUs" 2095 ); 2096 }); 2097 2098 let _ = child.kill(); 2099 let output = child.wait_with_output().unwrap(); 2100 2101 handle_child_output(r, &output); 2102 } 2103 2104 #[test] 2105 #[cfg(not(feature = "mshv"))] 2106 fn test_cpu_topology_421() { 2107 test_cpu_topology(4, 2, 1, false); 2108 } 2109 2110 #[test] 2111 #[cfg(not(feature = "mshv"))] 2112 fn test_cpu_topology_142() { 2113 test_cpu_topology(1, 4, 2, false); 2114 } 2115 2116 #[test] 2117 #[cfg(not(feature = "mshv"))] 2118 fn test_cpu_topology_262() { 2119 test_cpu_topology(2, 6, 2, false); 2120 } 2121 2122 #[test] 2123 #[cfg(target_arch = "x86_64")] 2124 #[cfg(not(feature = "mshv"))] 2125 fn test_cpu_physical_bits() { 2126 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2127 let guest = Guest::new(Box::new(focal)); 2128 let max_phys_bits: u8 = 36; 2129 let mut child = GuestCommand::new(&guest) 2130 .args(&["--cpus", &format!("max_phys_bits={}", max_phys_bits)]) 2131 .args(&["--memory", "size=512M"]) 2132 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2133 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2134 .default_disks() 2135 .default_net() 2136 .capture_output() 2137 .spawn() 2138 .unwrap(); 2139 2140 let r = std::panic::catch_unwind(|| { 2141 guest.wait_vm_boot(None).unwrap(); 2142 2143 assert!( 2144 guest 2145 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2146 .unwrap() 2147 .trim() 2148 .parse::<u8>() 2149 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2150 ); 2151 }); 2152 2153 let _ = child.kill(); 2154 let output = child.wait_with_output().unwrap(); 2155 2156 handle_child_output(r, &output); 2157 } 2158 2159 #[test] 2160 fn test_cpu_affinity() { 2161 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2162 let guest = Guest::new(Box::new(focal)); 2163 2164 // We need the host to have at least 4 CPUs if we want to be able 2165 // to run this test. 2166 let host_cpus_count = exec_host_command_output("nproc"); 2167 assert!( 2168 String::from_utf8_lossy(&host_cpus_count.stdout) 2169 .trim() 2170 .parse::<u8>() 2171 .unwrap_or(0) 2172 >= 4 2173 ); 2174 2175 let mut child = GuestCommand::new(&guest) 2176 .args(&["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2177 .args(&["--memory", "size=512M"]) 2178 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2179 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2180 .default_disks() 2181 .default_net() 2182 .capture_output() 2183 .spawn() 2184 .unwrap(); 2185 2186 let r = std::panic::catch_unwind(|| { 2187 guest.wait_vm_boot(None).unwrap(); 2188 let pid = child.id(); 2189 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"", pid).as_str()); 2190 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2191 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"", pid).as_str()); 2192 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2193 }); 2194 2195 let _ = child.kill(); 2196 let output = child.wait_with_output().unwrap(); 2197 2198 handle_child_output(r, &output); 2199 } 2200 2201 #[test] 2202 #[cfg(not(feature = "mshv"))] 2203 fn test_large_vm() { 2204 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2205 let guest = Guest::new(Box::new(focal)); 2206 let mut cmd = GuestCommand::new(&guest); 2207 cmd.args(&["--cpus", "boot=48"]) 2208 .args(&["--memory", "size=5120M"]) 2209 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2210 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2211 .args(&["--serial", "tty"]) 2212 .args(&["--console", "off"]) 2213 .capture_output() 2214 .default_disks() 2215 .default_net(); 2216 2217 let mut child = cmd.spawn().unwrap(); 2218 2219 guest.wait_vm_boot(None).unwrap(); 2220 2221 let r = std::panic::catch_unwind(|| { 2222 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2223 assert_eq!( 2224 guest 2225 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2226 .unwrap() 2227 .trim(), 2228 "0-47" 2229 ); 2230 2231 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2232 }); 2233 2234 let _ = child.kill(); 2235 let output = child.wait_with_output().unwrap(); 2236 2237 handle_child_output(r, &output); 2238 } 2239 2240 #[test] 2241 #[cfg(not(feature = "mshv"))] 2242 fn test_huge_memory() { 2243 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2244 let guest = Guest::new(Box::new(focal)); 2245 let mut cmd = GuestCommand::new(&guest); 2246 cmd.args(&["--cpus", "boot=1"]) 2247 .args(&["--memory", "size=128G"]) 2248 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2249 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2250 .capture_output() 2251 .default_disks() 2252 .default_net(); 2253 2254 let mut child = cmd.spawn().unwrap(); 2255 2256 guest.wait_vm_boot(Some(120)).unwrap(); 2257 2258 let r = std::panic::catch_unwind(|| { 2259 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2260 }); 2261 2262 let _ = child.kill(); 2263 let output = child.wait_with_output().unwrap(); 2264 2265 handle_child_output(r, &output); 2266 } 2267 2268 #[test] 2269 fn test_power_button() { 2270 _test_power_button(false); 2271 } 2272 2273 #[test] 2274 #[cfg(not(feature = "mshv"))] 2275 fn test_user_defined_memory_regions() { 2276 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2277 let guest = Guest::new(Box::new(focal)); 2278 let api_socket = temp_api_path(&guest.tmp_dir); 2279 2280 let kernel_path = direct_kernel_boot_path(); 2281 2282 let mut child = GuestCommand::new(&guest) 2283 .args(&["--cpus", "boot=1"]) 2284 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 2285 .args(&[ 2286 "--memory-zone", 2287 "id=mem0,size=1G,hotplug_size=2G", 2288 "id=mem1,size=1G,file=/dev/shm", 2289 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2290 ]) 2291 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2292 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2293 .args(&["--api-socket", &api_socket]) 2294 .capture_output() 2295 .default_disks() 2296 .default_net() 2297 .spawn() 2298 .unwrap(); 2299 2300 let r = std::panic::catch_unwind(|| { 2301 guest.wait_vm_boot(None).unwrap(); 2302 2303 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2304 2305 guest.enable_memory_hotplug(); 2306 2307 resize_zone_command(&api_socket, "mem0", "3G"); 2308 thread::sleep(std::time::Duration::new(5, 0)); 2309 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2310 resize_zone_command(&api_socket, "mem2", "3G"); 2311 thread::sleep(std::time::Duration::new(5, 0)); 2312 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2313 resize_zone_command(&api_socket, "mem0", "2G"); 2314 thread::sleep(std::time::Duration::new(5, 0)); 2315 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2316 resize_zone_command(&api_socket, "mem2", "2G"); 2317 thread::sleep(std::time::Duration::new(5, 0)); 2318 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2319 2320 guest.reboot_linux(0, None); 2321 2322 // Check the amount of RAM after reboot 2323 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2324 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2325 2326 // Check if we can still resize down to the initial 'boot'size 2327 resize_zone_command(&api_socket, "mem0", "1G"); 2328 thread::sleep(std::time::Duration::new(5, 0)); 2329 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2330 resize_zone_command(&api_socket, "mem2", "1G"); 2331 thread::sleep(std::time::Duration::new(5, 0)); 2332 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2333 }); 2334 2335 let _ = child.kill(); 2336 let output = child.wait_with_output().unwrap(); 2337 2338 handle_child_output(r, &output); 2339 } 2340 2341 #[test] 2342 #[cfg(not(feature = "mshv"))] 2343 fn test_guest_numa_nodes() { 2344 _test_guest_numa_nodes(false); 2345 } 2346 2347 #[test] 2348 #[cfg(target_arch = "x86_64")] 2349 fn test_iommu_segments() { 2350 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2351 let focal = UbuntuDiskConfig::new(focal_image); 2352 let guest = Guest::new(Box::new(focal)); 2353 2354 // Prepare another disk file for the virtio-disk device 2355 let test_disk_path = String::from( 2356 guest 2357 .tmp_dir 2358 .as_path() 2359 .join("test-disk.raw") 2360 .to_str() 2361 .unwrap(), 2362 ); 2363 assert!( 2364 exec_host_command_status(format!("truncate {} -s 4M", test_disk_path).as_str()) 2365 .success() 2366 ); 2367 assert!( 2368 exec_host_command_status(format!("mkfs.ext4 {}", test_disk_path).as_str()).success() 2369 ); 2370 2371 let api_socket = temp_api_path(&guest.tmp_dir); 2372 let mut cmd = GuestCommand::new(&guest); 2373 2374 cmd.args(&["--cpus", "boot=1"]) 2375 .args(&["--api-socket", &api_socket]) 2376 .args(&["--memory", "size=512M"]) 2377 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2378 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2379 .args(&["--platform", "num_pci_segments=16,iommu_segments=[1]"]) 2380 .default_disks() 2381 .capture_output() 2382 .default_net(); 2383 2384 let mut child = cmd.spawn().unwrap(); 2385 2386 guest.wait_vm_boot(None).unwrap(); 2387 2388 let r = std::panic::catch_unwind(|| { 2389 let (cmd_success, cmd_output) = remote_command_w_output( 2390 &api_socket, 2391 "add-disk", 2392 Some( 2393 format!( 2394 "path={},id=test0,pci_segment=1,iommu=on", 2395 test_disk_path.as_str() 2396 ) 2397 .as_str(), 2398 ), 2399 ); 2400 assert!(cmd_success); 2401 assert!(String::from_utf8_lossy(&cmd_output) 2402 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2403 2404 // Check IOMMU setup 2405 assert!(guest 2406 .does_device_vendor_pair_match("0x1057", "0x1af4") 2407 .unwrap_or_default()); 2408 assert_eq!( 2409 guest 2410 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2411 .unwrap() 2412 .trim(), 2413 "0001:00:01.0" 2414 ); 2415 }); 2416 2417 let _ = child.kill(); 2418 let output = child.wait_with_output().unwrap(); 2419 2420 handle_child_output(r, &output); 2421 } 2422 2423 #[test] 2424 fn test_pci_msi() { 2425 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2426 let guest = Guest::new(Box::new(focal)); 2427 let mut cmd = GuestCommand::new(&guest); 2428 cmd.args(&["--cpus", "boot=1"]) 2429 .args(&["--memory", "size=512M"]) 2430 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2431 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2432 .capture_output() 2433 .default_disks() 2434 .default_net(); 2435 2436 let mut child = cmd.spawn().unwrap(); 2437 2438 guest.wait_vm_boot(None).unwrap(); 2439 2440 #[cfg(target_arch = "x86_64")] 2441 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2442 #[cfg(target_arch = "aarch64")] 2443 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2444 2445 let r = std::panic::catch_unwind(|| { 2446 assert_eq!( 2447 guest 2448 .ssh_command(grep_cmd) 2449 .unwrap() 2450 .trim() 2451 .parse::<u32>() 2452 .unwrap_or_default(), 2453 12 2454 ); 2455 }); 2456 2457 let _ = child.kill(); 2458 let output = child.wait_with_output().unwrap(); 2459 2460 handle_child_output(r, &output); 2461 } 2462 2463 #[test] 2464 fn test_virtio_net_ctrl_queue() { 2465 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2466 let guest = Guest::new(Box::new(focal)); 2467 let mut cmd = GuestCommand::new(&guest); 2468 cmd.args(&["--cpus", "boot=1"]) 2469 .args(&["--memory", "size=512M"]) 2470 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2471 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2472 .capture_output() 2473 .default_disks() 2474 .default_net(); 2475 2476 let mut child = cmd.spawn().unwrap(); 2477 2478 guest.wait_vm_boot(None).unwrap(); 2479 2480 #[cfg(target_arch = "aarch64")] 2481 let iface = "enp0s4"; 2482 #[cfg(target_arch = "x86_64")] 2483 let iface = "ens4"; 2484 2485 let r = std::panic::catch_unwind(|| { 2486 assert_eq!( 2487 guest 2488 .ssh_command( 2489 format!("sudo ethtool -K {} rx-gro-hw off && echo success", iface).as_str() 2490 ) 2491 .unwrap() 2492 .trim(), 2493 "success" 2494 ); 2495 }); 2496 2497 let _ = child.kill(); 2498 let output = child.wait_with_output().unwrap(); 2499 2500 handle_child_output(r, &output); 2501 } 2502 2503 #[test] 2504 #[cfg(not(feature = "mshv"))] 2505 fn test_pci_multiple_segments() { 2506 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2507 let guest = Guest::new(Box::new(focal)); 2508 2509 // Prepare another disk file for the virtio-disk device 2510 let test_disk_path = String::from( 2511 guest 2512 .tmp_dir 2513 .as_path() 2514 .join("test-disk.raw") 2515 .to_str() 2516 .unwrap(), 2517 ); 2518 assert!( 2519 exec_host_command_status(format!("truncate {} -s 4M", test_disk_path).as_str()) 2520 .success() 2521 ); 2522 assert!( 2523 exec_host_command_status(format!("mkfs.ext4 {}", test_disk_path).as_str()).success() 2524 ); 2525 2526 let mut cmd = GuestCommand::new(&guest); 2527 cmd.args(&["--cpus", "boot=1"]) 2528 .args(&["--memory", "size=512M"]) 2529 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2530 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2531 .args(&["--platform", "num_pci_segments=16"]) 2532 .args(&[ 2533 "--disk", 2534 format!( 2535 "path={}", 2536 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2537 ) 2538 .as_str(), 2539 format!( 2540 "path={}", 2541 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2542 ) 2543 .as_str(), 2544 format!("path={},pci_segment=15", test_disk_path).as_str(), 2545 ]) 2546 .capture_output() 2547 .default_net(); 2548 2549 let mut child = cmd.spawn().unwrap(); 2550 2551 guest.wait_vm_boot(None).unwrap(); 2552 2553 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2554 2555 let r = std::panic::catch_unwind(|| { 2556 // There should be 16 PCI host bridges in the guest. 2557 assert_eq!( 2558 guest 2559 .ssh_command(grep_cmd) 2560 .unwrap() 2561 .trim() 2562 .parse::<u32>() 2563 .unwrap_or_default(), 2564 16 2565 ); 2566 2567 // Check both if /dev/vdc exists and if the block size is 4M. 2568 assert_eq!( 2569 guest 2570 .ssh_command("lsblk | grep vdc | grep -c 4M") 2571 .unwrap() 2572 .trim() 2573 .parse::<u32>() 2574 .unwrap_or_default(), 2575 1 2576 ); 2577 2578 // Mount the device. 2579 guest.ssh_command("mkdir mount_image").unwrap(); 2580 guest 2581 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 2582 .unwrap(); 2583 // Grant all users with write permission. 2584 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 2585 2586 // Write something to the device. 2587 guest 2588 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 2589 .unwrap(); 2590 2591 // Check the content of the block device. The file "foo" should 2592 // contain "bar". 2593 assert_eq!( 2594 guest 2595 .ssh_command("sudo cat mount_image/foo") 2596 .unwrap() 2597 .trim(), 2598 "bar" 2599 ); 2600 }); 2601 2602 let _ = child.kill(); 2603 let output = child.wait_with_output().unwrap(); 2604 2605 handle_child_output(r, &output); 2606 } 2607 2608 #[test] 2609 fn test_direct_kernel_boot() { 2610 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2611 let guest = Guest::new(Box::new(focal)); 2612 2613 let kernel_path = direct_kernel_boot_path(); 2614 2615 let mut child = GuestCommand::new(&guest) 2616 .args(&["--cpus", "boot=1"]) 2617 .args(&["--memory", "size=512M"]) 2618 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2619 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2620 .default_disks() 2621 .default_net() 2622 .capture_output() 2623 .spawn() 2624 .unwrap(); 2625 2626 let r = std::panic::catch_unwind(|| { 2627 guest.wait_vm_boot(None).unwrap(); 2628 2629 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2630 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2631 2632 let grep_cmd = if cfg!(target_arch = "x86_64") { 2633 "grep -c PCI-MSI /proc/interrupts" 2634 } else { 2635 "grep -c ITS-MSI /proc/interrupts" 2636 }; 2637 assert_eq!( 2638 guest 2639 .ssh_command(grep_cmd) 2640 .unwrap() 2641 .trim() 2642 .parse::<u32>() 2643 .unwrap_or_default(), 2644 12 2645 ); 2646 }); 2647 2648 let _ = child.kill(); 2649 let output = child.wait_with_output().unwrap(); 2650 2651 handle_child_output(r, &output); 2652 } 2653 2654 fn _test_virtio_block(image_name: &str, disable_io_uring: bool) { 2655 let focal = UbuntuDiskConfig::new(image_name.to_string()); 2656 let guest = Guest::new(Box::new(focal)); 2657 2658 let mut workload_path = dirs::home_dir().unwrap(); 2659 workload_path.push("workloads"); 2660 2661 let mut blk_file_path = workload_path; 2662 blk_file_path.push("blk.img"); 2663 2664 let kernel_path = direct_kernel_boot_path(); 2665 2666 let mut cloud_child = GuestCommand::new(&guest) 2667 .args(&["--cpus", "boot=4"]) 2668 .args(&["--memory", "size=512M,shared=on"]) 2669 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2670 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2671 .args(&[ 2672 "--disk", 2673 format!( 2674 "path={}", 2675 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2676 ) 2677 .as_str(), 2678 format!( 2679 "path={}", 2680 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2681 ) 2682 .as_str(), 2683 format!( 2684 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={}", 2685 blk_file_path.to_str().unwrap(), 2686 disable_io_uring 2687 ) 2688 .as_str(), 2689 ]) 2690 .default_net() 2691 .capture_output() 2692 .spawn() 2693 .unwrap(); 2694 2695 let r = std::panic::catch_unwind(|| { 2696 guest.wait_vm_boot(None).unwrap(); 2697 2698 // Check both if /dev/vdc exists and if the block size is 16M. 2699 assert_eq!( 2700 guest 2701 .ssh_command("lsblk | grep vdc | grep -c 16M") 2702 .unwrap() 2703 .trim() 2704 .parse::<u32>() 2705 .unwrap_or_default(), 2706 1 2707 ); 2708 2709 // Check both if /dev/vdc exists and if this block is RO. 2710 assert_eq!( 2711 guest 2712 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 2713 .unwrap() 2714 .trim() 2715 .parse::<u32>() 2716 .unwrap_or_default(), 2717 1 2718 ); 2719 2720 // Check if the number of queues is 4. 2721 assert_eq!( 2722 guest 2723 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 2724 .unwrap() 2725 .trim() 2726 .parse::<u32>() 2727 .unwrap_or_default(), 2728 4 2729 ); 2730 }); 2731 2732 let _ = cloud_child.kill(); 2733 let output = cloud_child.wait_with_output().unwrap(); 2734 2735 handle_child_output(r, &output); 2736 } 2737 2738 #[test] 2739 fn test_virtio_block() { 2740 _test_virtio_block(FOCAL_IMAGE_NAME, false) 2741 } 2742 2743 #[test] 2744 fn test_virtio_block_disable_io_uring() { 2745 _test_virtio_block(FOCAL_IMAGE_NAME, true) 2746 } 2747 2748 #[test] 2749 fn test_virtio_block_qcow2() { 2750 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false) 2751 } 2752 2753 #[test] 2754 fn test_virtio_block_vhd() { 2755 let mut workload_path = dirs::home_dir().unwrap(); 2756 workload_path.push("workloads"); 2757 2758 let mut raw_file_path = workload_path.clone(); 2759 let mut vhd_file_path = workload_path; 2760 raw_file_path.push(FOCAL_IMAGE_NAME); 2761 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 2762 2763 // Generate VHD file from RAW file 2764 std::process::Command::new("qemu-img") 2765 .arg("convert") 2766 .arg("-p") 2767 .args(&["-f", "raw"]) 2768 .args(&["-O", "vpc"]) 2769 .args(&["-o", "subformat=fixed"]) 2770 .arg(raw_file_path.to_str().unwrap()) 2771 .arg(vhd_file_path.to_str().unwrap()) 2772 .output() 2773 .expect("Expect generating VHD image from RAW image"); 2774 2775 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false) 2776 } 2777 2778 #[test] 2779 fn test_virtio_block_vhdx() { 2780 let mut workload_path = dirs::home_dir().unwrap(); 2781 workload_path.push("workloads"); 2782 2783 let mut raw_file_path = workload_path.clone(); 2784 let mut vhdx_file_path = workload_path; 2785 raw_file_path.push(FOCAL_IMAGE_NAME); 2786 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 2787 2788 // Generate dynamic VHDX file from RAW file 2789 std::process::Command::new("qemu-img") 2790 .arg("convert") 2791 .arg("-p") 2792 .args(&["-f", "raw"]) 2793 .args(&["-O", "vhdx"]) 2794 .arg(raw_file_path.to_str().unwrap()) 2795 .arg(vhdx_file_path.to_str().unwrap()) 2796 .output() 2797 .expect("Expect generating dynamic VHDx image from RAW image"); 2798 2799 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false) 2800 } 2801 2802 #[test] 2803 fn test_virtio_block_dynamic_vhdx_expand() { 2804 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 2805 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 2806 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 2807 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 2808 2809 let mut workload_path = dirs::home_dir().unwrap(); 2810 workload_path.push("workloads"); 2811 2812 let mut vhdx_file_path = workload_path; 2813 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 2814 let vhdx_path = vhdx_file_path.to_str().unwrap(); 2815 2816 // Generate a 100 MiB dynamic VHDX file 2817 std::process::Command::new("qemu-img") 2818 .arg("create") 2819 .args(&["-f", "vhdx"]) 2820 .arg(vhdx_path) 2821 .arg(VIRTUAL_DISK_SIZE.to_string()) 2822 .output() 2823 .expect("Expect generating dynamic VHDx image from RAW image"); 2824 2825 // Check if the size matches with empty VHDx file size 2826 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 2827 2828 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2829 let guest = Guest::new(Box::new(focal)); 2830 let kernel_path = direct_kernel_boot_path(); 2831 2832 let mut cloud_child = GuestCommand::new(&guest) 2833 .args(&["--cpus", "boot=1"]) 2834 .args(&["--memory", "size=512M"]) 2835 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2836 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2837 .args(&[ 2838 "--disk", 2839 format!( 2840 "path={}", 2841 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2842 ) 2843 .as_str(), 2844 format!( 2845 "path={}", 2846 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2847 ) 2848 .as_str(), 2849 format!("path={}", vhdx_path).as_str(), 2850 ]) 2851 .default_net() 2852 .capture_output() 2853 .spawn() 2854 .unwrap(); 2855 2856 let r = std::panic::catch_unwind(|| { 2857 guest.wait_vm_boot(None).unwrap(); 2858 2859 // Check both if /dev/vdc exists and if the block size is 100 MiB. 2860 assert_eq!( 2861 guest 2862 .ssh_command("lsblk | grep vdc | grep -c 100M") 2863 .unwrap() 2864 .trim() 2865 .parse::<u32>() 2866 .unwrap_or_default(), 2867 1 2868 ); 2869 2870 // Write 100 MB of data to the VHDx disk 2871 guest 2872 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 2873 .unwrap(); 2874 }); 2875 2876 // Check if the size matches with expected expanded VHDx file size 2877 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 2878 2879 let _ = cloud_child.kill(); 2880 let output = cloud_child.wait_with_output().unwrap(); 2881 2882 handle_child_output(r, &output); 2883 } 2884 2885 fn vhdx_image_size(disk_name: &str) -> u64 { 2886 std::fs::File::open(disk_name) 2887 .unwrap() 2888 .seek(SeekFrom::End(0)) 2889 .unwrap() 2890 } 2891 2892 #[test] 2893 fn test_virtio_block_direct_and_firmware() { 2894 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2895 let guest = Guest::new(Box::new(focal)); 2896 2897 // The OS disk must be copied to a location that is not backed by 2898 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 2899 // with EINVAL because tmpfs doesn't support this flag. 2900 let mut workloads_path = dirs::home_dir().unwrap(); 2901 workloads_path.push("workloads"); 2902 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 2903 let mut os_path = os_dir.as_path().to_path_buf(); 2904 os_path.push("osdisk.img"); 2905 rate_limited_copy( 2906 &guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 2907 os_path.as_path(), 2908 ) 2909 .expect("copying of OS disk failed"); 2910 2911 let mut child = GuestCommand::new(&guest) 2912 .args(&["--cpus", "boot=1"]) 2913 .args(&["--memory", "size=512M"]) 2914 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 2915 .args(&[ 2916 "--disk", 2917 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 2918 format!( 2919 "path={}", 2920 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2921 ) 2922 .as_str(), 2923 ]) 2924 .default_net() 2925 .capture_output() 2926 .spawn() 2927 .unwrap(); 2928 2929 let r = std::panic::catch_unwind(|| { 2930 guest.wait_vm_boot(Some(120)).unwrap(); 2931 }); 2932 2933 let _ = child.kill(); 2934 let output = child.wait_with_output().unwrap(); 2935 2936 handle_child_output(r, &output); 2937 } 2938 2939 #[test] 2940 fn test_vhost_user_net_default() { 2941 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 2942 } 2943 2944 #[test] 2945 fn test_vhost_user_net_named_tap() { 2946 test_vhost_user_net( 2947 Some("mytap0"), 2948 2, 2949 &prepare_vhost_user_net_daemon, 2950 false, 2951 false, 2952 ) 2953 } 2954 2955 #[test] 2956 fn test_vhost_user_net_existing_tap() { 2957 test_vhost_user_net( 2958 Some("vunet-tap0"), 2959 2, 2960 &prepare_vhost_user_net_daemon, 2961 false, 2962 false, 2963 ) 2964 } 2965 2966 #[test] 2967 fn test_vhost_user_net_multiple_queues() { 2968 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 2969 } 2970 2971 #[test] 2972 fn test_vhost_user_net_tap_multiple_queues() { 2973 test_vhost_user_net( 2974 Some("vunet-tap1"), 2975 4, 2976 &prepare_vhost_user_net_daemon, 2977 false, 2978 false, 2979 ) 2980 } 2981 2982 #[test] 2983 fn test_vhost_user_net_host_mac() { 2984 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 2985 } 2986 2987 #[test] 2988 fn test_vhost_user_net_client_mode() { 2989 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 2990 } 2991 2992 #[test] 2993 fn test_vhost_user_blk_default() { 2994 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 2995 } 2996 2997 #[test] 2998 fn test_vhost_user_blk_readonly() { 2999 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3000 } 3001 3002 #[test] 3003 fn test_vhost_user_blk_direct() { 3004 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3005 } 3006 3007 #[test] 3008 fn test_boot_from_vhost_user_blk_default() { 3009 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3010 } 3011 3012 #[test] 3013 #[cfg(target_arch = "x86_64")] 3014 fn test_split_irqchip() { 3015 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3016 let guest = Guest::new(Box::new(focal)); 3017 3018 let mut child = GuestCommand::new(&guest) 3019 .args(&["--cpus", "boot=1"]) 3020 .args(&["--memory", "size=512M"]) 3021 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3022 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3023 .default_disks() 3024 .default_net() 3025 .capture_output() 3026 .spawn() 3027 .unwrap(); 3028 3029 let r = std::panic::catch_unwind(|| { 3030 guest.wait_vm_boot(None).unwrap(); 3031 3032 assert_eq!( 3033 guest 3034 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3035 .unwrap() 3036 .trim() 3037 .parse::<u32>() 3038 .unwrap_or(1), 3039 0 3040 ); 3041 assert_eq!( 3042 guest 3043 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3044 .unwrap() 3045 .trim() 3046 .parse::<u32>() 3047 .unwrap_or(1), 3048 0 3049 ); 3050 }); 3051 3052 let _ = child.kill(); 3053 let output = child.wait_with_output().unwrap(); 3054 3055 handle_child_output(r, &output); 3056 } 3057 3058 #[test] 3059 #[cfg(target_arch = "x86_64")] 3060 fn test_dmi_serial_number() { 3061 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3062 let guest = Guest::new(Box::new(focal)); 3063 3064 let mut child = GuestCommand::new(&guest) 3065 .args(&["--cpus", "boot=1"]) 3066 .args(&["--memory", "size=512M"]) 3067 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3068 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3069 .args(&["--platform", "serial_number=a=b;c=d"]) 3070 .default_disks() 3071 .default_net() 3072 .capture_output() 3073 .spawn() 3074 .unwrap(); 3075 3076 let r = std::panic::catch_unwind(|| { 3077 guest.wait_vm_boot(None).unwrap(); 3078 3079 assert_eq!( 3080 guest 3081 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3082 .unwrap() 3083 .trim(), 3084 "a=b;c=d" 3085 ); 3086 }); 3087 3088 let _ = child.kill(); 3089 let output = child.wait_with_output().unwrap(); 3090 3091 handle_child_output(r, &output); 3092 } 3093 3094 #[test] 3095 fn test_virtio_fs() { 3096 _test_virtio_fs(&prepare_virtiofsd, false, None) 3097 } 3098 3099 #[test] 3100 fn test_virtio_fs_hotplug() { 3101 _test_virtio_fs(&prepare_virtiofsd, true, None) 3102 } 3103 3104 #[test] 3105 #[cfg(not(feature = "mshv"))] 3106 fn test_virtio_fs_multi_segment_hotplug() { 3107 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3108 } 3109 3110 #[test] 3111 #[cfg(not(feature = "mshv"))] 3112 fn test_virtio_fs_multi_segment() { 3113 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3114 } 3115 3116 #[test] 3117 fn test_virtio_pmem_persist_writes() { 3118 test_virtio_pmem(false, false) 3119 } 3120 3121 #[test] 3122 fn test_virtio_pmem_discard_writes() { 3123 test_virtio_pmem(true, false) 3124 } 3125 3126 #[test] 3127 fn test_virtio_pmem_with_size() { 3128 test_virtio_pmem(true, true) 3129 } 3130 3131 #[test] 3132 fn test_boot_from_virtio_pmem() { 3133 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3134 let guest = Guest::new(Box::new(focal)); 3135 3136 let kernel_path = direct_kernel_boot_path(); 3137 3138 let mut child = GuestCommand::new(&guest) 3139 .args(&["--cpus", "boot=1"]) 3140 .args(&["--memory", "size=512M"]) 3141 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3142 .args(&[ 3143 "--disk", 3144 format!( 3145 "path={}", 3146 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3147 ) 3148 .as_str(), 3149 ]) 3150 .default_net() 3151 .args(&[ 3152 "--pmem", 3153 format!( 3154 "file={},size={}", 3155 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3156 fs::metadata(&guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3157 .unwrap() 3158 .len() 3159 ) 3160 .as_str(), 3161 ]) 3162 .args(&[ 3163 "--cmdline", 3164 DIRECT_KERNEL_BOOT_CMDLINE 3165 .replace("vda1", "pmem0p1") 3166 .as_str(), 3167 ]) 3168 .capture_output() 3169 .spawn() 3170 .unwrap(); 3171 3172 let r = std::panic::catch_unwind(|| { 3173 guest.wait_vm_boot(None).unwrap(); 3174 3175 // Simple checks to validate the VM booted properly 3176 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3177 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3178 }); 3179 3180 let _ = child.kill(); 3181 let output = child.wait_with_output().unwrap(); 3182 3183 handle_child_output(r, &output); 3184 } 3185 3186 #[test] 3187 fn test_multiple_network_interfaces() { 3188 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3189 let guest = Guest::new(Box::new(focal)); 3190 3191 let kernel_path = direct_kernel_boot_path(); 3192 3193 let mut child = GuestCommand::new(&guest) 3194 .args(&["--cpus", "boot=1"]) 3195 .args(&["--memory", "size=512M"]) 3196 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3197 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3198 .default_disks() 3199 .args(&[ 3200 "--net", 3201 guest.default_net_string().as_str(), 3202 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3203 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3204 ]) 3205 .capture_output() 3206 .spawn() 3207 .unwrap(); 3208 3209 let r = std::panic::catch_unwind(|| { 3210 guest.wait_vm_boot(None).unwrap(); 3211 3212 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3213 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3214 3215 // 3 network interfaces + default localhost ==> 4 interfaces 3216 assert_eq!( 3217 guest 3218 .ssh_command("ip -o link | wc -l") 3219 .unwrap() 3220 .trim() 3221 .parse::<u32>() 3222 .unwrap_or_default(), 3223 4 3224 ); 3225 }); 3226 3227 let _ = child.kill(); 3228 let output = child.wait_with_output().unwrap(); 3229 3230 handle_child_output(r, &output); 3231 } 3232 3233 #[test] 3234 #[cfg(target_arch = "aarch64")] 3235 fn test_pmu_on() { 3236 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3237 let guest = Guest::new(Box::new(focal)); 3238 let mut child = GuestCommand::new(&guest) 3239 .args(&["--cpus", "boot=1"]) 3240 .args(&["--memory", "size=512M"]) 3241 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3242 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3243 .default_disks() 3244 .default_net() 3245 .capture_output() 3246 .spawn() 3247 .unwrap(); 3248 3249 let r = std::panic::catch_unwind(|| { 3250 guest.wait_vm_boot(None).unwrap(); 3251 3252 // Test that PMU exists. 3253 assert_eq!( 3254 guest 3255 .ssh_command(GREP_PMU_IRQ_CMD) 3256 .unwrap() 3257 .trim() 3258 .parse::<u32>() 3259 .unwrap_or_default(), 3260 1 3261 ); 3262 }); 3263 3264 let _ = child.kill(); 3265 let output = child.wait_with_output().unwrap(); 3266 3267 handle_child_output(r, &output); 3268 } 3269 3270 #[test] 3271 fn test_serial_off() { 3272 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3273 let guest = Guest::new(Box::new(focal)); 3274 let mut child = GuestCommand::new(&guest) 3275 .args(&["--cpus", "boot=1"]) 3276 .args(&["--memory", "size=512M"]) 3277 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3278 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3279 .default_disks() 3280 .default_net() 3281 .args(&["--serial", "off"]) 3282 .capture_output() 3283 .spawn() 3284 .unwrap(); 3285 3286 let r = std::panic::catch_unwind(|| { 3287 guest.wait_vm_boot(None).unwrap(); 3288 3289 // Test that there is no ttyS0 3290 assert_eq!( 3291 guest 3292 .ssh_command(GREP_SERIAL_IRQ_CMD) 3293 .unwrap() 3294 .trim() 3295 .parse::<u32>() 3296 .unwrap_or(1), 3297 0 3298 ); 3299 }); 3300 3301 let _ = child.kill(); 3302 let output = child.wait_with_output().unwrap(); 3303 3304 handle_child_output(r, &output); 3305 } 3306 3307 #[test] 3308 fn test_serial_null() { 3309 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3310 let guest = Guest::new(Box::new(focal)); 3311 let mut cmd = GuestCommand::new(&guest); 3312 #[cfg(target_arch = "x86_64")] 3313 let console_str: &str = "console=ttyS0"; 3314 #[cfg(target_arch = "aarch64")] 3315 let console_str: &str = "console=ttyAMA0"; 3316 3317 cmd.args(&["--cpus", "boot=1"]) 3318 .args(&["--memory", "size=512M"]) 3319 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3320 .args(&[ 3321 "--cmdline", 3322 DIRECT_KERNEL_BOOT_CMDLINE 3323 .replace("console=hvc0 ", console_str) 3324 .as_str(), 3325 ]) 3326 .default_disks() 3327 .default_net() 3328 .args(&["--serial", "null"]) 3329 .args(&["--console", "off"]) 3330 .capture_output(); 3331 3332 let mut child = cmd.spawn().unwrap(); 3333 3334 let r = std::panic::catch_unwind(|| { 3335 guest.wait_vm_boot(None).unwrap(); 3336 3337 // Test that there is a ttyS0 3338 assert_eq!( 3339 guest 3340 .ssh_command(GREP_SERIAL_IRQ_CMD) 3341 .unwrap() 3342 .trim() 3343 .parse::<u32>() 3344 .unwrap_or_default(), 3345 1 3346 ); 3347 }); 3348 3349 let _ = child.kill(); 3350 let output = child.wait_with_output().unwrap(); 3351 handle_child_output(r, &output); 3352 3353 let r = std::panic::catch_unwind(|| { 3354 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3355 }); 3356 3357 handle_child_output(r, &output); 3358 } 3359 3360 #[test] 3361 fn test_serial_tty() { 3362 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3363 let guest = Guest::new(Box::new(focal)); 3364 3365 let kernel_path = direct_kernel_boot_path(); 3366 3367 #[cfg(target_arch = "x86_64")] 3368 let console_str: &str = "console=ttyS0"; 3369 #[cfg(target_arch = "aarch64")] 3370 let console_str: &str = "console=ttyAMA0"; 3371 3372 let mut child = GuestCommand::new(&guest) 3373 .args(&["--cpus", "boot=1"]) 3374 .args(&["--memory", "size=512M"]) 3375 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3376 .args(&[ 3377 "--cmdline", 3378 DIRECT_KERNEL_BOOT_CMDLINE 3379 .replace("console=hvc0 ", console_str) 3380 .as_str(), 3381 ]) 3382 .default_disks() 3383 .default_net() 3384 .args(&["--serial", "tty"]) 3385 .args(&["--console", "off"]) 3386 .capture_output() 3387 .spawn() 3388 .unwrap(); 3389 3390 let r = std::panic::catch_unwind(|| { 3391 guest.wait_vm_boot(None).unwrap(); 3392 3393 // Test that there is a ttyS0 3394 assert_eq!( 3395 guest 3396 .ssh_command(GREP_SERIAL_IRQ_CMD) 3397 .unwrap() 3398 .trim() 3399 .parse::<u32>() 3400 .unwrap_or_default(), 3401 1 3402 ); 3403 }); 3404 3405 // This sleep is needed to wait for the login prompt 3406 thread::sleep(std::time::Duration::new(2, 0)); 3407 3408 let _ = child.kill(); 3409 let output = child.wait_with_output().unwrap(); 3410 handle_child_output(r, &output); 3411 3412 let r = std::panic::catch_unwind(|| { 3413 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3414 }); 3415 3416 handle_child_output(r, &output); 3417 } 3418 3419 #[test] 3420 fn test_serial_file() { 3421 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3422 let guest = Guest::new(Box::new(focal)); 3423 3424 let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output"); 3425 #[cfg(target_arch = "x86_64")] 3426 let console_str: &str = "console=ttyS0"; 3427 #[cfg(target_arch = "aarch64")] 3428 let console_str: &str = "console=ttyAMA0"; 3429 3430 let mut child = GuestCommand::new(&guest) 3431 .args(&["--cpus", "boot=1"]) 3432 .args(&["--memory", "size=512M"]) 3433 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3434 .args(&[ 3435 "--cmdline", 3436 DIRECT_KERNEL_BOOT_CMDLINE 3437 .replace("console=hvc0 ", console_str) 3438 .as_str(), 3439 ]) 3440 .default_disks() 3441 .default_net() 3442 .args(&[ 3443 "--serial", 3444 format!("file={}", serial_path.to_str().unwrap()).as_str(), 3445 ]) 3446 .capture_output() 3447 .spawn() 3448 .unwrap(); 3449 3450 let r = std::panic::catch_unwind(|| { 3451 guest.wait_vm_boot(None).unwrap(); 3452 3453 // Test that there is a ttyS0 3454 assert_eq!( 3455 guest 3456 .ssh_command(GREP_SERIAL_IRQ_CMD) 3457 .unwrap() 3458 .trim() 3459 .parse::<u32>() 3460 .unwrap_or_default(), 3461 1 3462 ); 3463 3464 guest.ssh_command("sudo shutdown -h now").unwrap(); 3465 }); 3466 3467 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3468 let _ = child.kill(); 3469 let output = child.wait_with_output().unwrap(); 3470 handle_child_output(r, &output); 3471 3472 let r = std::panic::catch_unwind(|| { 3473 // Check that the cloud-hypervisor binary actually terminated 3474 assert!(output.status.success()); 3475 3476 // Do this check after shutdown of the VM as an easy way to ensure 3477 // all writes are flushed to disk 3478 let mut f = std::fs::File::open(serial_path).unwrap(); 3479 let mut buf = String::new(); 3480 f.read_to_string(&mut buf).unwrap(); 3481 assert!(buf.contains(CONSOLE_TEST_STRING)); 3482 }); 3483 3484 handle_child_output(r, &output); 3485 } 3486 3487 #[test] 3488 fn test_pty_interaction() { 3489 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3490 let guest = Guest::new(Box::new(focal)); 3491 let api_socket = temp_api_path(&guest.tmp_dir); 3492 let serial_option = if cfg!(target_arch = "x86_64") { 3493 " console=ttyS0" 3494 } else { 3495 " console=ttyAMA0" 3496 }; 3497 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 3498 3499 let mut child = GuestCommand::new(&guest) 3500 .args(&["--cpus", "boot=1"]) 3501 .args(&["--memory", "size=512M"]) 3502 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3503 .args(&["--cmdline", &cmdline]) 3504 .default_disks() 3505 .default_net() 3506 .args(&["--serial", "null"]) 3507 .args(&["--console", "pty"]) 3508 .args(&["--api-socket", &api_socket]) 3509 .spawn() 3510 .unwrap(); 3511 3512 let r = std::panic::catch_unwind(|| { 3513 guest.wait_vm_boot(None).unwrap(); 3514 // Get pty fd for console 3515 let console_path = get_pty_path(&api_socket, "console"); 3516 // TODO: Get serial pty test working 3517 let mut cf = std::fs::OpenOptions::new() 3518 .write(true) 3519 .read(true) 3520 .open(console_path) 3521 .unwrap(); 3522 3523 // Some dumb sleeps but we don't want to write 3524 // before the console is up and we don't want 3525 // to try and write the next line before the 3526 // login process is ready. 3527 thread::sleep(std::time::Duration::new(5, 0)); 3528 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 3529 thread::sleep(std::time::Duration::new(2, 0)); 3530 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 3531 thread::sleep(std::time::Duration::new(2, 0)); 3532 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 3533 thread::sleep(std::time::Duration::new(2, 0)); 3534 3535 // read pty and ensure they have a login shell 3536 // some fairly hacky workarounds to avoid looping 3537 // forever in case the channel is blocked getting output 3538 let ptyc = pty_read(cf); 3539 let mut empty = 0; 3540 let mut prev = String::new(); 3541 loop { 3542 thread::sleep(std::time::Duration::new(2, 0)); 3543 match ptyc.try_recv() { 3544 Ok(line) => { 3545 empty = 0; 3546 prev = prev + &line; 3547 if prev.contains("test_pty_console") { 3548 break; 3549 } 3550 } 3551 Err(mpsc::TryRecvError::Empty) => { 3552 empty += 1; 3553 assert!(empty <= 5, "No login on pty"); 3554 } 3555 _ => panic!("No login on pty"), 3556 } 3557 } 3558 3559 guest.ssh_command("sudo shutdown -h now").unwrap(); 3560 }); 3561 3562 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3563 let _ = child.kill(); 3564 let output = child.wait_with_output().unwrap(); 3565 handle_child_output(r, &output); 3566 3567 let r = std::panic::catch_unwind(|| { 3568 // Check that the cloud-hypervisor binary actually terminated 3569 assert!(output.status.success()) 3570 }); 3571 handle_child_output(r, &output); 3572 } 3573 3574 #[test] 3575 fn test_virtio_console() { 3576 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3577 let guest = Guest::new(Box::new(focal)); 3578 3579 let kernel_path = direct_kernel_boot_path(); 3580 3581 let mut child = GuestCommand::new(&guest) 3582 .args(&["--cpus", "boot=1"]) 3583 .args(&["--memory", "size=512M"]) 3584 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3585 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3586 .default_disks() 3587 .default_net() 3588 .args(&["--console", "tty"]) 3589 .args(&["--serial", "null"]) 3590 .capture_output() 3591 .spawn() 3592 .unwrap(); 3593 3594 let text = String::from("On a branch floating down river a cricket, singing."); 3595 let cmd = format!("echo {} | sudo tee /dev/hvc0", text); 3596 3597 let r = std::panic::catch_unwind(|| { 3598 guest.wait_vm_boot(None).unwrap(); 3599 3600 assert!(guest 3601 .does_device_vendor_pair_match("0x1043", "0x1af4") 3602 .unwrap_or_default()); 3603 3604 guest.ssh_command(&cmd).unwrap(); 3605 }); 3606 3607 let _ = child.kill(); 3608 let output = child.wait_with_output().unwrap(); 3609 handle_child_output(r, &output); 3610 3611 let r = std::panic::catch_unwind(|| { 3612 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 3613 }); 3614 3615 handle_child_output(r, &output); 3616 } 3617 3618 #[test] 3619 fn test_console_file() { 3620 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3621 let guest = Guest::new(Box::new(focal)); 3622 3623 let console_path = guest.tmp_dir.as_path().join("/tmp/console-output"); 3624 let mut child = GuestCommand::new(&guest) 3625 .args(&["--cpus", "boot=1"]) 3626 .args(&["--memory", "size=512M"]) 3627 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3628 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3629 .default_disks() 3630 .default_net() 3631 .args(&[ 3632 "--console", 3633 format!("file={}", console_path.to_str().unwrap()).as_str(), 3634 ]) 3635 .capture_output() 3636 .spawn() 3637 .unwrap(); 3638 3639 guest.wait_vm_boot(None).unwrap(); 3640 3641 guest.ssh_command("sudo shutdown -h now").unwrap(); 3642 3643 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3644 let _ = child.kill(); 3645 let output = child.wait_with_output().unwrap(); 3646 3647 let r = std::panic::catch_unwind(|| { 3648 // Check that the cloud-hypervisor binary actually terminated 3649 assert!(output.status.success()); 3650 3651 // Do this check after shutdown of the VM as an easy way to ensure 3652 // all writes are flushed to disk 3653 let mut f = std::fs::File::open(console_path).unwrap(); 3654 let mut buf = String::new(); 3655 f.read_to_string(&mut buf).unwrap(); 3656 3657 if !buf.contains(CONSOLE_TEST_STRING) { 3658 eprintln!( 3659 "\n\n==== Console file output ====\n\n{}\n\n==== End console file output ====", 3660 buf 3661 ); 3662 } 3663 assert!(buf.contains(CONSOLE_TEST_STRING)); 3664 }); 3665 3666 handle_child_output(r, &output); 3667 } 3668 3669 #[test] 3670 #[cfg(target_arch = "x86_64")] 3671 #[cfg(not(feature = "mshv"))] 3672 #[ignore = "See #4324"] 3673 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 3674 // backed networking interfaces, bound through a simple bridge on the host. 3675 // So if the nested cloud-hypervisor succeeds in getting a directly 3676 // assigned interface from its cloud-hypervisor host, we should be able to 3677 // ssh into it, and verify that it's running with the right kernel command 3678 // line (We tag the command line from cloud-hypervisor for that purpose). 3679 // The third device is added to validate that hotplug works correctly since 3680 // it is being added to the L2 VM through hotplugging mechanism. 3681 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 3682 // vfio device support 3683 fn test_vfio() { 3684 setup_vfio_network_interfaces(); 3685 3686 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3687 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 3688 3689 let mut workload_path = dirs::home_dir().unwrap(); 3690 workload_path.push("workloads"); 3691 3692 let kernel_path = direct_kernel_boot_path(); 3693 3694 let mut vfio_path = workload_path.clone(); 3695 vfio_path.push("vfio"); 3696 3697 let mut cloud_init_vfio_base_path = vfio_path.clone(); 3698 cloud_init_vfio_base_path.push("cloudinit.img"); 3699 3700 // We copy our cloudinit into the vfio mount point, for the nested 3701 // cloud-hypervisor guest to use. 3702 rate_limited_copy( 3703 &guest.disk_config.disk(DiskType::CloudInit).unwrap(), 3704 &cloud_init_vfio_base_path, 3705 ) 3706 .expect("copying of cloud-init disk failed"); 3707 3708 let mut vfio_disk_path = workload_path.clone(); 3709 vfio_disk_path.push("vfio.img"); 3710 3711 // Create the vfio disk image 3712 let output = Command::new("mkfs.ext4") 3713 .arg("-d") 3714 .arg(vfio_path.to_str().unwrap()) 3715 .arg(vfio_disk_path.to_str().unwrap()) 3716 .arg("2g") 3717 .output() 3718 .unwrap(); 3719 if !output.status.success() { 3720 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 3721 panic!("mkfs.ext4 command generated an error"); 3722 } 3723 3724 let mut blk_file_path = workload_path; 3725 blk_file_path.push("blk.img"); 3726 3727 let vfio_tap0 = "vfio-tap0"; 3728 let vfio_tap1 = "vfio-tap1"; 3729 let vfio_tap2 = "vfio-tap2"; 3730 let vfio_tap3 = "vfio-tap3"; 3731 3732 let mut child = GuestCommand::new(&guest) 3733 .args(&["--cpus", "boot=4"]) 3734 .args(&["--memory", "size=2G,hugepages=on,shared=on"]) 3735 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3736 .args(&[ 3737 "--disk", 3738 format!( 3739 "path={}", 3740 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3741 ) 3742 .as_str(), 3743 format!( 3744 "path={}", 3745 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3746 ) 3747 .as_str(), 3748 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 3749 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 3750 ]) 3751 .args(&[ 3752 "--cmdline", 3753 format!( 3754 "{} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts", 3755 DIRECT_KERNEL_BOOT_CMDLINE 3756 ) 3757 .as_str(), 3758 ]) 3759 .args(&[ 3760 "--net", 3761 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 3762 format!( 3763 "tap={},mac={},iommu=on", 3764 vfio_tap1, guest.network.l2_guest_mac1 3765 ) 3766 .as_str(), 3767 format!( 3768 "tap={},mac={},iommu=on", 3769 vfio_tap2, guest.network.l2_guest_mac2 3770 ) 3771 .as_str(), 3772 format!( 3773 "tap={},mac={},iommu=on", 3774 vfio_tap3, guest.network.l2_guest_mac3 3775 ) 3776 .as_str(), 3777 ]) 3778 .capture_output() 3779 .spawn() 3780 .unwrap(); 3781 3782 thread::sleep(std::time::Duration::new(30, 0)); 3783 3784 let r = std::panic::catch_unwind(|| { 3785 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 3786 thread::sleep(std::time::Duration::new(120, 0)); 3787 3788 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 3789 // added to its kernel command line. 3790 // Let's ssh into it and verify that it's there. If it is it means 3791 // we're in the right guest (The L2 one) because the QEMU L1 guest 3792 // does not have this command line tag. 3793 assert_eq!( 3794 guest 3795 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 3796 .unwrap() 3797 .trim() 3798 .parse::<u32>() 3799 .unwrap_or_default(), 3800 1 3801 ); 3802 3803 // Let's also verify from the second virtio-net device passed to 3804 // the L2 VM. 3805 assert_eq!( 3806 guest 3807 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 3808 .unwrap() 3809 .trim() 3810 .parse::<u32>() 3811 .unwrap_or_default(), 3812 1 3813 ); 3814 3815 // Check the amount of PCI devices appearing in L2 VM. 3816 assert_eq!( 3817 guest 3818 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3819 .unwrap() 3820 .trim() 3821 .parse::<u32>() 3822 .unwrap_or_default(), 3823 8, 3824 ); 3825 3826 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 3827 assert_eq!( 3828 guest 3829 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 3830 .unwrap() 3831 .trim() 3832 .parse::<u32>() 3833 .unwrap_or_default(), 3834 1 3835 ); 3836 3837 // Hotplug an extra virtio-net device through L2 VM. 3838 guest 3839 .ssh_command_l1( 3840 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 3841 ) 3842 .unwrap(); 3843 guest 3844 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 3845 .unwrap(); 3846 let vfio_hotplug_output = guest 3847 .ssh_command_l1( 3848 "sudo /mnt/ch-remote \ 3849 --api-socket=/tmp/ch_api.sock \ 3850 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 3851 ) 3852 .unwrap(); 3853 assert!(vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}")); 3854 3855 thread::sleep(std::time::Duration::new(10, 0)); 3856 3857 // Let's also verify from the third virtio-net device passed to 3858 // the L2 VM. This third device has been hotplugged through the L2 3859 // VM, so this is our way to validate hotplug works for VFIO PCI. 3860 assert_eq!( 3861 guest 3862 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 3863 .unwrap() 3864 .trim() 3865 .parse::<u32>() 3866 .unwrap_or_default(), 3867 1 3868 ); 3869 3870 // Check the amount of PCI devices appearing in L2 VM. 3871 // There should be one more device than before, raising the count 3872 // up to 9 PCI devices. 3873 assert_eq!( 3874 guest 3875 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3876 .unwrap() 3877 .trim() 3878 .parse::<u32>() 3879 .unwrap_or_default(), 3880 9, 3881 ); 3882 3883 // Let's now verify that we can correctly remove the virtio-net 3884 // device through the "remove-device" command responsible for 3885 // unplugging VFIO devices. 3886 guest 3887 .ssh_command_l1( 3888 "sudo /mnt/ch-remote \ 3889 --api-socket=/tmp/ch_api.sock \ 3890 remove-device vfio123", 3891 ) 3892 .unwrap(); 3893 thread::sleep(std::time::Duration::new(10, 0)); 3894 3895 // Check the amount of PCI devices appearing in L2 VM is back down 3896 // to 8 devices. 3897 assert_eq!( 3898 guest 3899 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3900 .unwrap() 3901 .trim() 3902 .parse::<u32>() 3903 .unwrap_or_default(), 3904 8, 3905 ); 3906 3907 // Perform memory hotplug in L2 and validate the memory is showing 3908 // up as expected. In order to check, we will use the virtio-net 3909 // device already passed through L2 as a VFIO device, this will 3910 // verify that VFIO devices are functional with memory hotplug. 3911 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 3912 guest 3913 .ssh_command_l2_1( 3914 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 3915 ) 3916 .unwrap(); 3917 guest 3918 .ssh_command_l1( 3919 "sudo /mnt/ch-remote \ 3920 --api-socket=/tmp/ch_api.sock \ 3921 resize --memory=1073741824", 3922 ) 3923 .unwrap(); 3924 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 3925 }); 3926 3927 let _ = child.kill(); 3928 let output = child.wait_with_output().unwrap(); 3929 3930 cleanup_vfio_network_interfaces(); 3931 3932 handle_child_output(r, &output); 3933 } 3934 3935 #[test] 3936 fn test_direct_kernel_boot_noacpi() { 3937 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3938 let guest = Guest::new(Box::new(focal)); 3939 3940 let kernel_path = direct_kernel_boot_path(); 3941 3942 let mut child = GuestCommand::new(&guest) 3943 .args(&["--cpus", "boot=1"]) 3944 .args(&["--memory", "size=512M"]) 3945 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3946 .args(&[ 3947 "--cmdline", 3948 format!("{} acpi=off", DIRECT_KERNEL_BOOT_CMDLINE).as_str(), 3949 ]) 3950 .default_disks() 3951 .default_net() 3952 .capture_output() 3953 .spawn() 3954 .unwrap(); 3955 3956 let r = std::panic::catch_unwind(|| { 3957 guest.wait_vm_boot(None).unwrap(); 3958 3959 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3960 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3961 }); 3962 3963 let _ = child.kill(); 3964 let output = child.wait_with_output().unwrap(); 3965 3966 handle_child_output(r, &output); 3967 } 3968 3969 #[test] 3970 fn test_reboot() { 3971 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 3972 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3973 3974 vec![Box::new(bionic), Box::new(focal)] 3975 .drain(..) 3976 .for_each(|disk_config| { 3977 let guest = Guest::new(disk_config); 3978 3979 let mut cmd = GuestCommand::new(&guest); 3980 cmd.args(&["--cpus", "boot=1"]) 3981 .args(&["--memory", "size=512M"]) 3982 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3983 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3984 .default_disks() 3985 .default_net() 3986 .capture_output(); 3987 3988 let mut child = cmd.spawn().unwrap(); 3989 3990 let r = std::panic::catch_unwind(|| { 3991 guest.wait_vm_boot(Some(120)).unwrap(); 3992 3993 let fd_count_1 = get_fd_count(child.id()); 3994 guest.reboot_linux(0, Some(120)); 3995 let fd_count_2 = get_fd_count(child.id()); 3996 assert_eq!(fd_count_1, fd_count_2); 3997 3998 guest.ssh_command("sudo shutdown -h now").unwrap(); 3999 }); 4000 4001 let _ = child.wait_timeout(std::time::Duration::from_secs(40)); 4002 let _ = child.kill(); 4003 let output = child.wait_with_output().unwrap(); 4004 handle_child_output(r, &output); 4005 4006 let r = std::panic::catch_unwind(|| { 4007 // Check that the cloud-hypervisor binary actually terminated 4008 assert!(output.status.success()); 4009 }); 4010 4011 handle_child_output(r, &output); 4012 }); 4013 } 4014 4015 #[test] 4016 fn test_virtio_vsock() { 4017 _test_virtio_vsock(false) 4018 } 4019 4020 #[test] 4021 fn test_virtio_vsock_hotplug() { 4022 _test_virtio_vsock(true); 4023 } 4024 4025 #[test] 4026 // Start cloud-hypervisor with no VM parameters, only the API server running. 4027 // From the API: Create a VM, boot it and check that it looks as expected. 4028 fn test_api_create_boot() { 4029 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4030 let guest = Guest::new(Box::new(focal)); 4031 4032 let api_socket = temp_api_path(&guest.tmp_dir); 4033 4034 let mut child = GuestCommand::new(&guest) 4035 .args(&["--api-socket", &api_socket]) 4036 .capture_output() 4037 .spawn() 4038 .unwrap(); 4039 4040 thread::sleep(std::time::Duration::new(1, 0)); 4041 4042 // Verify API server is running 4043 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4044 4045 // Create the VM first 4046 let cpu_count: u8 = 4; 4047 let http_body = guest.api_create_body( 4048 cpu_count, 4049 direct_kernel_boot_path().to_str().unwrap(), 4050 DIRECT_KERNEL_BOOT_CMDLINE, 4051 ); 4052 4053 let temp_config_path = guest.tmp_dir.as_path().join("config"); 4054 std::fs::write(&temp_config_path, http_body).unwrap(); 4055 4056 remote_command( 4057 &api_socket, 4058 "create", 4059 Some(temp_config_path.as_os_str().to_str().unwrap()), 4060 ); 4061 4062 // Then boot it 4063 remote_command(&api_socket, "boot", None); 4064 thread::sleep(std::time::Duration::new(20, 0)); 4065 4066 let r = std::panic::catch_unwind(|| { 4067 // Check that the VM booted as expected 4068 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4069 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4070 }); 4071 4072 let _ = child.kill(); 4073 let output = child.wait_with_output().unwrap(); 4074 4075 handle_child_output(r, &output); 4076 } 4077 4078 #[test] 4079 // Start cloud-hypervisor with no VM parameters, only the API server running. 4080 // From the API: Create a VM, boot it and check it can be shutdown and then 4081 // booted again 4082 fn test_api_shutdown() { 4083 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4084 let guest = Guest::new(Box::new(focal)); 4085 4086 let api_socket = temp_api_path(&guest.tmp_dir); 4087 4088 let mut child = GuestCommand::new(&guest) 4089 .args(&["--api-socket", &api_socket]) 4090 .capture_output() 4091 .spawn() 4092 .unwrap(); 4093 4094 thread::sleep(std::time::Duration::new(1, 0)); 4095 4096 // Verify API server is running 4097 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4098 4099 // Create the VM first 4100 let cpu_count: u8 = 4; 4101 let http_body = guest.api_create_body( 4102 cpu_count, 4103 direct_kernel_boot_path().to_str().unwrap(), 4104 DIRECT_KERNEL_BOOT_CMDLINE, 4105 ); 4106 4107 curl_command( 4108 &api_socket, 4109 "PUT", 4110 "http://localhost/api/v1/vm.create", 4111 Some(&http_body), 4112 ); 4113 4114 // Then boot it 4115 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4116 4117 let r = std::panic::catch_unwind(|| { 4118 guest.wait_vm_boot(None).unwrap(); 4119 4120 // Check that the VM booted as expected 4121 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4122 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4123 4124 // Shutdown without powering off to prevent filesystem corruption 4125 guest.ssh_command("sudo shutdown -H now").unwrap(); 4126 4127 // Then shut it down 4128 curl_command( 4129 &api_socket, 4130 "PUT", 4131 "http://localhost/api/v1/vm.shutdown", 4132 None, 4133 ); 4134 4135 // Then boot it again 4136 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4137 4138 guest.wait_vm_boot(None).unwrap(); 4139 4140 // Check that the VM booted as expected 4141 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4142 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4143 }); 4144 4145 let _ = child.kill(); 4146 let output = child.wait_with_output().unwrap(); 4147 4148 handle_child_output(r, &output); 4149 } 4150 4151 #[test] 4152 // Start cloud-hypervisor with no VM parameters, only the API server running. 4153 // From the API: Create a VM, boot it and check it can be deleted and then recreated 4154 // booted again. 4155 fn test_api_delete() { 4156 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4157 let guest = Guest::new(Box::new(focal)); 4158 4159 let api_socket = temp_api_path(&guest.tmp_dir); 4160 4161 let mut child = GuestCommand::new(&guest) 4162 .args(&["--api-socket", &api_socket]) 4163 .capture_output() 4164 .spawn() 4165 .unwrap(); 4166 4167 thread::sleep(std::time::Duration::new(1, 0)); 4168 4169 // Verify API server is running 4170 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4171 4172 // Create the VM first 4173 let cpu_count: u8 = 4; 4174 let http_body = guest.api_create_body( 4175 cpu_count, 4176 direct_kernel_boot_path().to_str().unwrap(), 4177 DIRECT_KERNEL_BOOT_CMDLINE, 4178 ); 4179 4180 let r = std::panic::catch_unwind(|| { 4181 curl_command( 4182 &api_socket, 4183 "PUT", 4184 "http://localhost/api/v1/vm.create", 4185 Some(&http_body), 4186 ); 4187 4188 // Then boot it 4189 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4190 4191 guest.wait_vm_boot(None).unwrap(); 4192 4193 // Check that the VM booted as expected 4194 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4195 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4196 4197 // Shutdown without powering off to prevent filesystem corruption 4198 guest.ssh_command("sudo shutdown -H now").unwrap(); 4199 4200 // Then delete it 4201 curl_command( 4202 &api_socket, 4203 "PUT", 4204 "http://localhost/api/v1/vm.delete", 4205 None, 4206 ); 4207 4208 curl_command( 4209 &api_socket, 4210 "PUT", 4211 "http://localhost/api/v1/vm.create", 4212 Some(&http_body), 4213 ); 4214 4215 // Then boot it again 4216 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4217 4218 guest.wait_vm_boot(None).unwrap(); 4219 4220 // Check that the VM booted as expected 4221 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4222 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4223 }); 4224 4225 let _ = child.kill(); 4226 let output = child.wait_with_output().unwrap(); 4227 4228 handle_child_output(r, &output); 4229 } 4230 4231 #[test] 4232 // Start cloud-hypervisor with no VM parameters, only the API server running. 4233 // From the API: Create a VM, boot it and check that it looks as expected. 4234 // Then we pause the VM, check that it's no longer available. 4235 // Finally we resume the VM and check that it's available. 4236 fn test_api_pause_resume() { 4237 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4238 let guest = Guest::new(Box::new(focal)); 4239 4240 let api_socket = temp_api_path(&guest.tmp_dir); 4241 4242 let mut child = GuestCommand::new(&guest) 4243 .args(&["--api-socket", &api_socket]) 4244 .capture_output() 4245 .spawn() 4246 .unwrap(); 4247 4248 thread::sleep(std::time::Duration::new(1, 0)); 4249 4250 // Verify API server is running 4251 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4252 4253 // Create the VM first 4254 let cpu_count: u8 = 4; 4255 let http_body = guest.api_create_body( 4256 cpu_count, 4257 direct_kernel_boot_path().to_str().unwrap(), 4258 DIRECT_KERNEL_BOOT_CMDLINE, 4259 ); 4260 curl_command( 4261 &api_socket, 4262 "PUT", 4263 "http://localhost/api/v1/vm.create", 4264 Some(&http_body), 4265 ); 4266 4267 // Then boot it 4268 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4269 thread::sleep(std::time::Duration::new(20, 0)); 4270 4271 let r = std::panic::catch_unwind(|| { 4272 // Check that the VM booted as expected 4273 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4274 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4275 4276 // We now pause the VM 4277 assert!(remote_command(&api_socket, "pause", None)); 4278 4279 // Check pausing again fails 4280 assert!(!remote_command(&api_socket, "pause", None)); 4281 4282 thread::sleep(std::time::Duration::new(2, 0)); 4283 4284 // SSH into the VM should fail 4285 assert!(ssh_command_ip( 4286 "grep -c processor /proc/cpuinfo", 4287 &guest.network.guest_ip, 4288 2, 4289 5 4290 ) 4291 .is_err()); 4292 4293 // Resume the VM 4294 assert!(remote_command(&api_socket, "resume", None)); 4295 4296 // Check resuming again fails 4297 assert!(!remote_command(&api_socket, "resume", None)); 4298 4299 thread::sleep(std::time::Duration::new(2, 0)); 4300 4301 // Now we should be able to SSH back in and get the right number of CPUs 4302 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4303 }); 4304 4305 let _ = child.kill(); 4306 let output = child.wait_with_output().unwrap(); 4307 4308 handle_child_output(r, &output); 4309 } 4310 4311 #[test] 4312 fn test_virtio_iommu() { 4313 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4314 } 4315 4316 #[test] 4317 // We cannot force the software running in the guest to reprogram the BAR 4318 // with some different addresses, but we have a reliable way of testing it 4319 // with a standard Linux kernel. 4320 // By removing a device from the PCI tree, and then rescanning the tree, 4321 // Linux consistently chooses to reorganize the PCI device BARs to other 4322 // locations in the guest address space. 4323 // This test creates a dedicated PCI network device to be checked as being 4324 // properly probed first, then removing it, and adding it again by doing a 4325 // rescan. 4326 fn test_pci_bar_reprogramming() { 4327 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4328 let guest = Guest::new(Box::new(focal)); 4329 4330 #[cfg(target_arch = "x86_64")] 4331 let kernel_path = direct_kernel_boot_path(); 4332 #[cfg(target_arch = "aarch64")] 4333 let kernel_path = edk2_path(); 4334 4335 let mut child = GuestCommand::new(&guest) 4336 .args(&["--cpus", "boot=1"]) 4337 .args(&["--memory", "size=512M"]) 4338 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4339 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4340 .default_disks() 4341 .args(&[ 4342 "--net", 4343 guest.default_net_string().as_str(), 4344 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4345 ]) 4346 .capture_output() 4347 .spawn() 4348 .unwrap(); 4349 4350 let r = std::panic::catch_unwind(|| { 4351 guest.wait_vm_boot(None).unwrap(); 4352 4353 // 2 network interfaces + default localhost ==> 3 interfaces 4354 assert_eq!( 4355 guest 4356 .ssh_command("ip -o link | wc -l") 4357 .unwrap() 4358 .trim() 4359 .parse::<u32>() 4360 .unwrap_or_default(), 4361 3 4362 ); 4363 4364 let init_bar_addr = guest 4365 .ssh_command( 4366 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4367 ) 4368 .unwrap(); 4369 4370 // Remove the PCI device 4371 guest 4372 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4373 .unwrap(); 4374 4375 // Only 1 network interface left + default localhost ==> 2 interfaces 4376 assert_eq!( 4377 guest 4378 .ssh_command("ip -o link | wc -l") 4379 .unwrap() 4380 .trim() 4381 .parse::<u32>() 4382 .unwrap_or_default(), 4383 2 4384 ); 4385 4386 // Remove the PCI device 4387 guest 4388 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4389 .unwrap(); 4390 4391 // Back to 2 network interface + default localhost ==> 3 interfaces 4392 assert_eq!( 4393 guest 4394 .ssh_command("ip -o link | wc -l") 4395 .unwrap() 4396 .trim() 4397 .parse::<u32>() 4398 .unwrap_or_default(), 4399 3 4400 ); 4401 4402 let new_bar_addr = guest 4403 .ssh_command( 4404 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4405 ) 4406 .unwrap(); 4407 4408 // Let's compare the BAR addresses for our virtio-net device. 4409 // They should be different as we expect the BAR reprogramming 4410 // to have happened. 4411 assert_ne!(init_bar_addr, new_bar_addr); 4412 }); 4413 4414 let _ = child.kill(); 4415 let output = child.wait_with_output().unwrap(); 4416 4417 handle_child_output(r, &output); 4418 } 4419 4420 #[test] 4421 fn test_memory_mergeable_off() { 4422 test_memory_mergeable(false) 4423 } 4424 4425 #[test] 4426 #[cfg(target_arch = "x86_64")] 4427 fn test_cpu_hotplug() { 4428 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4429 let guest = Guest::new(Box::new(focal)); 4430 let api_socket = temp_api_path(&guest.tmp_dir); 4431 4432 let kernel_path = direct_kernel_boot_path(); 4433 4434 let mut child = GuestCommand::new(&guest) 4435 .args(&["--cpus", "boot=2,max=4"]) 4436 .args(&["--memory", "size=512M"]) 4437 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4438 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4439 .default_disks() 4440 .default_net() 4441 .args(&["--api-socket", &api_socket]) 4442 .capture_output() 4443 .spawn() 4444 .unwrap(); 4445 4446 let r = std::panic::catch_unwind(|| { 4447 guest.wait_vm_boot(None).unwrap(); 4448 4449 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4450 4451 // Resize the VM 4452 let desired_vcpus = 4; 4453 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4454 4455 guest 4456 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4457 .unwrap(); 4458 guest 4459 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4460 .unwrap(); 4461 thread::sleep(std::time::Duration::new(10, 0)); 4462 assert_eq!( 4463 guest.get_cpu_count().unwrap_or_default(), 4464 u32::from(desired_vcpus) 4465 ); 4466 4467 guest.reboot_linux(0, None); 4468 4469 assert_eq!( 4470 guest.get_cpu_count().unwrap_or_default(), 4471 u32::from(desired_vcpus) 4472 ); 4473 4474 // Resize the VM 4475 let desired_vcpus = 2; 4476 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4477 4478 thread::sleep(std::time::Duration::new(10, 0)); 4479 assert_eq!( 4480 guest.get_cpu_count().unwrap_or_default(), 4481 u32::from(desired_vcpus) 4482 ); 4483 4484 // Resize the VM back up to 4 4485 let desired_vcpus = 4; 4486 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4487 4488 guest 4489 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4490 .unwrap(); 4491 guest 4492 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4493 .unwrap(); 4494 thread::sleep(std::time::Duration::new(10, 0)); 4495 assert_eq!( 4496 guest.get_cpu_count().unwrap_or_default(), 4497 u32::from(desired_vcpus) 4498 ); 4499 }); 4500 4501 let _ = child.kill(); 4502 let output = child.wait_with_output().unwrap(); 4503 4504 handle_child_output(r, &output); 4505 } 4506 4507 #[test] 4508 fn test_memory_hotplug() { 4509 #[cfg(target_arch = "aarch64")] 4510 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4511 #[cfg(target_arch = "x86_64")] 4512 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4513 let focal = UbuntuDiskConfig::new(focal_image); 4514 let guest = Guest::new(Box::new(focal)); 4515 let api_socket = temp_api_path(&guest.tmp_dir); 4516 4517 #[cfg(target_arch = "aarch64")] 4518 let kernel_path = edk2_path(); 4519 #[cfg(target_arch = "x86_64")] 4520 let kernel_path = direct_kernel_boot_path(); 4521 4522 let mut child = GuestCommand::new(&guest) 4523 .args(&["--cpus", "boot=2,max=4"]) 4524 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4525 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4526 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4527 .default_disks() 4528 .default_net() 4529 .args(&["--balloon", "size=0"]) 4530 .args(&["--api-socket", &api_socket]) 4531 .capture_output() 4532 .spawn() 4533 .unwrap(); 4534 4535 let r = std::panic::catch_unwind(|| { 4536 guest.wait_vm_boot(None).unwrap(); 4537 4538 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4539 4540 guest.enable_memory_hotplug(); 4541 4542 // Add RAM to the VM 4543 let desired_ram = 1024 << 20; 4544 resize_command(&api_socket, None, Some(desired_ram), None, None); 4545 4546 thread::sleep(std::time::Duration::new(10, 0)); 4547 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4548 4549 // Use balloon to remove RAM from the VM 4550 let desired_balloon = 512 << 20; 4551 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4552 4553 thread::sleep(std::time::Duration::new(10, 0)); 4554 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4555 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4556 4557 guest.reboot_linux(0, None); 4558 4559 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4560 4561 // Use balloon add RAM to the VM 4562 let desired_balloon = 0; 4563 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4564 4565 thread::sleep(std::time::Duration::new(10, 0)); 4566 4567 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4568 4569 guest.enable_memory_hotplug(); 4570 4571 // Add RAM to the VM 4572 let desired_ram = 2048 << 20; 4573 resize_command(&api_socket, None, Some(desired_ram), None, None); 4574 4575 thread::sleep(std::time::Duration::new(10, 0)); 4576 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4577 4578 // Remove RAM to the VM (only applies after reboot) 4579 let desired_ram = 1024 << 20; 4580 resize_command(&api_socket, None, Some(desired_ram), None, None); 4581 4582 guest.reboot_linux(1, None); 4583 4584 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4585 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4586 }); 4587 4588 let _ = child.kill(); 4589 let output = child.wait_with_output().unwrap(); 4590 4591 handle_child_output(r, &output); 4592 } 4593 4594 #[test] 4595 #[cfg(not(feature = "mshv"))] 4596 fn test_virtio_mem() { 4597 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4598 let guest = Guest::new(Box::new(focal)); 4599 let api_socket = temp_api_path(&guest.tmp_dir); 4600 4601 let kernel_path = direct_kernel_boot_path(); 4602 4603 let mut child = GuestCommand::new(&guest) 4604 .args(&["--cpus", "boot=2,max=4"]) 4605 .args(&[ 4606 "--memory", 4607 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4608 ]) 4609 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4610 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4611 .default_disks() 4612 .default_net() 4613 .args(&["--api-socket", &api_socket]) 4614 .capture_output() 4615 .spawn() 4616 .unwrap(); 4617 4618 let r = std::panic::catch_unwind(|| { 4619 guest.wait_vm_boot(None).unwrap(); 4620 4621 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4622 4623 guest.enable_memory_hotplug(); 4624 4625 // Add RAM to the VM 4626 let desired_ram = 1024 << 20; 4627 resize_command(&api_socket, None, Some(desired_ram), None, None); 4628 4629 thread::sleep(std::time::Duration::new(10, 0)); 4630 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4631 4632 // Add RAM to the VM 4633 let desired_ram = 2048 << 20; 4634 resize_command(&api_socket, None, Some(desired_ram), None, None); 4635 4636 thread::sleep(std::time::Duration::new(10, 0)); 4637 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4638 4639 // Remove RAM from the VM 4640 let desired_ram = 1024 << 20; 4641 resize_command(&api_socket, None, Some(desired_ram), None, None); 4642 4643 thread::sleep(std::time::Duration::new(10, 0)); 4644 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4645 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4646 4647 guest.reboot_linux(0, None); 4648 4649 // Check the amount of memory after reboot is 1GiB 4650 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4651 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4652 4653 // Check we can still resize to 512MiB 4654 let desired_ram = 512 << 20; 4655 resize_command(&api_socket, None, Some(desired_ram), None, None); 4656 thread::sleep(std::time::Duration::new(10, 0)); 4657 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4658 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4659 }); 4660 4661 let _ = child.kill(); 4662 let output = child.wait_with_output().unwrap(); 4663 4664 handle_child_output(r, &output); 4665 } 4666 4667 #[test] 4668 #[cfg(target_arch = "x86_64")] 4669 #[cfg(not(feature = "mshv"))] 4670 // Test both vCPU and memory resizing together 4671 fn test_resize() { 4672 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4673 let guest = Guest::new(Box::new(focal)); 4674 let api_socket = temp_api_path(&guest.tmp_dir); 4675 4676 let kernel_path = direct_kernel_boot_path(); 4677 4678 let mut child = GuestCommand::new(&guest) 4679 .args(&["--cpus", "boot=2,max=4"]) 4680 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4681 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4682 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4683 .default_disks() 4684 .default_net() 4685 .args(&["--api-socket", &api_socket]) 4686 .capture_output() 4687 .spawn() 4688 .unwrap(); 4689 4690 let r = std::panic::catch_unwind(|| { 4691 guest.wait_vm_boot(None).unwrap(); 4692 4693 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4694 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4695 4696 guest.enable_memory_hotplug(); 4697 4698 // Resize the VM 4699 let desired_vcpus = 4; 4700 let desired_ram = 1024 << 20; 4701 resize_command( 4702 &api_socket, 4703 Some(desired_vcpus), 4704 Some(desired_ram), 4705 None, 4706 None, 4707 ); 4708 4709 guest 4710 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4711 .unwrap(); 4712 guest 4713 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4714 .unwrap(); 4715 thread::sleep(std::time::Duration::new(10, 0)); 4716 assert_eq!( 4717 guest.get_cpu_count().unwrap_or_default(), 4718 u32::from(desired_vcpus) 4719 ); 4720 4721 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4722 }); 4723 4724 let _ = child.kill(); 4725 let output = child.wait_with_output().unwrap(); 4726 4727 handle_child_output(r, &output); 4728 } 4729 4730 #[test] 4731 fn test_memory_overhead() { 4732 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4733 let guest = Guest::new(Box::new(focal)); 4734 4735 let kernel_path = direct_kernel_boot_path(); 4736 4737 let guest_memory_size_kb = 512 * 1024; 4738 4739 let mut child = GuestCommand::new(&guest) 4740 .args(&["--cpus", "boot=1"]) 4741 .args(&[ 4742 "--memory", 4743 format!("size={}K", guest_memory_size_kb).as_str(), 4744 ]) 4745 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4746 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4747 .default_disks() 4748 .capture_output() 4749 .spawn() 4750 .unwrap(); 4751 4752 thread::sleep(std::time::Duration::new(20, 0)); 4753 4754 let r = std::panic::catch_unwind(|| { 4755 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 4756 eprintln!( 4757 "Guest memory overhead: {} vs {}", 4758 overhead, MAXIMUM_VMM_OVERHEAD_KB 4759 ); 4760 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 4761 }); 4762 4763 let _ = child.kill(); 4764 let output = child.wait_with_output().unwrap(); 4765 4766 handle_child_output(r, &output); 4767 } 4768 4769 #[test] 4770 fn test_disk_hotplug() { 4771 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4772 let guest = Guest::new(Box::new(focal)); 4773 4774 #[cfg(target_arch = "x86_64")] 4775 let kernel_path = direct_kernel_boot_path(); 4776 #[cfg(target_arch = "aarch64")] 4777 let kernel_path = edk2_path(); 4778 4779 let api_socket = temp_api_path(&guest.tmp_dir); 4780 4781 let mut child = GuestCommand::new(&guest) 4782 .args(&["--api-socket", &api_socket]) 4783 .args(&["--cpus", "boot=1"]) 4784 .args(&["--memory", "size=512M"]) 4785 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4786 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4787 .default_disks() 4788 .default_net() 4789 .capture_output() 4790 .spawn() 4791 .unwrap(); 4792 4793 let r = std::panic::catch_unwind(|| { 4794 guest.wait_vm_boot(None).unwrap(); 4795 4796 // Check /dev/vdc is not there 4797 assert_eq!( 4798 guest 4799 .ssh_command("lsblk | grep -c vdc.*16M || true") 4800 .unwrap() 4801 .trim() 4802 .parse::<u32>() 4803 .unwrap_or(1), 4804 0 4805 ); 4806 4807 // Now let's add the extra disk. 4808 let mut blk_file_path = dirs::home_dir().unwrap(); 4809 blk_file_path.push("workloads"); 4810 blk_file_path.push("blk.img"); 4811 let (cmd_success, cmd_output) = remote_command_w_output( 4812 &api_socket, 4813 "add-disk", 4814 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4815 ); 4816 assert!(cmd_success); 4817 assert!(String::from_utf8_lossy(&cmd_output) 4818 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4819 4820 thread::sleep(std::time::Duration::new(10, 0)); 4821 4822 // Check that /dev/vdc exists and the block size is 16M. 4823 assert_eq!( 4824 guest 4825 .ssh_command("lsblk | grep vdc | grep -c 16M") 4826 .unwrap() 4827 .trim() 4828 .parse::<u32>() 4829 .unwrap_or_default(), 4830 1 4831 ); 4832 // And check the block device can be read. 4833 guest 4834 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4835 .unwrap(); 4836 4837 // Let's remove it the extra disk. 4838 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4839 thread::sleep(std::time::Duration::new(5, 0)); 4840 // And check /dev/vdc is not there 4841 assert_eq!( 4842 guest 4843 .ssh_command("lsblk | grep -c vdc.*16M || true") 4844 .unwrap() 4845 .trim() 4846 .parse::<u32>() 4847 .unwrap_or(1), 4848 0 4849 ); 4850 4851 // And add it back to validate unplug did work correctly. 4852 let (cmd_success, cmd_output) = remote_command_w_output( 4853 &api_socket, 4854 "add-disk", 4855 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4856 ); 4857 assert!(cmd_success); 4858 assert!(String::from_utf8_lossy(&cmd_output) 4859 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4860 4861 thread::sleep(std::time::Duration::new(10, 0)); 4862 4863 // Check that /dev/vdc exists and the block size is 16M. 4864 assert_eq!( 4865 guest 4866 .ssh_command("lsblk | grep vdc | grep -c 16M") 4867 .unwrap() 4868 .trim() 4869 .parse::<u32>() 4870 .unwrap_or_default(), 4871 1 4872 ); 4873 // And check the block device can be read. 4874 guest 4875 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4876 .unwrap(); 4877 4878 // Reboot the VM. 4879 guest.reboot_linux(0, None); 4880 4881 // Check still there after reboot 4882 assert_eq!( 4883 guest 4884 .ssh_command("lsblk | grep vdc | grep -c 16M") 4885 .unwrap() 4886 .trim() 4887 .parse::<u32>() 4888 .unwrap_or_default(), 4889 1 4890 ); 4891 4892 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4893 4894 thread::sleep(std::time::Duration::new(20, 0)); 4895 4896 // Check device has gone away 4897 assert_eq!( 4898 guest 4899 .ssh_command("lsblk | grep -c vdc.*16M || true") 4900 .unwrap() 4901 .trim() 4902 .parse::<u32>() 4903 .unwrap_or(1), 4904 0 4905 ); 4906 4907 guest.reboot_linux(1, None); 4908 4909 // Check device still absent 4910 assert_eq!( 4911 guest 4912 .ssh_command("lsblk | grep -c vdc.*16M || true") 4913 .unwrap() 4914 .trim() 4915 .parse::<u32>() 4916 .unwrap_or(1), 4917 0 4918 ); 4919 }); 4920 4921 let _ = child.kill(); 4922 let output = child.wait_with_output().unwrap(); 4923 4924 handle_child_output(r, &output); 4925 } 4926 4927 #[test] 4928 #[cfg(target_arch = "x86_64")] 4929 fn test_virtio_block_topology() { 4930 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4931 let guest = Guest::new(Box::new(focal)); 4932 4933 let kernel_path = direct_kernel_boot_path(); 4934 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 4935 4936 Command::new("qemu-img") 4937 .args(&[ 4938 "create", 4939 "-f", 4940 "raw", 4941 test_disk_path.to_str().unwrap(), 4942 "16M", 4943 ]) 4944 .output() 4945 .expect("qemu-img command failed"); 4946 let out = Command::new("losetup") 4947 .args(&[ 4948 "--show", 4949 "--find", 4950 "--sector-size=4096", 4951 test_disk_path.to_str().unwrap(), 4952 ]) 4953 .output() 4954 .expect("failed to create loop device") 4955 .stdout; 4956 let _tmp = String::from_utf8_lossy(&out); 4957 let loop_dev = _tmp.trim(); 4958 4959 let mut child = GuestCommand::new(&guest) 4960 .args(&["--cpus", "boot=1"]) 4961 .args(&["--memory", "size=512M"]) 4962 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4963 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4964 .args(&[ 4965 "--disk", 4966 format!( 4967 "path={}", 4968 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4969 ) 4970 .as_str(), 4971 format!( 4972 "path={}", 4973 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4974 ) 4975 .as_str(), 4976 format!("path={}", loop_dev).as_str(), 4977 ]) 4978 .default_net() 4979 .capture_output() 4980 .spawn() 4981 .unwrap(); 4982 4983 let r = std::panic::catch_unwind(|| { 4984 guest.wait_vm_boot(None).unwrap(); 4985 4986 // MIN-IO column 4987 assert_eq!( 4988 guest 4989 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 4990 .unwrap() 4991 .trim() 4992 .parse::<u32>() 4993 .unwrap_or_default(), 4994 4096 4995 ); 4996 // PHY-SEC column 4997 assert_eq!( 4998 guest 4999 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5000 .unwrap() 5001 .trim() 5002 .parse::<u32>() 5003 .unwrap_or_default(), 5004 4096 5005 ); 5006 // LOG-SEC column 5007 assert_eq!( 5008 guest 5009 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5010 .unwrap() 5011 .trim() 5012 .parse::<u32>() 5013 .unwrap_or_default(), 5014 4096 5015 ); 5016 }); 5017 5018 let _ = child.kill(); 5019 let output = child.wait_with_output().unwrap(); 5020 5021 handle_child_output(r, &output); 5022 5023 Command::new("losetup") 5024 .args(&["-d", loop_dev]) 5025 .output() 5026 .expect("loop device not found"); 5027 } 5028 5029 #[test] 5030 fn test_virtio_balloon_deflate_on_oom() { 5031 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5032 let guest = Guest::new(Box::new(focal)); 5033 5034 let kernel_path = direct_kernel_boot_path(); 5035 5036 let api_socket = temp_api_path(&guest.tmp_dir); 5037 5038 //Let's start a 4G guest with balloon occupied 2G memory 5039 let mut child = GuestCommand::new(&guest) 5040 .args(&["--api-socket", &api_socket]) 5041 .args(&["--cpus", "boot=1"]) 5042 .args(&["--memory", "size=4G"]) 5043 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5044 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5045 .args(&["--balloon", "size=2G,deflate_on_oom=on"]) 5046 .default_disks() 5047 .default_net() 5048 .capture_output() 5049 .spawn() 5050 .unwrap(); 5051 5052 let r = std::panic::catch_unwind(|| { 5053 guest.wait_vm_boot(None).unwrap(); 5054 5055 // Wait for balloon memory's initialization and check its size. 5056 // The virtio-balloon driver might take a few seconds to report the 5057 // balloon effective size back to the VMM. 5058 thread::sleep(std::time::Duration::new(20, 0)); 5059 5060 let orig_balloon = balloon_size(&api_socket); 5061 println!("The original balloon memory size is {} bytes", orig_balloon); 5062 assert!(orig_balloon == 2147483648); 5063 5064 // Two steps to verify if the 'deflate_on_oom' parameter works. 5065 // 1st: run a command in guest to eat up memory heavily, which 5066 // will consume much more memory than $(total_mem - balloon_size) 5067 // to trigger an oom. 5068 guest 5069 .ssh_command("stress --vm 25 --vm-keep --vm-bytes 1G --timeout 20") 5070 .unwrap(); 5071 5072 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5073 let deflated_balloon = balloon_size(&api_socket); 5074 println!( 5075 "After deflating, balloon memory size is {} bytes", 5076 deflated_balloon 5077 ); 5078 // Verify the balloon size deflated 5079 assert!(deflated_balloon < 2147483648); 5080 }); 5081 5082 let _ = child.kill(); 5083 let output = child.wait_with_output().unwrap(); 5084 5085 handle_child_output(r, &output); 5086 } 5087 5088 #[test] 5089 fn test_virtio_balloon_free_page_reporting() { 5090 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5091 let guest = Guest::new(Box::new(focal)); 5092 5093 //Let's start a 4G guest with balloon occupied 2G memory 5094 let mut child = GuestCommand::new(&guest) 5095 .args(&["--cpus", "boot=1"]) 5096 .args(&["--memory", "size=4G"]) 5097 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5098 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5099 .args(&["--balloon", "size=0,free_page_reporting=on"]) 5100 .default_disks() 5101 .default_net() 5102 .capture_output() 5103 .spawn() 5104 .unwrap(); 5105 5106 let pid = child.id(); 5107 let r = std::panic::catch_unwind(|| { 5108 guest.wait_vm_boot(None).unwrap(); 5109 5110 // Check the initial RSS is less than 1GiB 5111 let rss = process_rss_kib(pid); 5112 println!("RSS {} < 1048576", rss); 5113 assert!(rss < 1048576); 5114 5115 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5116 // seconds 5117 let guest_ip = guest.network.guest_ip.clone(); 5118 thread::spawn(move || { 5119 ssh_command_ip( 5120 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5121 &guest_ip, 5122 DEFAULT_SSH_RETRIES, 5123 DEFAULT_SSH_TIMEOUT, 5124 ) 5125 .unwrap(); 5126 }); 5127 5128 // Wait for 50 seconds to make sure the stress command is consuming 5129 // the expected amount of memory. 5130 thread::sleep(std::time::Duration::new(50, 0)); 5131 let rss = process_rss_kib(pid); 5132 println!("RSS {} >= 2097152", rss); 5133 assert!(rss >= 2097152); 5134 5135 // Wait for an extra minute to make sure the stress command has 5136 // completed and that the guest reported the free pages to the VMM 5137 // through the virtio-balloon device. We expect the RSS to be under 5138 // 2GiB. 5139 thread::sleep(std::time::Duration::new(60, 0)); 5140 let rss = process_rss_kib(pid); 5141 println!("RSS {} < 2097152", rss); 5142 assert!(rss < 2097152); 5143 }); 5144 5145 let _ = child.kill(); 5146 let output = child.wait_with_output().unwrap(); 5147 5148 handle_child_output(r, &output); 5149 } 5150 5151 #[test] 5152 fn test_pmem_hotplug() { 5153 _test_pmem_hotplug(None) 5154 } 5155 5156 #[test] 5157 fn test_pmem_multi_segment_hotplug() { 5158 _test_pmem_hotplug(Some(15)) 5159 } 5160 5161 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5162 #[cfg(target_arch = "aarch64")] 5163 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5164 #[cfg(target_arch = "x86_64")] 5165 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5166 let focal = UbuntuDiskConfig::new(focal_image); 5167 let guest = Guest::new(Box::new(focal)); 5168 5169 #[cfg(target_arch = "x86_64")] 5170 let kernel_path = direct_kernel_boot_path(); 5171 #[cfg(target_arch = "aarch64")] 5172 let kernel_path = edk2_path(); 5173 5174 let api_socket = temp_api_path(&guest.tmp_dir); 5175 5176 let mut cmd = GuestCommand::new(&guest); 5177 5178 cmd.args(&["--api-socket", &api_socket]) 5179 .args(&["--cpus", "boot=1"]) 5180 .args(&["--memory", "size=512M"]) 5181 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5182 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5183 .default_disks() 5184 .default_net() 5185 .capture_output(); 5186 5187 if pci_segment.is_some() { 5188 cmd.args(&["--platform", "num_pci_segments=16"]); 5189 } 5190 5191 let mut child = cmd.spawn().unwrap(); 5192 5193 let r = std::panic::catch_unwind(|| { 5194 guest.wait_vm_boot(None).unwrap(); 5195 5196 // Check /dev/pmem0 is not there 5197 assert_eq!( 5198 guest 5199 .ssh_command("lsblk | grep -c pmem0 || true") 5200 .unwrap() 5201 .trim() 5202 .parse::<u32>() 5203 .unwrap_or(1), 5204 0 5205 ); 5206 5207 let pmem_temp_file = TempFile::new().unwrap(); 5208 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5209 let (cmd_success, cmd_output) = remote_command_w_output( 5210 &api_socket, 5211 "add-pmem", 5212 Some(&format!( 5213 "file={},id=test0{}", 5214 pmem_temp_file.as_path().to_str().unwrap(), 5215 if let Some(pci_segment) = pci_segment { 5216 format!(",pci_segment={}", pci_segment) 5217 } else { 5218 "".to_owned() 5219 } 5220 )), 5221 ); 5222 assert!(cmd_success); 5223 if let Some(pci_segment) = pci_segment { 5224 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5225 "{{\"id\":\"test0\",\"bdf\":\"{:04x}:00:01.0\"}}", 5226 pci_segment 5227 ))); 5228 } else { 5229 assert!(String::from_utf8_lossy(&cmd_output) 5230 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5231 } 5232 5233 // Check that /dev/pmem0 exists and the block size is 128M 5234 assert_eq!( 5235 guest 5236 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5237 .unwrap() 5238 .trim() 5239 .parse::<u32>() 5240 .unwrap_or_default(), 5241 1 5242 ); 5243 5244 guest.reboot_linux(0, None); 5245 5246 // Check still there after reboot 5247 assert_eq!( 5248 guest 5249 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5250 .unwrap() 5251 .trim() 5252 .parse::<u32>() 5253 .unwrap_or_default(), 5254 1 5255 ); 5256 5257 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5258 5259 thread::sleep(std::time::Duration::new(20, 0)); 5260 5261 // Check device has gone away 5262 assert_eq!( 5263 guest 5264 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5265 .unwrap() 5266 .trim() 5267 .parse::<u32>() 5268 .unwrap_or(1), 5269 0 5270 ); 5271 5272 guest.reboot_linux(1, None); 5273 5274 // Check still absent after reboot 5275 assert_eq!( 5276 guest 5277 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5278 .unwrap() 5279 .trim() 5280 .parse::<u32>() 5281 .unwrap_or(1), 5282 0 5283 ); 5284 }); 5285 5286 let _ = child.kill(); 5287 let output = child.wait_with_output().unwrap(); 5288 5289 handle_child_output(r, &output); 5290 } 5291 5292 #[test] 5293 fn test_net_hotplug() { 5294 _test_net_hotplug(None) 5295 } 5296 5297 #[test] 5298 fn test_net_multi_segment_hotplug() { 5299 _test_net_hotplug(Some(15)) 5300 } 5301 5302 fn _test_net_hotplug(pci_segment: Option<u16>) { 5303 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5304 let guest = Guest::new(Box::new(focal)); 5305 5306 #[cfg(target_arch = "x86_64")] 5307 let kernel_path = direct_kernel_boot_path(); 5308 #[cfg(target_arch = "aarch64")] 5309 let kernel_path = edk2_path(); 5310 5311 let api_socket = temp_api_path(&guest.tmp_dir); 5312 5313 // Boot without network 5314 let mut cmd = GuestCommand::new(&guest); 5315 5316 cmd.args(&["--api-socket", &api_socket]) 5317 .args(&["--cpus", "boot=1"]) 5318 .args(&["--memory", "size=512M"]) 5319 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5320 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5321 .default_disks() 5322 .capture_output(); 5323 5324 if pci_segment.is_some() { 5325 cmd.args(&["--platform", "num_pci_segments=16"]); 5326 } 5327 5328 let mut child = cmd.spawn().unwrap(); 5329 5330 thread::sleep(std::time::Duration::new(20, 0)); 5331 5332 let r = std::panic::catch_unwind(|| { 5333 // Add network 5334 let (cmd_success, cmd_output) = remote_command_w_output( 5335 &api_socket, 5336 "add-net", 5337 Some( 5338 format!( 5339 "{}{},id=test0", 5340 guest.default_net_string(), 5341 if let Some(pci_segment) = pci_segment { 5342 format!(",pci_segment={}", pci_segment) 5343 } else { 5344 "".to_owned() 5345 } 5346 ) 5347 .as_str(), 5348 ), 5349 ); 5350 assert!(cmd_success); 5351 5352 if let Some(pci_segment) = pci_segment { 5353 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5354 "{{\"id\":\"test0\",\"bdf\":\"{:04x}:00:01.0\"}}", 5355 pci_segment 5356 ))); 5357 } else { 5358 assert!(String::from_utf8_lossy(&cmd_output) 5359 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5360 } 5361 5362 thread::sleep(std::time::Duration::new(5, 0)); 5363 5364 // 1 network interfaces + default localhost ==> 2 interfaces 5365 assert_eq!( 5366 guest 5367 .ssh_command("ip -o link | wc -l") 5368 .unwrap() 5369 .trim() 5370 .parse::<u32>() 5371 .unwrap_or_default(), 5372 2 5373 ); 5374 5375 // Remove network 5376 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5377 thread::sleep(std::time::Duration::new(5, 0)); 5378 5379 let (cmd_success, cmd_output) = remote_command_w_output( 5380 &api_socket, 5381 "add-net", 5382 Some( 5383 format!( 5384 "{}{},id=test1", 5385 guest.default_net_string(), 5386 if let Some(pci_segment) = pci_segment { 5387 format!(",pci_segment={}", pci_segment) 5388 } else { 5389 "".to_owned() 5390 } 5391 ) 5392 .as_str(), 5393 ), 5394 ); 5395 assert!(cmd_success); 5396 5397 if let Some(pci_segment) = pci_segment { 5398 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5399 "{{\"id\":\"test1\",\"bdf\":\"{:04x}:00:01.0\"}}", 5400 pci_segment 5401 ))); 5402 } else { 5403 assert!(String::from_utf8_lossy(&cmd_output) 5404 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5405 } 5406 5407 thread::sleep(std::time::Duration::new(5, 0)); 5408 5409 // 1 network interfaces + default localhost ==> 2 interfaces 5410 assert_eq!( 5411 guest 5412 .ssh_command("ip -o link | wc -l") 5413 .unwrap() 5414 .trim() 5415 .parse::<u32>() 5416 .unwrap_or_default(), 5417 2 5418 ); 5419 5420 guest.reboot_linux(0, None); 5421 5422 // Check still there after reboot 5423 // 1 network interfaces + default localhost ==> 2 interfaces 5424 assert_eq!( 5425 guest 5426 .ssh_command("ip -o link | wc -l") 5427 .unwrap() 5428 .trim() 5429 .parse::<u32>() 5430 .unwrap_or_default(), 5431 2 5432 ); 5433 }); 5434 5435 let _ = child.kill(); 5436 let output = child.wait_with_output().unwrap(); 5437 5438 handle_child_output(r, &output); 5439 } 5440 5441 #[test] 5442 fn test_initramfs() { 5443 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5444 let guest = Guest::new(Box::new(focal)); 5445 let mut workload_path = dirs::home_dir().unwrap(); 5446 workload_path.push("workloads"); 5447 5448 #[cfg(target_arch = "x86_64")] 5449 let mut kernels = vec![direct_kernel_boot_path()]; 5450 #[cfg(target_arch = "aarch64")] 5451 let kernels = vec![direct_kernel_boot_path()]; 5452 5453 #[cfg(target_arch = "x86_64")] 5454 { 5455 let mut pvh_kernel_path = workload_path.clone(); 5456 pvh_kernel_path.push("vmlinux"); 5457 kernels.push(pvh_kernel_path); 5458 } 5459 5460 let mut initramfs_path = workload_path; 5461 initramfs_path.push("alpine_initramfs.img"); 5462 5463 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5464 let cmdline = format!("console=hvc0 quiet TEST_STRING={}", test_string); 5465 5466 kernels.iter().for_each(|k_path| { 5467 let mut child = GuestCommand::new(&guest) 5468 .args(&["--kernel", k_path.to_str().unwrap()]) 5469 .args(&["--initramfs", initramfs_path.to_str().unwrap()]) 5470 .args(&["--cmdline", &cmdline]) 5471 .capture_output() 5472 .spawn() 5473 .unwrap(); 5474 5475 thread::sleep(std::time::Duration::new(20, 0)); 5476 5477 let _ = child.kill(); 5478 let output = child.wait_with_output().unwrap(); 5479 5480 let r = std::panic::catch_unwind(|| { 5481 let s = String::from_utf8_lossy(&output.stdout); 5482 5483 assert_ne!(s.lines().position(|line| line == test_string), None); 5484 }); 5485 5486 handle_child_output(r, &output); 5487 }); 5488 } 5489 5490 // One thing to note about this test. The virtio-net device is heavily used 5491 // through each ssh command. There's no need to perform a dedicated test to 5492 // verify the migration went well for virtio-net. 5493 #[test] 5494 fn test_snapshot_restore() { 5495 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5496 let guest = Guest::new(Box::new(focal)); 5497 let kernel_path = direct_kernel_boot_path(); 5498 5499 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 5500 5501 let net_id = "net123"; 5502 let net_params = format!( 5503 "id={},tap=,mac={},ip={},mask=255.255.255.0", 5504 net_id, guest.network.guest_mac, guest.network.host_ip 5505 ); 5506 5507 let cloudinit_params = format!( 5508 "path={},iommu=on", 5509 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5510 ); 5511 5512 let socket = temp_vsock_path(&guest.tmp_dir); 5513 let event_path = temp_event_monitor_path(&guest.tmp_dir); 5514 5515 let mut child = GuestCommand::new(&guest) 5516 .args(&["--api-socket", &api_socket_source]) 5517 .args(&["--event-monitor", format!("path={}", event_path).as_str()]) 5518 .args(&["--cpus", "boot=4"]) 5519 .args(&[ 5520 "--memory", 5521 "size=4G,hotplug_method=virtio-mem,hotplug_size=32G", 5522 ]) 5523 .args(&["--balloon", "size=0"]) 5524 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5525 .args(&[ 5526 "--disk", 5527 format!( 5528 "path={}", 5529 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5530 ) 5531 .as_str(), 5532 cloudinit_params.as_str(), 5533 ]) 5534 .args(&["--net", net_params.as_str()]) 5535 .args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]) 5536 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5537 .capture_output() 5538 .spawn() 5539 .unwrap(); 5540 5541 let console_text = String::from("On a branch floating down river a cricket, singing."); 5542 // Create the snapshot directory 5543 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 5544 5545 let r = std::panic::catch_unwind(|| { 5546 guest.wait_vm_boot(None).unwrap(); 5547 5548 // Check the number of vCPUs 5549 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5550 // Check the guest RAM 5551 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5552 // Increase guest RAM with virtio-mem 5553 resize_command( 5554 &api_socket_source, 5555 None, 5556 Some(6 << 30), 5557 None, 5558 Some(&event_path), 5559 ); 5560 thread::sleep(std::time::Duration::new(5, 0)); 5561 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5562 // Use balloon to remove RAM from the VM 5563 resize_command( 5564 &api_socket_source, 5565 None, 5566 None, 5567 Some(1 << 30), 5568 Some(&event_path), 5569 ); 5570 thread::sleep(std::time::Duration::new(5, 0)); 5571 let total_memory = guest.get_total_memory().unwrap_or_default(); 5572 assert!(total_memory > 4_800_000); 5573 assert!(total_memory < 5_760_000); 5574 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 5575 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5576 5577 // x86_64: We check that removing and adding back the virtio-net device 5578 // does not break the snapshot/restore support for virtio-pci. 5579 // This is an important thing to test as the hotplug will 5580 // trigger a PCI BAR reprogramming, which is a good way of 5581 // checking if the stored resources are correctly restored. 5582 // Unplug the virtio-net device 5583 // AArch64: Device hotplug is currently not supported, skipping here. 5584 #[cfg(target_arch = "x86_64")] 5585 { 5586 assert!(remote_command( 5587 &api_socket_source, 5588 "remove-device", 5589 Some(net_id), 5590 )); 5591 thread::sleep(std::time::Duration::new(10, 0)); 5592 let latest_events = [&MetaEvent { 5593 event: "device-removed".to_string(), 5594 device_id: Some(net_id.to_string()), 5595 }]; 5596 assert!(check_latest_events_exact(&latest_events, &event_path)); 5597 5598 // Plug the virtio-net device again 5599 assert!(remote_command( 5600 &api_socket_source, 5601 "add-net", 5602 Some(net_params.as_str()), 5603 )); 5604 thread::sleep(std::time::Duration::new(10, 0)); 5605 } 5606 5607 // Pause the VM 5608 assert!(remote_command(&api_socket_source, "pause", None)); 5609 let latest_events = [ 5610 &MetaEvent { 5611 event: "pausing".to_string(), 5612 device_id: None, 5613 }, 5614 &MetaEvent { 5615 event: "paused".to_string(), 5616 device_id: None, 5617 }, 5618 ]; 5619 assert!(check_latest_events_exact(&latest_events, &event_path)); 5620 5621 // Take a snapshot from the VM 5622 assert!(remote_command( 5623 &api_socket_source, 5624 "snapshot", 5625 Some(format!("file://{}", snapshot_dir).as_str()), 5626 )); 5627 5628 // Wait to make sure the snapshot is completed 5629 thread::sleep(std::time::Duration::new(10, 0)); 5630 5631 let latest_events = [ 5632 &MetaEvent { 5633 event: "snapshotting".to_string(), 5634 device_id: None, 5635 }, 5636 &MetaEvent { 5637 event: "snapshotted".to_string(), 5638 device_id: None, 5639 }, 5640 ]; 5641 assert!(check_latest_events_exact(&latest_events, &event_path)); 5642 }); 5643 5644 // Shutdown the source VM and check console output 5645 let _ = child.kill(); 5646 let output = child.wait_with_output().unwrap(); 5647 handle_child_output(r, &output); 5648 5649 let r = std::panic::catch_unwind(|| { 5650 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5651 }); 5652 5653 handle_child_output(r, &output); 5654 5655 // Remove the vsock socket file. 5656 Command::new("rm") 5657 .arg("-f") 5658 .arg(socket.as_str()) 5659 .output() 5660 .unwrap(); 5661 5662 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 5663 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 5664 5665 // Restore the VM from the snapshot 5666 let mut child = GuestCommand::new(&guest) 5667 .args(&["--api-socket", &api_socket_restored]) 5668 .args(&[ 5669 "--event-monitor", 5670 format!("path={}", event_path_restored).as_str(), 5671 ]) 5672 .args(&[ 5673 "--restore", 5674 format!("source_url=file://{}", snapshot_dir).as_str(), 5675 ]) 5676 .capture_output() 5677 .spawn() 5678 .unwrap(); 5679 5680 // Wait for the VM to be restored 5681 thread::sleep(std::time::Duration::new(10, 0)); 5682 let expected_events = [ 5683 &MetaEvent { 5684 event: "starting".to_string(), 5685 device_id: None, 5686 }, 5687 &MetaEvent { 5688 event: "restoring".to_string(), 5689 device_id: None, 5690 }, 5691 ]; 5692 assert!(check_sequential_events_exact( 5693 &expected_events, 5694 &event_path_restored 5695 )); 5696 let latest_events = [&MetaEvent { 5697 event: "restored".to_string(), 5698 device_id: None, 5699 }]; 5700 assert!(check_latest_events_exact( 5701 &latest_events, 5702 &event_path_restored 5703 )); 5704 5705 let r = std::panic::catch_unwind(|| { 5706 // Resume the VM 5707 assert!(remote_command(&api_socket_restored, "resume", None)); 5708 let latest_events = [ 5709 &MetaEvent { 5710 event: "resuming".to_string(), 5711 device_id: None, 5712 }, 5713 &MetaEvent { 5714 event: "resumed".to_string(), 5715 device_id: None, 5716 }, 5717 ]; 5718 assert!(check_latest_events_exact( 5719 &latest_events, 5720 &event_path_restored 5721 )); 5722 5723 // Perform same checks to validate VM has been properly restored 5724 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5725 let total_memory = guest.get_total_memory().unwrap_or_default(); 5726 assert!(total_memory > 4_800_000); 5727 assert!(total_memory < 5_760_000); 5728 // Deflate balloon to restore entire RAM to the VM 5729 resize_command(&api_socket_restored, None, None, Some(0), None); 5730 thread::sleep(std::time::Duration::new(5, 0)); 5731 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5732 // Decrease guest RAM with virtio-mem 5733 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 5734 thread::sleep(std::time::Duration::new(5, 0)); 5735 let total_memory = guest.get_total_memory().unwrap_or_default(); 5736 assert!(total_memory > 4_800_000); 5737 assert!(total_memory < 5_760_000); 5738 5739 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5740 }); 5741 // Shutdown the target VM and check console output 5742 let _ = child.kill(); 5743 let output = child.wait_with_output().unwrap(); 5744 handle_child_output(r, &output); 5745 5746 let r = std::panic::catch_unwind(|| { 5747 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5748 }); 5749 5750 handle_child_output(r, &output); 5751 } 5752 5753 #[test] 5754 fn test_counters() { 5755 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5756 let guest = Guest::new(Box::new(focal)); 5757 let api_socket = temp_api_path(&guest.tmp_dir); 5758 5759 let mut cmd = GuestCommand::new(&guest); 5760 cmd.args(&["--cpus", "boot=1"]) 5761 .args(&["--memory", "size=512M"]) 5762 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5763 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5764 .default_disks() 5765 .args(&["--net", guest.default_net_string().as_str()]) 5766 .args(&["--api-socket", &api_socket]) 5767 .capture_output(); 5768 5769 let mut child = cmd.spawn().unwrap(); 5770 5771 let r = std::panic::catch_unwind(|| { 5772 guest.wait_vm_boot(None).unwrap(); 5773 5774 let orig_counters = get_counters(&api_socket); 5775 guest 5776 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 5777 .unwrap(); 5778 5779 let new_counters = get_counters(&api_socket); 5780 5781 // Check that all the counters have increased 5782 assert!(new_counters > orig_counters); 5783 }); 5784 5785 let _ = child.kill(); 5786 let output = child.wait_with_output().unwrap(); 5787 5788 handle_child_output(r, &output); 5789 } 5790 5791 #[test] 5792 #[cfg(feature = "guest_debug")] 5793 fn test_coredump() { 5794 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5795 let guest = Guest::new(Box::new(focal)); 5796 let api_socket = temp_api_path(&guest.tmp_dir); 5797 5798 let mut cmd = GuestCommand::new(&guest); 5799 cmd.args(&["--cpus", "boot=4"]) 5800 .args(&["--memory", "size=4G"]) 5801 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 5802 .default_disks() 5803 .args(&["--net", guest.default_net_string().as_str()]) 5804 .args(&["--api-socket", &api_socket]) 5805 .capture_output(); 5806 5807 let mut child = cmd.spawn().unwrap(); 5808 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 5809 5810 let r = std::panic::catch_unwind(|| { 5811 guest.wait_vm_boot(None).unwrap(); 5812 5813 assert!(remote_command(&api_socket, "pause", None)); 5814 5815 assert!(remote_command( 5816 &api_socket, 5817 "coredump", 5818 Some(format!("file://{}", vmcore_file).as_str()), 5819 )); 5820 5821 // the num of CORE notes should equals to vcpu 5822 let readelf_core_num_cmd = format!( 5823 "readelf --all {} |grep CORE |grep -v Type |wc -l", 5824 vmcore_file 5825 ); 5826 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 5827 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 5828 5829 // the num of QEMU notes should equals to vcpu 5830 let readelf_vmm_num_cmd = format!("readelf --all {} |grep QEMU |wc -l", vmcore_file); 5831 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 5832 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 5833 }); 5834 5835 let _ = child.kill(); 5836 let output = child.wait_with_output().unwrap(); 5837 5838 handle_child_output(r, &output); 5839 } 5840 5841 #[test] 5842 fn test_watchdog() { 5843 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5844 let guest = Guest::new(Box::new(focal)); 5845 let api_socket = temp_api_path(&guest.tmp_dir); 5846 5847 let kernel_path = direct_kernel_boot_path(); 5848 5849 let mut cmd = GuestCommand::new(&guest); 5850 cmd.args(&["--cpus", "boot=1"]) 5851 .args(&["--memory", "size=512M"]) 5852 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5853 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5854 .default_disks() 5855 .args(&["--net", guest.default_net_string().as_str()]) 5856 .args(&["--watchdog"]) 5857 .args(&["--api-socket", &api_socket]) 5858 .capture_output(); 5859 5860 let mut child = cmd.spawn().unwrap(); 5861 5862 let r = std::panic::catch_unwind(|| { 5863 guest.wait_vm_boot(None).unwrap(); 5864 let mut current_reboot_count = 1; 5865 enable_guest_watchdog(&guest, &mut current_reboot_count); 5866 check_guest_watchdog_no_reboot(&guest, ¤t_reboot_count); 5867 check_guest_watchdog_one_reboot(&guest, &api_socket, &mut current_reboot_count); 5868 }); 5869 5870 let _ = child.kill(); 5871 let output = child.wait_with_output().unwrap(); 5872 5873 handle_child_output(r, &output); 5874 } 5875 5876 #[test] 5877 fn test_tap_from_fd() { 5878 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5879 let guest = Guest::new(Box::new(focal)); 5880 let kernel_path = direct_kernel_boot_path(); 5881 5882 // Create a TAP interface with multi-queue enabled 5883 let num_queue_pairs: usize = 2; 5884 5885 use std::str::FromStr; 5886 let taps = net_util::open_tap( 5887 Some("chtap0"), 5888 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 5889 None, 5890 &mut None, 5891 num_queue_pairs, 5892 Some(libc::O_RDWR | libc::O_NONBLOCK), 5893 ) 5894 .unwrap(); 5895 5896 let mut child = GuestCommand::new(&guest) 5897 .args(&["--cpus", &format!("boot={}", num_queue_pairs)]) 5898 .args(&["--memory", "size=512M"]) 5899 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5900 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5901 .default_disks() 5902 .args(&[ 5903 "--net", 5904 &format!( 5905 "fd=[{},{}],mac={},num_queues={}", 5906 taps[0].as_raw_fd(), 5907 taps[1].as_raw_fd(), 5908 guest.network.guest_mac, 5909 num_queue_pairs * 2 5910 ), 5911 ]) 5912 .capture_output() 5913 .spawn() 5914 .unwrap(); 5915 5916 let r = std::panic::catch_unwind(|| { 5917 guest.wait_vm_boot(None).unwrap(); 5918 5919 assert_eq!( 5920 guest 5921 .ssh_command("ip -o link | wc -l") 5922 .unwrap() 5923 .trim() 5924 .parse::<u32>() 5925 .unwrap_or_default(), 5926 2 5927 ); 5928 5929 guest.reboot_linux(0, None); 5930 5931 assert_eq!( 5932 guest 5933 .ssh_command("ip -o link | wc -l") 5934 .unwrap() 5935 .trim() 5936 .parse::<u32>() 5937 .unwrap_or_default(), 5938 2 5939 ); 5940 }); 5941 5942 let _ = child.kill(); 5943 let output = child.wait_with_output().unwrap(); 5944 5945 handle_child_output(r, &output); 5946 } 5947 5948 // By design, a guest VM won't be able to connect to the host 5949 // machine when using a macvtap network interface (while it can 5950 // communicate externally). As a workaround, this integration 5951 // test creates two macvtap interfaces in 'bridge' mode on the 5952 // same physical net interface, one for the guest and one for 5953 // the host. With additional setup on the IP address and the 5954 // routing table, it enables the communications between the 5955 // guest VM and the host machine. 5956 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 5957 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 5958 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5959 let guest = Guest::new(Box::new(focal)); 5960 let api_socket = temp_api_path(&guest.tmp_dir); 5961 5962 #[cfg(target_arch = "x86_64")] 5963 let kernel_path = direct_kernel_boot_path(); 5964 #[cfg(target_arch = "aarch64")] 5965 let kernel_path = edk2_path(); 5966 5967 let phy_net = "eth0"; 5968 5969 // Create a macvtap interface for the guest VM to use 5970 assert!(exec_host_command_status(&format!( 5971 "sudo ip link add link {} name {} type macvtap mod bridge", 5972 phy_net, guest_macvtap_name 5973 )) 5974 .success()); 5975 assert!(exec_host_command_status(&format!( 5976 "sudo ip link set {} address {} up", 5977 guest_macvtap_name, guest.network.guest_mac 5978 )) 5979 .success()); 5980 assert!( 5981 exec_host_command_status(&format!("sudo ip link show {}", guest_macvtap_name)) 5982 .success() 5983 ); 5984 5985 let tap_index = 5986 fs::read_to_string(&format!("/sys/class/net/{}/ifindex", guest_macvtap_name)).unwrap(); 5987 let tap_device = format!("/dev/tap{}", tap_index.trim()); 5988 5989 assert!( 5990 exec_host_command_status(&format!("sudo chown $UID.$UID {}", tap_device)).success() 5991 ); 5992 5993 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 5994 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 5995 assert!(tap_fd1 > 0); 5996 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 5997 assert!(tap_fd2 > 0); 5998 5999 // Create a macvtap on the same physical net interface for 6000 // the host machine to use 6001 assert!(exec_host_command_status(&format!( 6002 "sudo ip link add link {} name {} type macvtap mod bridge", 6003 phy_net, host_macvtap_name 6004 )) 6005 .success()); 6006 // Use default mask "255.255.255.0" 6007 assert!(exec_host_command_status(&format!( 6008 "sudo ip address add {}/24 dev {}", 6009 guest.network.host_ip, host_macvtap_name 6010 )) 6011 .success()); 6012 assert!(exec_host_command_status(&format!( 6013 "sudo ip link set dev {} up", 6014 host_macvtap_name 6015 )) 6016 .success()); 6017 6018 let mut guest_command = GuestCommand::new(&guest); 6019 guest_command 6020 .args(&["--cpus", "boot=2"]) 6021 .args(&["--memory", "size=512M"]) 6022 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6023 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6024 .default_disks() 6025 .args(&["--api-socket", &api_socket]); 6026 6027 let net_params = format!( 6028 "fd=[{},{}],mac={},num_queues=4", 6029 tap_fd1, tap_fd2, guest.network.guest_mac 6030 ); 6031 6032 if !hotplug { 6033 guest_command.args(&["--net", &net_params]); 6034 } 6035 6036 let mut child = guest_command.capture_output().spawn().unwrap(); 6037 6038 if hotplug { 6039 // Give some time to the VMM process to listen to the API 6040 // socket. This is the only requirement to avoid the following 6041 // call to ch-remote from failing. 6042 thread::sleep(std::time::Duration::new(10, 0)); 6043 // Hotplug the virtio-net device 6044 let (cmd_success, cmd_output) = 6045 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6046 assert!(cmd_success); 6047 #[cfg(target_arch = "x86_64")] 6048 assert!(String::from_utf8_lossy(&cmd_output) 6049 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6050 #[cfg(target_arch = "aarch64")] 6051 assert!(String::from_utf8_lossy(&cmd_output) 6052 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6053 } 6054 6055 // The functional connectivity provided by the virtio-net device 6056 // gets tested through wait_vm_boot() as it expects to receive a 6057 // HTTP request, and through the SSH command as well. 6058 let r = std::panic::catch_unwind(|| { 6059 guest.wait_vm_boot(None).unwrap(); 6060 6061 assert_eq!( 6062 guest 6063 .ssh_command("ip -o link | wc -l") 6064 .unwrap() 6065 .trim() 6066 .parse::<u32>() 6067 .unwrap_or_default(), 6068 2 6069 ); 6070 }); 6071 6072 let _ = child.kill(); 6073 6074 exec_host_command_status(&format!("sudo ip link del {}", guest_macvtap_name)); 6075 exec_host_command_status(&format!("sudo ip link del {}", host_macvtap_name)); 6076 6077 let output = child.wait_with_output().unwrap(); 6078 6079 handle_child_output(r, &output); 6080 } 6081 6082 #[test] 6083 fn test_macvtap() { 6084 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6085 } 6086 6087 #[test] 6088 fn test_macvtap_hotplug() { 6089 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6090 } 6091 6092 #[test] 6093 #[cfg(not(feature = "mshv"))] 6094 fn test_ovs_dpdk() { 6095 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6096 let guest1 = Guest::new(Box::new(focal1)); 6097 6098 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6099 let guest2 = Guest::new(Box::new(focal2)); 6100 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6101 6102 let (mut child1, mut child2) = 6103 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6104 6105 // Create the snapshot directory 6106 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6107 6108 let r = std::panic::catch_unwind(|| { 6109 // Remove one of the two ports from the OVS bridge 6110 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6111 6112 // Spawn a new netcat listener in the first VM 6113 let guest_ip = guest1.network.guest_ip.clone(); 6114 thread::spawn(move || { 6115 ssh_command_ip( 6116 "nc -l 12345", 6117 &guest_ip, 6118 DEFAULT_SSH_RETRIES, 6119 DEFAULT_SSH_TIMEOUT, 6120 ) 6121 .unwrap(); 6122 }); 6123 6124 // Wait for the server to be listening 6125 thread::sleep(std::time::Duration::new(5, 0)); 6126 6127 // Check the connection fails this time 6128 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6129 6130 // Add the OVS port back 6131 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6132 6133 // And finally check the connection is functional again 6134 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6135 6136 // Pause the VM 6137 assert!(remote_command(&api_socket_source, "pause", None)); 6138 6139 // Take a snapshot from the VM 6140 assert!(remote_command( 6141 &api_socket_source, 6142 "snapshot", 6143 Some(format!("file://{}", snapshot_dir).as_str()), 6144 )); 6145 6146 // Wait to make sure the snapshot is completed 6147 thread::sleep(std::time::Duration::new(10, 0)); 6148 }); 6149 6150 // Shutdown the source VM 6151 let _ = child2.kill(); 6152 let output = child2.wait_with_output().unwrap(); 6153 handle_child_output(r, &output); 6154 6155 // Remove the vhost-user socket file. 6156 Command::new("rm") 6157 .arg("-f") 6158 .arg("/tmp/dpdkvhostclient2") 6159 .output() 6160 .unwrap(); 6161 6162 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6163 // Restore the VM from the snapshot 6164 let mut child2 = GuestCommand::new(&guest2) 6165 .args(&["--api-socket", &api_socket_restored]) 6166 .args(&[ 6167 "--restore", 6168 format!("source_url=file://{}", snapshot_dir).as_str(), 6169 ]) 6170 .capture_output() 6171 .spawn() 6172 .unwrap(); 6173 6174 // Wait for the VM to be restored 6175 thread::sleep(std::time::Duration::new(10, 0)); 6176 6177 let r = std::panic::catch_unwind(|| { 6178 // Resume the VM 6179 assert!(remote_command(&api_socket_restored, "resume", None)); 6180 6181 // Spawn a new netcat listener in the first VM 6182 let guest_ip = guest1.network.guest_ip.clone(); 6183 thread::spawn(move || { 6184 ssh_command_ip( 6185 "nc -l 12345", 6186 &guest_ip, 6187 DEFAULT_SSH_RETRIES, 6188 DEFAULT_SSH_TIMEOUT, 6189 ) 6190 .unwrap(); 6191 }); 6192 6193 // Wait for the server to be listening 6194 thread::sleep(std::time::Duration::new(5, 0)); 6195 6196 // And check the connection is still functional after restore 6197 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6198 }); 6199 6200 cleanup_ovs_dpdk(); 6201 6202 let _ = child1.kill(); 6203 let _ = child2.kill(); 6204 6205 let output = child1.wait_with_output().unwrap(); 6206 child2.wait().unwrap(); 6207 6208 handle_child_output(r, &output); 6209 } 6210 6211 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6212 cleanup_spdk_nvme(); 6213 6214 assert!(exec_host_command_status(&format!( 6215 "mkdir -p {}", 6216 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6217 )) 6218 .success()); 6219 assert!(exec_host_command_status(&format!( 6220 "truncate {} -s 128M", 6221 nvme_dir.join("test-disk.raw").to_str().unwrap() 6222 )) 6223 .success()); 6224 assert!(exec_host_command_status(&format!( 6225 "mkfs.ext4 {}", 6226 nvme_dir.join("test-disk.raw").to_str().unwrap() 6227 )) 6228 .success()); 6229 6230 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6231 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6232 .args(&["-i", "0", "-m", "0x1"]) 6233 .spawn() 6234 .unwrap(); 6235 thread::sleep(std::time::Duration::new(2, 0)); 6236 6237 assert!(exec_host_command_status( 6238 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER" 6239 ) 6240 .success()); 6241 assert!(exec_host_command_status(&format!( 6242 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6243 nvme_dir.join("test-disk.raw").to_str().unwrap() 6244 )) 6245 .success()); 6246 assert!(exec_host_command_status( 6247 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6248 ) 6249 .success()); 6250 assert!(exec_host_command_status( 6251 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6252 ) 6253 .success()); 6254 assert!(exec_host_command_status(&format!( 6255 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6256 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6257 )) 6258 .success()); 6259 } 6260 6261 fn cleanup_spdk_nvme() { 6262 exec_host_command_status("pkill -f nvmf_tgt"); 6263 } 6264 6265 #[test] 6266 fn test_vfio_user() { 6267 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6268 let jammy = UbuntuDiskConfig::new(jammy_image); 6269 let guest = Guest::new(Box::new(jammy)); 6270 6271 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6272 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6273 6274 let api_socket = temp_api_path(&guest.tmp_dir); 6275 let mut child = GuestCommand::new(&guest) 6276 .args(&["--api-socket", &api_socket]) 6277 .args(&["--cpus", "boot=1"]) 6278 .args(&["--memory", "size=512M,shared=on,hugepages=on"]) 6279 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6280 .args(&["--serial", "tty", "--console", "off"]) 6281 .default_disks() 6282 .default_net() 6283 .capture_output() 6284 .spawn() 6285 .unwrap(); 6286 6287 let r = std::panic::catch_unwind(|| { 6288 guest.wait_vm_boot(None).unwrap(); 6289 6290 // Hotplug the SPDK-NVMe device to the VM 6291 let (cmd_success, cmd_output) = remote_command_w_output( 6292 &api_socket, 6293 "add-user-device", 6294 Some(&format!( 6295 "socket={},id=vfio_user0", 6296 spdk_nvme_dir 6297 .as_path() 6298 .join("nvme-vfio-user/cntrl") 6299 .to_str() 6300 .unwrap(), 6301 )), 6302 ); 6303 assert!(cmd_success); 6304 assert!(String::from_utf8_lossy(&cmd_output) 6305 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6306 6307 thread::sleep(std::time::Duration::new(10, 0)); 6308 6309 // Check both if /dev/nvme exists and if the block size is 128M. 6310 assert_eq!( 6311 guest 6312 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6313 .unwrap() 6314 .trim() 6315 .parse::<u32>() 6316 .unwrap_or_default(), 6317 1 6318 ); 6319 6320 // Check changes persist after reboot 6321 assert_eq!( 6322 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6323 "" 6324 ); 6325 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6326 guest 6327 .ssh_command("echo test123 | sudo tee /mnt/test") 6328 .unwrap(); 6329 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6330 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6331 6332 guest.reboot_linux(0, None); 6333 assert_eq!( 6334 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6335 "" 6336 ); 6337 assert_eq!( 6338 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6339 "test123" 6340 ); 6341 }); 6342 6343 cleanup_spdk_nvme(); 6344 6345 let _ = child.kill(); 6346 let output = child.wait_with_output().unwrap(); 6347 6348 handle_child_output(r, &output); 6349 } 6350 6351 #[test] 6352 #[cfg(target_arch = "x86_64")] 6353 fn test_vdpa_block() { 6354 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6355 if !exec_host_command_status("lsmod | grep vdpa_sim_blk").success() { 6356 return; 6357 } 6358 6359 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6360 let guest = Guest::new(Box::new(focal)); 6361 let api_socket = temp_api_path(&guest.tmp_dir); 6362 6363 let kernel_path = direct_kernel_boot_path(); 6364 6365 let mut child = GuestCommand::new(&guest) 6366 .args(&["--cpus", "boot=2"]) 6367 .args(&["--memory", "size=512M,hugepages=on"]) 6368 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6369 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6370 .default_disks() 6371 .default_net() 6372 .args(&["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6373 .args(&["--platform", "num_pci_segments=2,iommu_segments=1"]) 6374 .args(&["--api-socket", &api_socket]) 6375 .capture_output() 6376 .spawn() 6377 .unwrap(); 6378 6379 let r = std::panic::catch_unwind(|| { 6380 guest.wait_vm_boot(None).unwrap(); 6381 6382 // Check both if /dev/vdc exists and if the block size is 128M. 6383 assert_eq!( 6384 guest 6385 .ssh_command("lsblk | grep vdc | grep -c 128M") 6386 .unwrap() 6387 .trim() 6388 .parse::<u32>() 6389 .unwrap_or_default(), 6390 1 6391 ); 6392 6393 // Check the content of the block device after we wrote to it. 6394 // The vpda-sim-blk should let us read what we previously wrote. 6395 guest 6396 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6397 .unwrap(); 6398 assert_eq!( 6399 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6400 "foobar" 6401 ); 6402 6403 // Hotplug an extra vDPA block device behind the vIOMMU 6404 // Add a new vDPA device to the VM 6405 let (cmd_success, cmd_output) = remote_command_w_output( 6406 &api_socket, 6407 "add-vdpa", 6408 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6409 ); 6410 assert!(cmd_success); 6411 assert!(String::from_utf8_lossy(&cmd_output) 6412 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6413 6414 thread::sleep(std::time::Duration::new(10, 0)); 6415 6416 // Check IOMMU setup 6417 assert!(guest 6418 .does_device_vendor_pair_match("0x1057", "0x1af4") 6419 .unwrap_or_default()); 6420 assert_eq!( 6421 guest 6422 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6423 .unwrap() 6424 .trim(), 6425 "0001:00:01.0" 6426 ); 6427 6428 // Check both if /dev/vdd exists and if the block size is 128M. 6429 assert_eq!( 6430 guest 6431 .ssh_command("lsblk | grep vdd | grep -c 128M") 6432 .unwrap() 6433 .trim() 6434 .parse::<u32>() 6435 .unwrap_or_default(), 6436 1 6437 ); 6438 6439 // Write some content to the block device we've just plugged. 6440 guest 6441 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6442 .unwrap(); 6443 6444 // Check we can read the content back. 6445 assert_eq!( 6446 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6447 "foobar" 6448 ); 6449 6450 // Unplug the device 6451 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6452 assert!(cmd_success); 6453 thread::sleep(std::time::Duration::new(10, 0)); 6454 6455 // Check /dev/vdd doesn't exist anymore 6456 assert_eq!( 6457 guest 6458 .ssh_command("lsblk | grep -c vdd || true") 6459 .unwrap() 6460 .trim() 6461 .parse::<u32>() 6462 .unwrap_or(1), 6463 0 6464 ); 6465 }); 6466 6467 let _ = child.kill(); 6468 let output = child.wait_with_output().unwrap(); 6469 6470 handle_child_output(r, &output); 6471 } 6472 6473 #[test] 6474 #[cfg(target_arch = "x86_64")] 6475 fn test_vdpa_net() { 6476 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6477 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6478 return; 6479 } 6480 6481 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6482 let guest = Guest::new(Box::new(focal)); 6483 6484 let kernel_path = direct_kernel_boot_path(); 6485 6486 let mut child = GuestCommand::new(&guest) 6487 .args(&["--cpus", "boot=2"]) 6488 .args(&["--memory", "size=512M,hugepages=on"]) 6489 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6490 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6491 .default_disks() 6492 .default_net() 6493 .args(&["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 6494 .capture_output() 6495 .spawn() 6496 .unwrap(); 6497 6498 let r = std::panic::catch_unwind(|| { 6499 guest.wait_vm_boot(None).unwrap(); 6500 6501 // Check we can find network interface related to vDPA device 6502 assert_eq!( 6503 guest 6504 .ssh_command("ip -o link | grep -c ens6") 6505 .unwrap() 6506 .trim() 6507 .parse::<u32>() 6508 .unwrap_or(0), 6509 1 6510 ); 6511 6512 guest 6513 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 6514 .unwrap(); 6515 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 6516 6517 // Check there is no packet yet on both TX/RX of the network interface 6518 assert_eq!( 6519 guest 6520 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 6521 .unwrap() 6522 .trim() 6523 .parse::<u32>() 6524 .unwrap_or(0), 6525 2 6526 ); 6527 6528 // Send 6 packets with ping command 6529 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 6530 6531 // Check we can find 6 packets on both TX/RX of the network interface 6532 assert_eq!( 6533 guest 6534 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 6535 .unwrap() 6536 .trim() 6537 .parse::<u32>() 6538 .unwrap_or(0), 6539 2 6540 ); 6541 6542 // No need to check for hotplug as we already tested it through 6543 // test_vdpa_block() 6544 }); 6545 6546 let _ = child.kill(); 6547 let output = child.wait_with_output().unwrap(); 6548 6549 handle_child_output(r, &output); 6550 } 6551 } 6552 6553 mod sequential { 6554 use crate::*; 6555 6556 #[test] 6557 fn test_memory_mergeable_on() { 6558 test_memory_mergeable(true) 6559 } 6560 } 6561 6562 #[cfg(target_arch = "x86_64")] 6563 mod windows { 6564 use crate::*; 6565 use once_cell::sync::Lazy; 6566 6567 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 6568 6569 struct WindowsGuest { 6570 guest: Guest, 6571 auth: PasswordAuth, 6572 } 6573 6574 trait FsType { 6575 const FS_FAT: u8; 6576 const FS_NTFS: u8; 6577 } 6578 impl FsType for WindowsGuest { 6579 const FS_FAT: u8 = 0; 6580 const FS_NTFS: u8 = 1; 6581 } 6582 6583 impl WindowsGuest { 6584 fn new() -> Self { 6585 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 6586 let guest = Guest::new(Box::new(disk)); 6587 let auth = PasswordAuth { 6588 username: String::from("administrator"), 6589 password: String::from("Admin123"), 6590 }; 6591 6592 WindowsGuest { guest, auth } 6593 } 6594 6595 fn guest(&self) -> &Guest { 6596 &self.guest 6597 } 6598 6599 fn ssh_cmd(&self, cmd: &str) -> String { 6600 ssh_command_ip_with_auth( 6601 cmd, 6602 &self.auth, 6603 &self.guest.network.guest_ip, 6604 DEFAULT_SSH_RETRIES, 6605 DEFAULT_SSH_TIMEOUT, 6606 ) 6607 .unwrap() 6608 } 6609 6610 fn cpu_count(&self) -> u8 { 6611 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 6612 .trim() 6613 .parse::<u8>() 6614 .unwrap_or(0) 6615 } 6616 6617 fn ram_size(&self) -> usize { 6618 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 6619 .trim() 6620 .parse::<usize>() 6621 .unwrap_or(0) 6622 } 6623 6624 fn netdev_count(&self) -> u8 { 6625 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6626 .trim() 6627 .parse::<u8>() 6628 .unwrap_or(0) 6629 } 6630 6631 fn disk_count(&self) -> u8 { 6632 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6633 .trim() 6634 .parse::<u8>() 6635 .unwrap_or(0) 6636 } 6637 6638 fn reboot(&self) { 6639 let _ = self.ssh_cmd("shutdown /r /t 0"); 6640 } 6641 6642 fn shutdown(&self) { 6643 let _ = self.ssh_cmd("shutdown /s /t 0"); 6644 } 6645 6646 fn run_dnsmasq(&self) -> std::process::Child { 6647 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 6648 let dhcp_host = format!( 6649 "--dhcp-host={},{}", 6650 self.guest.network.guest_mac, self.guest.network.guest_ip 6651 ); 6652 let dhcp_range = format!( 6653 "--dhcp-range=eth,{},{}", 6654 self.guest.network.guest_ip, self.guest.network.guest_ip 6655 ); 6656 6657 Command::new("dnsmasq") 6658 .arg("--no-daemon") 6659 .arg("--log-queries") 6660 .arg(listen_address.as_str()) 6661 .arg("--except-interface=lo") 6662 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 6663 .arg("--conf-file=/dev/null") 6664 .arg(dhcp_host.as_str()) 6665 .arg(dhcp_range.as_str()) 6666 .spawn() 6667 .unwrap() 6668 } 6669 6670 // TODO Cleanup image file explicitly after test, if there's some space issues. 6671 fn disk_new(&self, fs: u8, sz: usize) -> String { 6672 let mut guard = NEXT_DISK_ID.lock().unwrap(); 6673 let id = *guard; 6674 *guard = id + 1; 6675 6676 let img = PathBuf::from(format!("/tmp/test-hotplug-{}.raw", id)); 6677 let _ = fs::remove_file(&img); 6678 6679 // Create an image file 6680 let out = Command::new("qemu-img") 6681 .args(&[ 6682 "create", 6683 "-f", 6684 "raw", 6685 img.to_str().unwrap(), 6686 format!("{}m", sz).as_str(), 6687 ]) 6688 .output() 6689 .expect("qemu-img command failed") 6690 .stdout; 6691 println!("{:?}", out); 6692 6693 // Associate image to a loop device 6694 let out = Command::new("losetup") 6695 .args(&["--show", "-f", img.to_str().unwrap()]) 6696 .output() 6697 .expect("failed to create loop device") 6698 .stdout; 6699 let _tmp = String::from_utf8_lossy(&out); 6700 let loop_dev = _tmp.trim(); 6701 println!("{:?}", out); 6702 6703 // Create a partition table 6704 // echo 'type=7' | sudo sfdisk "${LOOP}" 6705 let mut child = Command::new("sfdisk") 6706 .args(&[loop_dev]) 6707 .stdin(Stdio::piped()) 6708 .spawn() 6709 .unwrap(); 6710 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 6711 stdin 6712 .write_all("type=7".as_bytes()) 6713 .expect("failed to write stdin"); 6714 let out = child.wait_with_output().expect("sfdisk failed").stdout; 6715 println!("{:?}", out); 6716 6717 // Disengage the loop device 6718 let out = Command::new("losetup") 6719 .args(&["-d", loop_dev]) 6720 .output() 6721 .expect("loop device not found") 6722 .stdout; 6723 println!("{:?}", out); 6724 6725 // Re-associate loop device pointing to the partition only 6726 let out = Command::new("losetup") 6727 .args(&[ 6728 "--show", 6729 "--offset", 6730 (512 * 2048).to_string().as_str(), 6731 "-f", 6732 img.to_str().unwrap(), 6733 ]) 6734 .output() 6735 .expect("failed to create loop device") 6736 .stdout; 6737 let _tmp = String::from_utf8_lossy(&out); 6738 let loop_dev = _tmp.trim(); 6739 println!("{:?}", out); 6740 6741 // Create filesystem. 6742 let fs_cmd = match fs { 6743 WindowsGuest::FS_FAT => "mkfs.msdos", 6744 WindowsGuest::FS_NTFS => "mkfs.ntfs", 6745 _ => panic!("Unknown filesystem type '{}'", fs), 6746 }; 6747 let out = Command::new(fs_cmd) 6748 .args(&[&loop_dev]) 6749 .output() 6750 .unwrap_or_else(|_| panic!("{} failed", fs_cmd)) 6751 .stdout; 6752 println!("{:?}", out); 6753 6754 // Disengage the loop device 6755 let out = Command::new("losetup") 6756 .args(&["-d", loop_dev]) 6757 .output() 6758 .unwrap_or_else(|_| panic!("loop device '{}' not found", loop_dev)) 6759 .stdout; 6760 println!("{:?}", out); 6761 6762 img.to_str().unwrap().to_string() 6763 } 6764 6765 fn disks_set_rw(&self) { 6766 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 6767 } 6768 6769 fn disks_online(&self) { 6770 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 6771 } 6772 6773 fn disk_file_put(&self, fname: &str, data: &str) { 6774 let _ = self.ssh_cmd(&format!( 6775 "powershell -Command \"'{}' | Set-Content -Path {}\"", 6776 data, fname 6777 )); 6778 } 6779 6780 fn disk_file_read(&self, fname: &str) -> String { 6781 self.ssh_cmd(&format!( 6782 "powershell -Command \"Get-Content -Path {}\"", 6783 fname 6784 )) 6785 } 6786 6787 fn wait_for_boot(&self) -> bool { 6788 let cmd = "dir /b c:\\ | find \"Windows\""; 6789 let tmo_max = 180; 6790 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 6791 // interval must be small. 6792 let tmo_int = 2; 6793 let out = ssh_command_ip_with_auth( 6794 cmd, 6795 &self.auth, 6796 &self.guest.network.guest_ip, 6797 { 6798 let mut ret = 1; 6799 let mut tmo_acc = 0; 6800 loop { 6801 tmo_acc += tmo_int * ret; 6802 if tmo_acc >= tmo_max { 6803 break; 6804 } 6805 ret += 1; 6806 } 6807 ret 6808 }, 6809 tmo_int, 6810 ) 6811 .unwrap(); 6812 6813 if "Windows" == out.trim() { 6814 return true; 6815 } 6816 6817 false 6818 } 6819 } 6820 6821 fn vcpu_threads_count(pid: u32) -> u8 { 6822 // ps -T -p 12345 | grep vcpu | wc -l 6823 let out = Command::new("ps") 6824 .args(&["-T", "-p", format!("{}", pid).as_str()]) 6825 .output() 6826 .expect("ps command failed") 6827 .stdout; 6828 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 6829 } 6830 6831 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 6832 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 6833 let out = Command::new("ps") 6834 .args(&["-T", "-p", format!("{}", pid).as_str()]) 6835 .output() 6836 .expect("ps command failed") 6837 .stdout; 6838 let mut n = 0; 6839 String::from_utf8_lossy(&out) 6840 .split_whitespace() 6841 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 6842 n 6843 } 6844 6845 fn disk_ctrl_threads_count(pid: u32) -> u8 { 6846 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 6847 let out = Command::new("ps") 6848 .args(&["-T", "-p", format!("{}", pid).as_str()]) 6849 .output() 6850 .expect("ps command failed") 6851 .stdout; 6852 let mut n = 0; 6853 String::from_utf8_lossy(&out) 6854 .split_whitespace() 6855 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 6856 n 6857 } 6858 6859 #[test] 6860 fn test_windows_guest() { 6861 let windows_guest = WindowsGuest::new(); 6862 6863 let mut ovmf_path = dirs::home_dir().unwrap(); 6864 ovmf_path.push("workloads"); 6865 ovmf_path.push(OVMF_NAME); 6866 6867 let mut child = GuestCommand::new(windows_guest.guest()) 6868 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6869 .args(&["--memory", "size=4G"]) 6870 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6871 .args(&["--serial", "tty"]) 6872 .args(&["--console", "off"]) 6873 .default_disks() 6874 .default_net() 6875 .capture_output() 6876 .spawn() 6877 .unwrap(); 6878 6879 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 6880 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6881 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 6882 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6883 6884 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 6885 6886 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6887 6888 let r = std::panic::catch_unwind(|| { 6889 // Wait to make sure Windows boots up 6890 assert!(windows_guest.wait_for_boot()); 6891 6892 windows_guest.shutdown(); 6893 }); 6894 6895 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6896 let _ = child.kill(); 6897 let output = child.wait_with_output().unwrap(); 6898 6899 let _ = child_dnsmasq.kill(); 6900 let _ = child_dnsmasq.wait(); 6901 6902 handle_child_output(r, &output); 6903 } 6904 6905 #[test] 6906 fn test_windows_guest_multiple_queues() { 6907 let windows_guest = WindowsGuest::new(); 6908 6909 let mut ovmf_path = dirs::home_dir().unwrap(); 6910 ovmf_path.push("workloads"); 6911 ovmf_path.push(OVMF_NAME); 6912 6913 let mut child = GuestCommand::new(windows_guest.guest()) 6914 .args(&["--cpus", "boot=4,kvm_hyperv=on"]) 6915 .args(&["--memory", "size=4G"]) 6916 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6917 .args(&["--serial", "tty"]) 6918 .args(&["--console", "off"]) 6919 .args(&[ 6920 "--disk", 6921 format!( 6922 "path={},num_queues=4", 6923 windows_guest 6924 .guest() 6925 .disk_config 6926 .disk(DiskType::OperatingSystem) 6927 .unwrap() 6928 ) 6929 .as_str(), 6930 ]) 6931 .args(&[ 6932 "--net", 6933 format!( 6934 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 6935 windows_guest.guest().network.guest_mac, 6936 windows_guest.guest().network.host_ip 6937 ) 6938 .as_str(), 6939 ]) 6940 .capture_output() 6941 .spawn() 6942 .unwrap(); 6943 6944 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 6945 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6946 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 6947 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6948 6949 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 6950 6951 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6952 6953 let r = std::panic::catch_unwind(|| { 6954 // Wait to make sure Windows boots up 6955 assert!(windows_guest.wait_for_boot()); 6956 6957 windows_guest.shutdown(); 6958 }); 6959 6960 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6961 let _ = child.kill(); 6962 let output = child.wait_with_output().unwrap(); 6963 6964 let _ = child_dnsmasq.kill(); 6965 let _ = child_dnsmasq.wait(); 6966 6967 handle_child_output(r, &output); 6968 } 6969 6970 #[test] 6971 #[cfg(not(feature = "mshv"))] 6972 #[ignore = "See #4327"] 6973 fn test_windows_guest_snapshot_restore() { 6974 let windows_guest = WindowsGuest::new(); 6975 6976 let mut ovmf_path = dirs::home_dir().unwrap(); 6977 ovmf_path.push("workloads"); 6978 ovmf_path.push(OVMF_NAME); 6979 6980 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6981 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 6982 6983 let mut child = GuestCommand::new(windows_guest.guest()) 6984 .args(&["--api-socket", &api_socket_source]) 6985 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6986 .args(&["--memory", "size=4G"]) 6987 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6988 .args(&["--serial", "tty"]) 6989 .args(&["--console", "off"]) 6990 .default_disks() 6991 .default_net() 6992 .capture_output() 6993 .spawn() 6994 .unwrap(); 6995 6996 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 6997 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 6998 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 6999 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7000 7001 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7002 7003 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7004 7005 // Wait to make sure Windows boots up 7006 assert!(windows_guest.wait_for_boot()); 7007 7008 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 7009 7010 // Pause the VM 7011 assert!(remote_command(&api_socket_source, "pause", None)); 7012 7013 // Take a snapshot from the VM 7014 assert!(remote_command( 7015 &api_socket_source, 7016 "snapshot", 7017 Some(format!("file://{}", snapshot_dir).as_str()), 7018 )); 7019 7020 // Wait to make sure the snapshot is completed 7021 thread::sleep(std::time::Duration::new(30, 0)); 7022 7023 let _ = child.kill(); 7024 child.wait().unwrap(); 7025 7026 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 7027 7028 // Restore the VM from the snapshot 7029 let mut child = GuestCommand::new(windows_guest.guest()) 7030 .args(&["--api-socket", &api_socket_restored]) 7031 .args(&[ 7032 "--restore", 7033 format!("source_url=file://{}", snapshot_dir).as_str(), 7034 ]) 7035 .capture_output() 7036 .spawn() 7037 .unwrap(); 7038 7039 // Wait for the VM to be restored 7040 thread::sleep(std::time::Duration::new(20, 0)); 7041 7042 let r = std::panic::catch_unwind(|| { 7043 // Resume the VM 7044 assert!(remote_command(&api_socket_restored, "resume", None)); 7045 7046 windows_guest.shutdown(); 7047 }); 7048 7049 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7050 let _ = child.kill(); 7051 let output = child.wait_with_output().unwrap(); 7052 7053 let _ = child_dnsmasq.kill(); 7054 let _ = child_dnsmasq.wait(); 7055 7056 handle_child_output(r, &output); 7057 } 7058 7059 #[test] 7060 #[cfg(not(feature = "mshv"))] 7061 fn test_windows_guest_cpu_hotplug() { 7062 let windows_guest = WindowsGuest::new(); 7063 7064 let mut ovmf_path = dirs::home_dir().unwrap(); 7065 ovmf_path.push("workloads"); 7066 ovmf_path.push(OVMF_NAME); 7067 7068 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7069 let api_socket = temp_api_path(&tmp_dir); 7070 7071 let mut child = GuestCommand::new(windows_guest.guest()) 7072 .args(&["--api-socket", &api_socket]) 7073 .args(&["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 7074 .args(&["--memory", "size=4G"]) 7075 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7076 .args(&["--serial", "tty"]) 7077 .args(&["--console", "off"]) 7078 .default_disks() 7079 .default_net() 7080 .capture_output() 7081 .spawn() 7082 .unwrap(); 7083 7084 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7085 7086 let r = std::panic::catch_unwind(|| { 7087 // Wait to make sure Windows boots up 7088 assert!(windows_guest.wait_for_boot()); 7089 7090 let vcpu_num = 2; 7091 // Check the initial number of CPUs the guest sees 7092 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7093 // Check the initial number of vcpu threads in the CH process 7094 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7095 7096 let vcpu_num = 6; 7097 // Hotplug some CPUs 7098 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7099 // Wait to make sure CPUs are added 7100 thread::sleep(std::time::Duration::new(10, 0)); 7101 // Check the guest sees the correct number 7102 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7103 // Check the CH process has the correct number of vcpu threads 7104 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7105 7106 let vcpu_num = 4; 7107 // Remove some CPUs. Note that Windows doesn't support hot-remove. 7108 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7109 // Wait to make sure CPUs are removed 7110 thread::sleep(std::time::Duration::new(10, 0)); 7111 // Reboot to let Windows catch up 7112 windows_guest.reboot(); 7113 // Wait to make sure Windows completely rebooted 7114 thread::sleep(std::time::Duration::new(60, 0)); 7115 // Check the guest sees the correct number 7116 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7117 // Check the CH process has the correct number of vcpu threads 7118 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7119 7120 windows_guest.shutdown(); 7121 }); 7122 7123 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7124 let _ = child.kill(); 7125 let output = child.wait_with_output().unwrap(); 7126 7127 let _ = child_dnsmasq.kill(); 7128 let _ = child_dnsmasq.wait(); 7129 7130 handle_child_output(r, &output); 7131 } 7132 7133 #[test] 7134 #[cfg(not(feature = "mshv"))] 7135 fn test_windows_guest_ram_hotplug() { 7136 let windows_guest = WindowsGuest::new(); 7137 7138 let mut ovmf_path = dirs::home_dir().unwrap(); 7139 ovmf_path.push("workloads"); 7140 ovmf_path.push(OVMF_NAME); 7141 7142 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7143 let api_socket = temp_api_path(&tmp_dir); 7144 7145 let mut child = GuestCommand::new(windows_guest.guest()) 7146 .args(&["--api-socket", &api_socket]) 7147 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7148 .args(&["--memory", "size=2G,hotplug_size=5G"]) 7149 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7150 .args(&["--serial", "tty"]) 7151 .args(&["--console", "off"]) 7152 .default_disks() 7153 .default_net() 7154 .capture_output() 7155 .spawn() 7156 .unwrap(); 7157 7158 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7159 7160 let r = std::panic::catch_unwind(|| { 7161 // Wait to make sure Windows boots up 7162 assert!(windows_guest.wait_for_boot()); 7163 7164 let ram_size = 2 * 1024 * 1024 * 1024; 7165 // Check the initial number of RAM the guest sees 7166 let current_ram_size = windows_guest.ram_size(); 7167 // This size seems to be reserved by the system and thus the 7168 // reported amount differs by this constant value. 7169 let reserved_ram_size = ram_size - current_ram_size; 7170 // Verify that there's not more than 4mb constant diff wasted 7171 // by the reserved ram. 7172 assert!(reserved_ram_size < 4 * 1024 * 1024); 7173 7174 let ram_size = 4 * 1024 * 1024 * 1024; 7175 // Hotplug some RAM 7176 resize_command(&api_socket, None, Some(ram_size), None, None); 7177 // Wait to make sure RAM has been added 7178 thread::sleep(std::time::Duration::new(10, 0)); 7179 // Check the guest sees the correct number 7180 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7181 7182 let ram_size = 3 * 1024 * 1024 * 1024; 7183 // Unplug some RAM. Note that hot-remove most likely won't work. 7184 resize_command(&api_socket, None, Some(ram_size), None, None); 7185 // Wait to make sure RAM has been added 7186 thread::sleep(std::time::Duration::new(10, 0)); 7187 // Reboot to let Windows catch up 7188 windows_guest.reboot(); 7189 // Wait to make sure guest completely rebooted 7190 thread::sleep(std::time::Duration::new(60, 0)); 7191 // Check the guest sees the correct number 7192 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7193 7194 windows_guest.shutdown(); 7195 }); 7196 7197 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7198 let _ = child.kill(); 7199 let output = child.wait_with_output().unwrap(); 7200 7201 let _ = child_dnsmasq.kill(); 7202 let _ = child_dnsmasq.wait(); 7203 7204 handle_child_output(r, &output); 7205 } 7206 7207 #[test] 7208 #[cfg(not(feature = "mshv"))] 7209 fn test_windows_guest_netdev_hotplug() { 7210 let windows_guest = WindowsGuest::new(); 7211 7212 let mut ovmf_path = dirs::home_dir().unwrap(); 7213 ovmf_path.push("workloads"); 7214 ovmf_path.push(OVMF_NAME); 7215 7216 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7217 let api_socket = temp_api_path(&tmp_dir); 7218 7219 let mut child = GuestCommand::new(windows_guest.guest()) 7220 .args(&["--api-socket", &api_socket]) 7221 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7222 .args(&["--memory", "size=4G"]) 7223 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7224 .args(&["--serial", "tty"]) 7225 .args(&["--console", "off"]) 7226 .default_disks() 7227 .default_net() 7228 .capture_output() 7229 .spawn() 7230 .unwrap(); 7231 7232 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7233 7234 let r = std::panic::catch_unwind(|| { 7235 // Wait to make sure Windows boots up 7236 assert!(windows_guest.wait_for_boot()); 7237 7238 // Initially present network device 7239 let netdev_num = 1; 7240 assert_eq!(windows_guest.netdev_count(), netdev_num); 7241 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7242 7243 // Hotplug network device 7244 let (cmd_success, cmd_output) = remote_command_w_output( 7245 &api_socket, 7246 "add-net", 7247 Some(windows_guest.guest().default_net_string().as_str()), 7248 ); 7249 assert!(cmd_success); 7250 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 7251 thread::sleep(std::time::Duration::new(5, 0)); 7252 // Verify the device is on the system 7253 let netdev_num = 2; 7254 assert_eq!(windows_guest.netdev_count(), netdev_num); 7255 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7256 7257 // Remove network device 7258 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 7259 assert!(cmd_success); 7260 thread::sleep(std::time::Duration::new(5, 0)); 7261 // Verify the device has been removed 7262 let netdev_num = 1; 7263 assert_eq!(windows_guest.netdev_count(), netdev_num); 7264 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7265 7266 windows_guest.shutdown(); 7267 }); 7268 7269 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7270 let _ = child.kill(); 7271 let output = child.wait_with_output().unwrap(); 7272 7273 let _ = child_dnsmasq.kill(); 7274 let _ = child_dnsmasq.wait(); 7275 7276 handle_child_output(r, &output); 7277 } 7278 7279 #[test] 7280 #[cfg(not(feature = "mshv"))] 7281 fn test_windows_guest_disk_hotplug() { 7282 let windows_guest = WindowsGuest::new(); 7283 7284 let mut ovmf_path = dirs::home_dir().unwrap(); 7285 ovmf_path.push("workloads"); 7286 ovmf_path.push(OVMF_NAME); 7287 7288 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7289 let api_socket = temp_api_path(&tmp_dir); 7290 7291 let mut child = GuestCommand::new(windows_guest.guest()) 7292 .args(&["--api-socket", &api_socket]) 7293 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7294 .args(&["--memory", "size=4G"]) 7295 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7296 .args(&["--serial", "tty"]) 7297 .args(&["--console", "off"]) 7298 .default_disks() 7299 .default_net() 7300 .capture_output() 7301 .spawn() 7302 .unwrap(); 7303 7304 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7305 7306 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 7307 7308 let r = std::panic::catch_unwind(|| { 7309 // Wait to make sure Windows boots up 7310 assert!(windows_guest.wait_for_boot()); 7311 7312 // Initially present disk device 7313 let disk_num = 1; 7314 assert_eq!(windows_guest.disk_count(), disk_num); 7315 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7316 7317 // Hotplug disk device 7318 let (cmd_success, cmd_output) = remote_command_w_output( 7319 &api_socket, 7320 "add-disk", 7321 Some(format!("path={},readonly=off", disk).as_str()), 7322 ); 7323 assert!(cmd_success); 7324 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 7325 thread::sleep(std::time::Duration::new(5, 0)); 7326 // Online disk device 7327 windows_guest.disks_set_rw(); 7328 windows_guest.disks_online(); 7329 // Verify the device is on the system 7330 let disk_num = 2; 7331 assert_eq!(windows_guest.disk_count(), disk_num); 7332 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7333 7334 let data = "hello"; 7335 let fname = "d:\\world"; 7336 windows_guest.disk_file_put(fname, data); 7337 7338 // Unmount disk device 7339 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 7340 assert!(cmd_success); 7341 thread::sleep(std::time::Duration::new(5, 0)); 7342 // Verify the device has been removed 7343 let disk_num = 1; 7344 assert_eq!(windows_guest.disk_count(), disk_num); 7345 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7346 7347 // Remount and check the file exists with the expected contents 7348 let (cmd_success, _cmd_output) = remote_command_w_output( 7349 &api_socket, 7350 "add-disk", 7351 Some(format!("path={},readonly=off", disk).as_str()), 7352 ); 7353 assert!(cmd_success); 7354 thread::sleep(std::time::Duration::new(5, 0)); 7355 let out = windows_guest.disk_file_read(fname); 7356 assert_eq!(data, out.trim()); 7357 7358 // Intentionally no unmount, it'll happen at shutdown. 7359 7360 windows_guest.shutdown(); 7361 }); 7362 7363 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7364 let _ = child.kill(); 7365 let output = child.wait_with_output().unwrap(); 7366 7367 let _ = child_dnsmasq.kill(); 7368 let _ = child_dnsmasq.wait(); 7369 7370 handle_child_output(r, &output); 7371 } 7372 7373 #[test] 7374 #[cfg(not(feature = "mshv"))] 7375 fn test_windows_guest_disk_hotplug_multi() { 7376 let windows_guest = WindowsGuest::new(); 7377 7378 let mut ovmf_path = dirs::home_dir().unwrap(); 7379 ovmf_path.push("workloads"); 7380 ovmf_path.push(OVMF_NAME); 7381 7382 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7383 let api_socket = temp_api_path(&tmp_dir); 7384 7385 let mut child = GuestCommand::new(windows_guest.guest()) 7386 .args(&["--api-socket", &api_socket]) 7387 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7388 .args(&["--memory", "size=2G"]) 7389 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7390 .args(&["--serial", "tty"]) 7391 .args(&["--console", "off"]) 7392 .default_disks() 7393 .default_net() 7394 .capture_output() 7395 .spawn() 7396 .unwrap(); 7397 7398 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7399 7400 // Predefined data to used at various test stages 7401 let disk_test_data: [[String; 4]; 2] = [ 7402 [ 7403 "_disk2".to_string(), 7404 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 7405 "d:\\world".to_string(), 7406 "hello".to_string(), 7407 ], 7408 [ 7409 "_disk3".to_string(), 7410 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 7411 "e:\\hello".to_string(), 7412 "world".to_string(), 7413 ], 7414 ]; 7415 7416 let r = std::panic::catch_unwind(|| { 7417 // Wait to make sure Windows boots up 7418 assert!(windows_guest.wait_for_boot()); 7419 7420 // Initially present disk device 7421 let disk_num = 1; 7422 assert_eq!(windows_guest.disk_count(), disk_num); 7423 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7424 7425 for it in &disk_test_data { 7426 let disk_id = it[0].as_str(); 7427 let disk = it[1].as_str(); 7428 // Hotplug disk device 7429 let (cmd_success, cmd_output) = remote_command_w_output( 7430 &api_socket, 7431 "add-disk", 7432 Some(format!("path={},readonly=off", disk).as_str()), 7433 ); 7434 assert!(cmd_success); 7435 assert!(String::from_utf8_lossy(&cmd_output) 7436 .contains(format!("\"id\":\"{}\"", disk_id).as_str())); 7437 thread::sleep(std::time::Duration::new(5, 0)); 7438 // Online disk devices 7439 windows_guest.disks_set_rw(); 7440 windows_guest.disks_online(); 7441 } 7442 // Verify the devices are on the system 7443 let disk_num = (disk_test_data.len() + 1) as u8; 7444 assert_eq!(windows_guest.disk_count(), disk_num); 7445 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7446 7447 // Put test data 7448 for it in &disk_test_data { 7449 let fname = it[2].as_str(); 7450 let data = it[3].as_str(); 7451 windows_guest.disk_file_put(fname, data); 7452 } 7453 7454 // Unmount disk devices 7455 for it in &disk_test_data { 7456 let disk_id = it[0].as_str(); 7457 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 7458 assert!(cmd_success); 7459 thread::sleep(std::time::Duration::new(5, 0)); 7460 } 7461 7462 // Verify the devices have been removed 7463 let disk_num = 1; 7464 assert_eq!(windows_guest.disk_count(), disk_num); 7465 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7466 7467 // Remount 7468 for it in &disk_test_data { 7469 let disk = it[1].as_str(); 7470 let (cmd_success, _cmd_output) = remote_command_w_output( 7471 &api_socket, 7472 "add-disk", 7473 Some(format!("path={},readonly=off", disk).as_str()), 7474 ); 7475 assert!(cmd_success); 7476 thread::sleep(std::time::Duration::new(5, 0)); 7477 } 7478 7479 // Check the files exists with the expected contents 7480 for it in &disk_test_data { 7481 let fname = it[2].as_str(); 7482 let data = it[3].as_str(); 7483 let out = windows_guest.disk_file_read(fname); 7484 assert_eq!(data, out.trim()); 7485 } 7486 7487 // Intentionally no unmount, it'll happen at shutdown. 7488 7489 windows_guest.shutdown(); 7490 }); 7491 7492 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7493 let _ = child.kill(); 7494 let output = child.wait_with_output().unwrap(); 7495 7496 let _ = child_dnsmasq.kill(); 7497 let _ = child_dnsmasq.wait(); 7498 7499 handle_child_output(r, &output); 7500 } 7501 7502 #[test] 7503 #[cfg(not(feature = "mshv"))] 7504 fn test_windows_guest_netdev_multi() { 7505 let windows_guest = WindowsGuest::new(); 7506 7507 let mut ovmf_path = dirs::home_dir().unwrap(); 7508 ovmf_path.push("workloads"); 7509 ovmf_path.push(OVMF_NAME); 7510 7511 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7512 let api_socket = temp_api_path(&tmp_dir); 7513 7514 let mut child = GuestCommand::new(windows_guest.guest()) 7515 .args(&["--api-socket", &api_socket]) 7516 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 7517 .args(&["--memory", "size=4G"]) 7518 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 7519 .args(&["--serial", "tty"]) 7520 .args(&["--console", "off"]) 7521 .default_disks() 7522 // The multi net dev config is borrowed from test_multiple_network_interfaces 7523 .args(&[ 7524 "--net", 7525 windows_guest.guest().default_net_string().as_str(), 7526 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 7527 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 7528 ]) 7529 .capture_output() 7530 .spawn() 7531 .unwrap(); 7532 7533 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7534 7535 let r = std::panic::catch_unwind(|| { 7536 // Wait to make sure Windows boots up 7537 assert!(windows_guest.wait_for_boot()); 7538 7539 let netdev_num = 3; 7540 assert_eq!(windows_guest.netdev_count(), netdev_num); 7541 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7542 7543 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 7544 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 7545 7546 windows_guest.shutdown(); 7547 }); 7548 7549 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7550 let _ = child.kill(); 7551 let output = child.wait_with_output().unwrap(); 7552 7553 let _ = child_dnsmasq.kill(); 7554 let _ = child_dnsmasq.wait(); 7555 7556 handle_child_output(r, &output); 7557 } 7558 } 7559 7560 #[cfg(target_arch = "x86_64")] 7561 mod sgx { 7562 use crate::*; 7563 7564 #[test] 7565 fn test_sgx() { 7566 let focal = UbuntuDiskConfig::new(FOCAL_SGX_IMAGE_NAME.to_string()); 7567 let guest = Guest::new(Box::new(focal)); 7568 let mut workload_path = dirs::home_dir().unwrap(); 7569 workload_path.push("workloads"); 7570 7571 let mut kernel_path = workload_path; 7572 kernel_path.push("vmlinux_w_sgx"); 7573 7574 let mut child = GuestCommand::new(&guest) 7575 .args(&["--cpus", "boot=1"]) 7576 .args(&["--memory", "size=512M"]) 7577 .args(&["--kernel", kernel_path.to_str().unwrap()]) 7578 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7579 .default_disks() 7580 .default_net() 7581 .args(&["--sgx-epc", "id=epc0,size=64M"]) 7582 .capture_output() 7583 .spawn() 7584 .unwrap(); 7585 7586 let r = std::panic::catch_unwind(|| { 7587 guest.wait_vm_boot(None).unwrap(); 7588 7589 // Check if SGX is correctly detected in the guest. 7590 guest.check_sgx_support().unwrap(); 7591 7592 // Validate the SGX EPC section is 64MiB. 7593 assert_eq!( 7594 guest 7595 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 7596 .unwrap() 7597 .trim(), 7598 "0x0000000004000000" 7599 ); 7600 7601 // Run a test relying on SGX enclaves and check if it runs 7602 // successfully. 7603 assert!(guest 7604 .ssh_command("cd /linux-sgx/SampleCode/LocalAttestation/bin/ && sudo ./app") 7605 .unwrap() 7606 .trim() 7607 .contains( 7608 "succeed to load enclaves.\nsucceed to \ 7609 establish secure channel.\nSucceed to exchange \ 7610 secure message...\nSucceed to close Session..." 7611 )); 7612 }); 7613 7614 let _ = child.kill(); 7615 let output = child.wait_with_output().unwrap(); 7616 7617 handle_child_output(r, &output); 7618 } 7619 } 7620 7621 #[cfg(target_arch = "x86_64")] 7622 mod vfio { 7623 use crate::*; 7624 7625 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 7626 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 7627 let guest = Guest::new(Box::new(hirsute)); 7628 let api_socket = temp_api_path(&guest.tmp_dir); 7629 7630 let mut child = GuestCommand::new(&guest) 7631 .args(&["--cpus", "boot=4"]) 7632 .args(&[ 7633 "--memory", 7634 format!("size=4G,hotplug_size=4G,hotplug_method={}", hotplug_method).as_str(), 7635 ]) 7636 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7637 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 7638 .args(&["--api-socket", &api_socket]) 7639 .default_disks() 7640 .default_net() 7641 .capture_output() 7642 .spawn() 7643 .unwrap(); 7644 7645 let r = std::panic::catch_unwind(|| { 7646 guest.wait_vm_boot(None).unwrap(); 7647 7648 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 7649 7650 guest.enable_memory_hotplug(); 7651 7652 // Add RAM to the VM 7653 let desired_ram = 6 << 30; 7654 resize_command(&api_socket, None, Some(desired_ram), None, None); 7655 thread::sleep(std::time::Duration::new(30, 0)); 7656 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7657 7658 // Check the VFIO device works when RAM is increased to 6GiB 7659 guest.check_nvidia_gpu(); 7660 }); 7661 7662 let _ = child.kill(); 7663 let output = child.wait_with_output().unwrap(); 7664 7665 handle_child_output(r, &output); 7666 } 7667 7668 #[test] 7669 fn test_nvidia_card_memory_hotplug_acpi() { 7670 test_nvidia_card_memory_hotplug("acpi") 7671 } 7672 7673 #[test] 7674 fn test_nvidia_card_memory_hotplug_virtio_mem() { 7675 test_nvidia_card_memory_hotplug("virtio-mem") 7676 } 7677 7678 #[test] 7679 fn test_nvidia_card_pci_hotplug() { 7680 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 7681 let guest = Guest::new(Box::new(hirsute)); 7682 let api_socket = temp_api_path(&guest.tmp_dir); 7683 7684 let mut child = GuestCommand::new(&guest) 7685 .args(&["--cpus", "boot=4"]) 7686 .args(&["--memory", "size=4G"]) 7687 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7688 .args(&["--api-socket", &api_socket]) 7689 .default_disks() 7690 .default_net() 7691 .capture_output() 7692 .spawn() 7693 .unwrap(); 7694 7695 let r = std::panic::catch_unwind(|| { 7696 guest.wait_vm_boot(None).unwrap(); 7697 7698 // Hotplug the card to the VM 7699 let (cmd_success, cmd_output) = remote_command_w_output( 7700 &api_socket, 7701 "add-device", 7702 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 7703 ); 7704 assert!(cmd_success); 7705 assert!(String::from_utf8_lossy(&cmd_output) 7706 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 7707 7708 thread::sleep(std::time::Duration::new(10, 0)); 7709 7710 // Check the VFIO device works after hotplug 7711 guest.check_nvidia_gpu(); 7712 }); 7713 7714 let _ = child.kill(); 7715 let output = child.wait_with_output().unwrap(); 7716 7717 handle_child_output(r, &output); 7718 } 7719 7720 #[test] 7721 fn test_nvidia_card_reboot() { 7722 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 7723 let guest = Guest::new(Box::new(hirsute)); 7724 let api_socket = temp_api_path(&guest.tmp_dir); 7725 7726 let mut child = GuestCommand::new(&guest) 7727 .args(&["--cpus", "boot=4"]) 7728 .args(&["--memory", "size=4G"]) 7729 .args(&["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7730 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 7731 .args(&["--api-socket", &api_socket]) 7732 .default_disks() 7733 .default_net() 7734 .capture_output() 7735 .spawn() 7736 .unwrap(); 7737 7738 let r = std::panic::catch_unwind(|| { 7739 guest.wait_vm_boot(None).unwrap(); 7740 7741 // Check the VFIO device works after boot 7742 guest.check_nvidia_gpu(); 7743 7744 guest.reboot_linux(0, None); 7745 7746 // Check the VFIO device works after reboot 7747 guest.check_nvidia_gpu(); 7748 }); 7749 7750 let _ = child.kill(); 7751 let output = child.wait_with_output().unwrap(); 7752 7753 handle_child_output(r, &output); 7754 } 7755 } 7756 7757 mod live_migration { 7758 use crate::*; 7759 7760 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 7761 // same host. It ensures the following behaviors: 7762 // 1. The source VM is up and functional (including various virtio-devices are working properly); 7763 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 7764 // 3. The source VM terminated gracefully after live migration; 7765 // 4. The destination VM is functional (including various virtio-devices are working properly) after 7766 // live migration; 7767 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 7768 fn _test_live_migration( 7769 upgrade_test: bool, 7770 numa: bool, 7771 local: bool, 7772 watchdog: bool, 7773 balloon: bool, 7774 ) { 7775 assert!( 7776 !(numa && balloon), 7777 "Invalid inputs: 'numa' and 'balloon' cannot be tested at the same time." 7778 ); 7779 7780 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7781 let guest = Guest::new(Box::new(focal)); 7782 let kernel_path = direct_kernel_boot_path(); 7783 let console_text = String::from("On a branch floating down river a cricket, singing."); 7784 let net_id = "net123"; 7785 let net_params = format!( 7786 "id={},tap=,mac={},ip={},mask=255.255.255.0", 7787 net_id, guest.network.guest_mac, guest.network.host_ip 7788 ); 7789 7790 let memory_param: &[&str] = match (balloon, local, numa) { 7791 (false, false, false) => &["--memory", "size=4G"], 7792 (false, false, true) => &[ 7793 "--memory", 7794 "size=0,hotplug_method=virtio-mem", 7795 "--memory-zone", 7796 "id=mem0,size=1G,hotplug_size=4G", 7797 "id=mem1,size=1G,hotplug_size=4G", 7798 "id=mem2,size=2G,hotplug_size=4G", 7799 "--numa", 7800 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 7801 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 7802 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 7803 ], 7804 (false, true, false) => &["--memory", "size=4G,shared=on"], 7805 (false, true, true) => &[ 7806 "--memory", 7807 "size=0,hotplug_method=virtio-mem,shared=on", 7808 "--memory-zone", 7809 "id=mem0,size=1G,hotplug_size=4G,shared=on", 7810 "id=mem1,size=1G,hotplug_size=4G,shared=on", 7811 "id=mem2,size=2G,hotplug_size=4G,shared=on", 7812 "--numa", 7813 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 7814 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 7815 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 7816 ], 7817 (true, false, _) => &[ 7818 "--memory", 7819 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 7820 "--balloon", 7821 "size=0", 7822 ], 7823 (true, true, _) => &[ 7824 "--memory", 7825 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 7826 "--balloon", 7827 "size=0", 7828 ], 7829 }; 7830 7831 let pmem_temp_file = TempFile::new().unwrap(); 7832 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 7833 std::process::Command::new("mkfs.ext4") 7834 .arg(pmem_temp_file.as_path()) 7835 .output() 7836 .expect("Expect creating disk image to succeed"); 7837 let pmem_path = String::from("/dev/pmem0"); 7838 7839 // Start the source VM 7840 let src_vm_path = if !upgrade_test { 7841 clh_command("cloud-hypervisor") 7842 } else { 7843 cloud_hypervisor_release_path() 7844 }; 7845 let src_api_socket = temp_api_path(&guest.tmp_dir); 7846 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 7847 src_vm_cmd 7848 .args(&["--cpus", "boot=6,max=12"]) 7849 .args(memory_param) 7850 .args(&["--kernel", kernel_path.to_str().unwrap()]) 7851 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7852 .default_disks() 7853 .args(&["--net", net_params.as_str()]) 7854 .args(&["--api-socket", &src_api_socket]) 7855 .args(&[ 7856 "--pmem", 7857 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 7858 ]); 7859 if watchdog { 7860 src_vm_cmd.args(&["--watchdog"]); 7861 } 7862 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 7863 7864 // Start the destination VM 7865 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 7866 dest_api_socket.push_str(".dest"); 7867 let mut dest_child = GuestCommand::new(&guest) 7868 .args(&["--api-socket", &dest_api_socket]) 7869 .capture_output() 7870 .spawn() 7871 .unwrap(); 7872 7873 let r = std::panic::catch_unwind(|| { 7874 guest.wait_vm_boot(None).unwrap(); 7875 7876 // Make sure the source VM is functaionl 7877 // Check the number of vCPUs 7878 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 6); 7879 7880 // Check the guest RAM 7881 if balloon { 7882 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 7883 // Increase the guest RAM 7884 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 7885 thread::sleep(std::time::Duration::new(5, 0)); 7886 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7887 // Use balloon to remove RAM from the VM 7888 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 7889 thread::sleep(std::time::Duration::new(5, 0)); 7890 let total_memory = guest.get_total_memory().unwrap_or_default(); 7891 assert!(total_memory > 4_800_000); 7892 assert!(total_memory < 5_760_000); 7893 } else if numa { 7894 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 7895 } else { 7896 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 7897 } 7898 7899 // Check the guest virtio-devices, e.g. block, rng, console, and net 7900 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 7901 7902 // Check the NUMA parameters are applied correctly and resize 7903 // each zone to test the case where we migrate a VM with the 7904 // virtio-mem regions being used. 7905 if numa { 7906 guest.check_numa_common( 7907 Some(&[960_000, 960_000, 1_920_000]), 7908 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 7909 Some(&["10 15 20", "20 10 25", "25 30 10"]), 7910 ); 7911 7912 // AArch64 currently does not support hotplug, and therefore we only 7913 // test hotplug-related function on x86_64 here. 7914 #[cfg(target_arch = "x86_64")] 7915 { 7916 guest.enable_memory_hotplug(); 7917 7918 // Resize every memory zone and check each associated NUMA node 7919 // has been assigned the right amount of memory. 7920 resize_zone_command(&src_api_socket, "mem0", "2G"); 7921 resize_zone_command(&src_api_socket, "mem1", "2G"); 7922 resize_zone_command(&src_api_socket, "mem2", "3G"); 7923 thread::sleep(std::time::Duration::new(5, 0)); 7924 7925 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 7926 } 7927 } 7928 7929 // x86_64: Following what's done in the `test_snapshot_restore`, we need 7930 // to make sure that removing and adding back the virtio-net device does 7931 // not break the live-migration support for virtio-pci. 7932 #[cfg(target_arch = "x86_64")] 7933 { 7934 assert!(remote_command( 7935 &src_api_socket, 7936 "remove-device", 7937 Some(net_id), 7938 )); 7939 thread::sleep(std::time::Duration::new(10, 0)); 7940 7941 // Plug the virtio-net device again 7942 assert!(remote_command( 7943 &src_api_socket, 7944 "add-net", 7945 Some(net_params.as_str()), 7946 )); 7947 thread::sleep(std::time::Duration::new(10, 0)); 7948 } 7949 7950 if watchdog { 7951 let mut current_reboot_count = 1; 7952 enable_guest_watchdog(&guest, &mut current_reboot_count); 7953 check_guest_watchdog_no_reboot(&guest, ¤t_reboot_count); 7954 } 7955 7956 // Start the live-migration 7957 let migration_socket = String::from( 7958 guest 7959 .tmp_dir 7960 .as_path() 7961 .join("live-migration.sock") 7962 .to_str() 7963 .unwrap(), 7964 ); 7965 // Start to receive migration from the destintion VM 7966 let mut receive_migration = Command::new(clh_command("ch-remote")) 7967 .args(&[ 7968 &format!("--api-socket={}", &dest_api_socket), 7969 "receive-migration", 7970 &format! {"unix:{}", migration_socket}, 7971 ]) 7972 .stderr(Stdio::piped()) 7973 .stdout(Stdio::piped()) 7974 .spawn() 7975 .unwrap(); 7976 // Give it '1s' to make sure the 'migration_socket' file is properly created 7977 thread::sleep(std::time::Duration::new(1, 0)); 7978 // Start to send migration from the source VM 7979 7980 let mut args = [ 7981 format!("--api-socket={}", &src_api_socket), 7982 "send-migration".to_string(), 7983 format! {"unix:{}", migration_socket}, 7984 ] 7985 .to_vec(); 7986 7987 if local { 7988 args.insert(2, "--local".to_string()); 7989 } 7990 7991 let mut send_migration = Command::new(clh_command("ch-remote")) 7992 .args(&args) 7993 .stderr(Stdio::piped()) 7994 .stdout(Stdio::piped()) 7995 .spawn() 7996 .unwrap(); 7997 7998 // The 'send-migration' command should be executed successfully within the given timeout 7999 let success = if let Some(status) = send_migration 8000 .wait_timeout(std::time::Duration::from_secs(30)) 8001 .unwrap() 8002 { 8003 status.success() 8004 } else { 8005 false 8006 }; 8007 8008 if !success { 8009 let _ = send_migration.kill(); 8010 let output = send_migration.wait_with_output().unwrap(); 8011 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 8012 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8013 } 8014 8015 // The 'receive-migration' command should be executed successfully within the given timeout 8016 let success = if let Some(status) = receive_migration 8017 .wait_timeout(std::time::Duration::from_secs(30)) 8018 .unwrap() 8019 { 8020 status.success() 8021 } else { 8022 false 8023 }; 8024 8025 if !success { 8026 let _ = receive_migration.kill(); 8027 let output = receive_migration.wait_with_output().unwrap(); 8028 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 8029 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8030 } 8031 assert!( 8032 success, 8033 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8034 ); 8035 }); 8036 8037 let print_and_panic = |src_vm: Child, dest_vm: Child, message: &str| -> ! { 8038 let mut src_vm = src_vm; 8039 let mut dest_vm = dest_vm; 8040 8041 let _ = src_vm.kill(); 8042 let src_output = src_vm.wait_with_output().unwrap(); 8043 eprintln!( 8044 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8045 String::from_utf8_lossy(&src_output.stdout) 8046 ); 8047 eprintln!( 8048 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8049 String::from_utf8_lossy(&src_output.stderr) 8050 ); 8051 let _ = dest_vm.kill(); 8052 let dest_output = dest_vm.wait_with_output().unwrap(); 8053 eprintln!( 8054 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8055 String::from_utf8_lossy(&dest_output.stdout) 8056 ); 8057 eprintln!( 8058 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8059 String::from_utf8_lossy(&dest_output.stderr) 8060 ); 8061 8062 panic!("Test failed: {}", message) 8063 }; 8064 8065 // Check and report any errors occured during the live-migration 8066 if r.is_err() { 8067 print_and_panic(src_child, dest_child, "Error occured during live-migration"); 8068 } 8069 8070 // Check the source vm has been terminated successful (give it '3s' to settle) 8071 thread::sleep(std::time::Duration::new(3, 0)); 8072 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8073 print_and_panic( 8074 src_child, 8075 dest_child, 8076 "source VM was not terminated successfully.", 8077 ); 8078 }; 8079 8080 // Post live-migration check to make sure the destination VM is funcational 8081 let r = std::panic::catch_unwind(|| { 8082 // Perform same checks to validate VM has been properly migrated 8083 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 6); 8084 #[cfg(target_arch = "x86_64")] 8085 if numa { 8086 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 8087 } else { 8088 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8089 } 8090 #[cfg(target_arch = "aarch64")] 8091 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8092 8093 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8094 8095 // Perform NUMA related checks 8096 if numa { 8097 #[cfg(target_arch = "aarch64")] 8098 { 8099 guest.check_numa_common( 8100 Some(&[960_000, 960_000, 1_920_000]), 8101 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8102 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8103 ); 8104 } 8105 8106 // AArch64 currently does not support hotplug, and therefore we only 8107 // test hotplug-related function on x86_64 here. 8108 #[cfg(target_arch = "x86_64")] 8109 { 8110 guest.check_numa_common( 8111 Some(&[1_920_000, 1_920_000, 2_880_000]), 8112 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8113 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8114 ); 8115 8116 guest.enable_memory_hotplug(); 8117 8118 // Resize every memory zone and check each associated NUMA node 8119 // has been assigned the right amount of memory. 8120 resize_zone_command(&dest_api_socket, "mem0", "4G"); 8121 resize_zone_command(&dest_api_socket, "mem1", "4G"); 8122 resize_zone_command(&dest_api_socket, "mem2", "4G"); 8123 // Resize to the maximum amount of CPUs and check each NUMA 8124 // node has been assigned the right CPUs set. 8125 resize_command(&dest_api_socket, Some(12), None, None, None); 8126 thread::sleep(std::time::Duration::new(5, 0)); 8127 8128 guest.check_numa_common( 8129 Some(&[3_840_000, 3_840_000, 3_840_000]), 8130 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 8131 None, 8132 ); 8133 } 8134 } 8135 8136 if watchdog { 8137 let mut current_reboot_count = 2; 8138 check_guest_watchdog_no_reboot(&guest, ¤t_reboot_count); 8139 check_guest_watchdog_one_reboot( 8140 &guest, 8141 &dest_api_socket, 8142 &mut current_reboot_count, 8143 ); 8144 } 8145 8146 if balloon { 8147 let total_memory = guest.get_total_memory().unwrap_or_default(); 8148 assert!(total_memory > 4_800_000); 8149 assert!(total_memory < 5_760_000); 8150 // Deflate balloon to restore entire RAM to the VM 8151 resize_command(&dest_api_socket, None, None, Some(0), None); 8152 thread::sleep(std::time::Duration::new(5, 0)); 8153 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8154 // Decrease guest RAM with virtio-mem 8155 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 8156 thread::sleep(std::time::Duration::new(5, 0)); 8157 let total_memory = guest.get_total_memory().unwrap_or_default(); 8158 assert!(total_memory > 4_800_000); 8159 assert!(total_memory < 5_760_000); 8160 } 8161 }); 8162 8163 // Clean-up the destination VM and make sure it terminated correctly 8164 let _ = dest_child.kill(); 8165 let dest_output = dest_child.wait_with_output().unwrap(); 8166 handle_child_output(r, &dest_output); 8167 8168 // Check the destination VM has the expected 'concole_text' from its output 8169 let r = std::panic::catch_unwind(|| { 8170 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8171 }); 8172 handle_child_output(r, &dest_output); 8173 } 8174 8175 #[test] 8176 fn test_live_migration_basic() { 8177 _test_live_migration(false, false, false, false, false) 8178 } 8179 8180 #[test] 8181 fn test_live_migration_local() { 8182 _test_live_migration(false, false, true, false, false) 8183 } 8184 8185 #[test] 8186 #[cfg(not(feature = "mshv"))] 8187 fn test_live_migration_numa() { 8188 _test_live_migration(false, true, false, false, false) 8189 } 8190 8191 #[test] 8192 #[cfg(not(feature = "mshv"))] 8193 fn test_live_migration_numa_local() { 8194 _test_live_migration(false, true, true, false, false) 8195 } 8196 8197 #[test] 8198 fn test_live_migration_watchdog() { 8199 _test_live_migration(false, false, false, true, false) 8200 } 8201 8202 #[test] 8203 fn test_live_migration_watchdog_local() { 8204 _test_live_migration(false, false, true, true, false) 8205 } 8206 8207 #[test] 8208 fn test_live_migration_balloon() { 8209 _test_live_migration(false, false, false, false, true) 8210 } 8211 8212 #[test] 8213 fn test_live_migration_balloon_local() { 8214 _test_live_migration(false, false, true, false, true) 8215 } 8216 8217 #[test] 8218 #[ignore] 8219 fn test_live_upgrade_basic() { 8220 _test_live_migration(true, false, false, false, false) 8221 } 8222 8223 #[test] 8224 #[ignore] 8225 fn test_live_upgrade_local() { 8226 _test_live_migration(true, false, true, false, false) 8227 } 8228 8229 #[test] 8230 #[ignore] 8231 #[cfg(not(feature = "mshv"))] 8232 fn test_live_upgrade_numa() { 8233 _test_live_migration(true, true, false, false, false) 8234 } 8235 8236 #[test] 8237 #[ignore] 8238 #[cfg(not(feature = "mshv"))] 8239 fn test_live_upgrade_numa_local() { 8240 _test_live_migration(true, true, true, false, false) 8241 } 8242 8243 #[test] 8244 #[ignore] 8245 fn test_live_upgrade_watchdog() { 8246 _test_live_migration(true, false, false, true, false) 8247 } 8248 8249 #[test] 8250 #[ignore] 8251 fn test_live_upgrade_watchdog_local() { 8252 _test_live_migration(true, false, true, true, false) 8253 } 8254 8255 #[test] 8256 #[ignore] 8257 fn test_live_upgrade_balloon() { 8258 _test_live_migration(true, false, false, false, true) 8259 } 8260 8261 #[test] 8262 #[ignore] 8263 fn test_live_upgrade_balloon_local() { 8264 _test_live_migration(true, false, true, false, true) 8265 } 8266 8267 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 8268 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8269 let ovs_guest = Guest::new(Box::new(ovs_focal)); 8270 8271 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8272 let migration_guest = Guest::new(Box::new(migration_focal)); 8273 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 8274 8275 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 8276 let (mut ovs_child, mut src_child) = 8277 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 8278 8279 // Start the destination VM 8280 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 8281 dest_api_socket.push_str(".dest"); 8282 let mut dest_child = GuestCommand::new(&migration_guest) 8283 .args(&["--api-socket", &dest_api_socket]) 8284 .capture_output() 8285 .spawn() 8286 .unwrap(); 8287 8288 let r = std::panic::catch_unwind(|| { 8289 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 8290 thread::sleep(std::time::Duration::new(1, 0)); 8291 8292 // Start the live-migration 8293 let migration_socket = String::from( 8294 migration_guest 8295 .tmp_dir 8296 .as_path() 8297 .join("live-migration.sock") 8298 .to_str() 8299 .unwrap(), 8300 ); 8301 // Start to receive migration from the destintion VM 8302 let mut receive_migration = Command::new(clh_command("ch-remote")) 8303 .args(&[ 8304 &format!("--api-socket={}", &dest_api_socket), 8305 "receive-migration", 8306 &format! {"unix:{}", migration_socket}, 8307 ]) 8308 .stderr(Stdio::piped()) 8309 .stdout(Stdio::piped()) 8310 .spawn() 8311 .unwrap(); 8312 // Give it '1s' to make sure the 'migration_socket' file is properly created 8313 thread::sleep(std::time::Duration::new(1, 0)); 8314 // Start to send migration from the source VM 8315 let mut args = [ 8316 format!("--api-socket={}", &src_api_socket), 8317 "send-migration".to_string(), 8318 format! {"unix:{}", migration_socket}, 8319 ] 8320 .to_vec(); 8321 8322 if local { 8323 args.insert(2, "--local".to_string()); 8324 } 8325 8326 let mut send_migration = Command::new(clh_command("ch-remote")) 8327 .args(&args) 8328 .stderr(Stdio::piped()) 8329 .stdout(Stdio::piped()) 8330 .spawn() 8331 .unwrap(); 8332 8333 // The 'send-migration' command should be executed successfully within the given timeout 8334 let success = if let Some(status) = send_migration 8335 .wait_timeout(std::time::Duration::from_secs(30)) 8336 .unwrap() 8337 { 8338 status.success() 8339 } else { 8340 false 8341 }; 8342 8343 if !success { 8344 let _ = send_migration.kill(); 8345 let output = send_migration.wait_with_output().unwrap(); 8346 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 8347 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8348 } 8349 8350 // The 'receive-migration' command should be executed successfully within the given timeout 8351 let success = if let Some(status) = receive_migration 8352 .wait_timeout(std::time::Duration::from_secs(30)) 8353 .unwrap() 8354 { 8355 status.success() 8356 } else { 8357 false 8358 }; 8359 8360 if !success { 8361 let _ = receive_migration.kill(); 8362 let output = receive_migration.wait_with_output().unwrap(); 8363 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 8364 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8365 } 8366 assert!( 8367 success, 8368 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8369 ); 8370 }); 8371 8372 let print_and_panic = |src_vm: Child, dest_vm: Child, ovs_vm: Child, message: &str| -> ! { 8373 let mut src_vm = src_vm; 8374 let mut dest_vm = dest_vm; 8375 let mut ovs_vm = ovs_vm; 8376 8377 let _ = src_vm.kill(); 8378 let src_output = src_vm.wait_with_output().unwrap(); 8379 eprintln!( 8380 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8381 String::from_utf8_lossy(&src_output.stdout) 8382 ); 8383 eprintln!( 8384 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8385 String::from_utf8_lossy(&src_output.stderr) 8386 ); 8387 let _ = dest_vm.kill(); 8388 let dest_output = dest_vm.wait_with_output().unwrap(); 8389 eprintln!( 8390 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8391 String::from_utf8_lossy(&dest_output.stdout) 8392 ); 8393 eprintln!( 8394 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8395 String::from_utf8_lossy(&dest_output.stderr) 8396 ); 8397 let _ = ovs_vm.kill(); 8398 let ovs_output = ovs_vm.wait_with_output().unwrap(); 8399 eprintln!( 8400 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 8401 String::from_utf8_lossy(&ovs_output.stdout) 8402 ); 8403 eprintln!( 8404 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 8405 String::from_utf8_lossy(&ovs_output.stderr) 8406 ); 8407 8408 cleanup_ovs_dpdk(); 8409 8410 panic!("Test failed: {}", message) 8411 }; 8412 8413 // Check and report any errors occured during the live-migration 8414 if r.is_err() { 8415 print_and_panic( 8416 src_child, 8417 dest_child, 8418 ovs_child, 8419 "Error occured during live-migration", 8420 ); 8421 } 8422 8423 // Check the source vm has been terminated successful (give it '3s' to settle) 8424 thread::sleep(std::time::Duration::new(3, 0)); 8425 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8426 print_and_panic( 8427 src_child, 8428 dest_child, 8429 ovs_child, 8430 "source VM was not terminated successfully.", 8431 ); 8432 }; 8433 8434 // Post live-migration check to make sure the destination VM is funcational 8435 let r = std::panic::catch_unwind(|| { 8436 // Perform same checks to validate VM has been properly migrated 8437 // Spawn a new netcat listener in the OVS VM 8438 let guest_ip = ovs_guest.network.guest_ip.clone(); 8439 thread::spawn(move || { 8440 ssh_command_ip( 8441 "nc -l 12345", 8442 &guest_ip, 8443 DEFAULT_SSH_RETRIES, 8444 DEFAULT_SSH_TIMEOUT, 8445 ) 8446 .unwrap(); 8447 }); 8448 8449 // Wait for the server to be listening 8450 thread::sleep(std::time::Duration::new(5, 0)); 8451 8452 // And check the connection is still functional after live-migration 8453 migration_guest 8454 .ssh_command("nc -vz 172.100.0.1 12345") 8455 .unwrap(); 8456 8457 cleanup_ovs_dpdk(); 8458 }); 8459 8460 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 8461 let _ = dest_child.kill(); 8462 let _ = ovs_child.kill(); 8463 let dest_output = dest_child.wait_with_output().unwrap(); 8464 handle_child_output(r, &dest_output); 8465 let ovs_output = ovs_child.wait_with_output().unwrap(); 8466 handle_child_output(Ok(()), &ovs_output); 8467 } 8468 8469 #[test] 8470 #[cfg(not(feature = "mshv"))] 8471 fn test_live_migration_ovs_dpdk() { 8472 _test_live_migration_ovs_dpdk(false, false); 8473 } 8474 8475 #[test] 8476 #[cfg(not(feature = "mshv"))] 8477 fn test_live_migration_ovs_dpdk_local() { 8478 _test_live_migration_ovs_dpdk(false, true); 8479 } 8480 8481 #[test] 8482 #[ignore] 8483 #[cfg(not(feature = "mshv"))] 8484 fn test_live_upgrade_ovs_dpdk() { 8485 _test_live_migration_ovs_dpdk(true, false); 8486 } 8487 8488 #[test] 8489 #[ignore] 8490 #[cfg(not(feature = "mshv"))] 8491 fn test_live_upgrade_ovs_dpdk_local() { 8492 _test_live_migration_ovs_dpdk(true, true); 8493 } 8494 } 8495 8496 #[cfg(target_arch = "aarch64")] 8497 mod aarch64_acpi { 8498 use crate::*; 8499 8500 #[test] 8501 fn test_simple_launch_acpi() { 8502 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8503 8504 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 8505 let guest = Guest::new(disk_config); 8506 8507 let mut child = GuestCommand::new(&guest) 8508 .args(&["--cpus", "boot=1"]) 8509 .args(&["--memory", "size=512M"]) 8510 .args(&["--kernel", edk2_path().to_str().unwrap()]) 8511 .default_disks() 8512 .default_net() 8513 .args(&["--serial", "tty", "--console", "off"]) 8514 .capture_output() 8515 .spawn() 8516 .unwrap(); 8517 8518 let r = std::panic::catch_unwind(|| { 8519 guest.wait_vm_boot(Some(120)).unwrap(); 8520 8521 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 8522 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 8523 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 8524 }); 8525 8526 let _ = child.kill(); 8527 let output = child.wait_with_output().unwrap(); 8528 8529 handle_child_output(r, &output); 8530 }); 8531 } 8532 8533 #[test] 8534 fn test_guest_numa_nodes_acpi() { 8535 _test_guest_numa_nodes(true); 8536 } 8537 8538 #[test] 8539 fn test_cpu_topology_421_acpi() { 8540 test_cpu_topology(4, 2, 1, true); 8541 } 8542 8543 #[test] 8544 fn test_cpu_topology_142_acpi() { 8545 test_cpu_topology(1, 4, 2, true); 8546 } 8547 8548 #[test] 8549 fn test_cpu_topology_262_acpi() { 8550 test_cpu_topology(2, 6, 2, true); 8551 } 8552 8553 #[test] 8554 fn test_power_button_acpi() { 8555 _test_power_button(true); 8556 } 8557 8558 #[test] 8559 fn test_virtio_iommu() { 8560 _test_virtio_iommu(true) 8561 } 8562 } 8563