1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 use std::sync::Mutex; 28 use std::thread; 29 use test_infra::*; 30 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 31 use wait_timeout::ChildExt; 32 33 #[cfg(target_arch = "x86_64")] 34 mod x86_64 { 35 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw"; 36 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 37 pub const FOCAL_SGX_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-sgx.raw"; 38 pub const HIRSUTE_NVIDIA_IMAGE_NAME: &str = "hirsute-server-cloudimg-amd64-nvidia.raw"; 39 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 40 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 41 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 42 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20220329-0.raw"; 43 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2019.raw"; 44 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 45 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 46 } 47 48 #[cfg(target_arch = "x86_64")] 49 use x86_64::*; 50 51 #[cfg(target_arch = "aarch64")] 52 mod aarch64 { 53 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 54 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 55 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 56 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 57 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 58 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 59 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 60 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 61 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 62 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 63 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 64 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 65 } 66 67 #[cfg(target_arch = "aarch64")] 68 use aarch64::*; 69 70 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 71 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 72 73 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 74 75 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 76 let mut workload_path = dirs::home_dir().unwrap(); 77 workload_path.push("workloads"); 78 79 let mut virtiofsd_path = workload_path; 80 virtiofsd_path.push("virtiofsd"); 81 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 82 83 let virtiofsd_socket_path = 84 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 85 86 // Start the daemon 87 let child = Command::new(virtiofsd_path.as_str()) 88 .args(["--shared-dir", shared_dir]) 89 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 90 .args(["--cache", "never"]) 91 .spawn() 92 .unwrap(); 93 94 thread::sleep(std::time::Duration::new(10, 0)); 95 96 (child, virtiofsd_socket_path) 97 } 98 99 fn prepare_vubd( 100 tmp_dir: &TempDir, 101 blk_img: &str, 102 num_queues: usize, 103 rdonly: bool, 104 direct: bool, 105 ) -> (std::process::Child, String) { 106 let mut workload_path = dirs::home_dir().unwrap(); 107 workload_path.push("workloads"); 108 109 let mut blk_file_path = workload_path; 110 blk_file_path.push(blk_img); 111 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 112 113 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 114 115 // Start the daemon 116 let child = Command::new(clh_command("vhost_user_block")) 117 .args([ 118 "--block-backend", 119 format!( 120 "path={},socket={},num_queues={},readonly={},direct={}", 121 blk_file_path, vubd_socket_path, num_queues, rdonly, direct 122 ) 123 .as_str(), 124 ]) 125 .spawn() 126 .unwrap(); 127 128 thread::sleep(std::time::Duration::new(10, 0)); 129 130 (child, vubd_socket_path) 131 } 132 133 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 134 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 135 } 136 137 fn temp_api_path(tmp_dir: &TempDir) -> String { 138 String::from( 139 tmp_dir 140 .as_path() 141 .join("cloud-hypervisor.sock") 142 .to_str() 143 .unwrap(), 144 ) 145 } 146 147 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 148 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 149 } 150 151 // Creates the directory and returns the path. 152 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 153 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 154 std::fs::create_dir(&snapshot_dir).unwrap(); 155 snapshot_dir 156 } 157 158 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 159 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 160 vmcore_file 161 } 162 163 // Creates the path for direct kernel boot and return the path. 164 // For x86_64, this function returns the vmlinux kernel path. 165 // For AArch64, this function returns the PE kernel path. 166 fn direct_kernel_boot_path() -> PathBuf { 167 let mut workload_path = dirs::home_dir().unwrap(); 168 workload_path.push("workloads"); 169 170 let mut kernel_path = workload_path; 171 #[cfg(target_arch = "x86_64")] 172 kernel_path.push("vmlinux"); 173 #[cfg(target_arch = "aarch64")] 174 kernel_path.push("Image"); 175 176 kernel_path 177 } 178 179 fn edk2_path() -> PathBuf { 180 let mut workload_path = dirs::home_dir().unwrap(); 181 workload_path.push("workloads"); 182 let mut edk2_path = workload_path; 183 edk2_path.push(OVMF_NAME); 184 185 edk2_path 186 } 187 188 fn cloud_hypervisor_release_path() -> String { 189 let mut workload_path = dirs::home_dir().unwrap(); 190 workload_path.push("workloads"); 191 192 let mut ch_release_path = workload_path; 193 #[cfg(target_arch = "x86_64")] 194 ch_release_path.push("cloud-hypervisor-static"); 195 #[cfg(target_arch = "aarch64")] 196 ch_release_path.push("cloud-hypervisor-static-aarch64"); 197 198 ch_release_path.into_os_string().into_string().unwrap() 199 } 200 201 fn prepare_vhost_user_net_daemon( 202 tmp_dir: &TempDir, 203 ip: &str, 204 tap: Option<&str>, 205 mtu: Option<u16>, 206 num_queues: usize, 207 client_mode: bool, 208 ) -> (std::process::Command, String) { 209 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 210 211 // Start the daemon 212 let mut net_params = format!( 213 "ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 214 ip, vunet_socket_path, num_queues, client_mode 215 ); 216 217 if let Some(tap) = tap { 218 net_params.push_str(format!(",tap={}", tap).as_str()); 219 } 220 221 if let Some(mtu) = mtu { 222 net_params.push_str(format!(",mtu={}", mtu).as_str()); 223 } 224 225 let mut command = Command::new(clh_command("vhost_user_net")); 226 command.args(["--net-backend", net_params.as_str()]); 227 228 (command, vunet_socket_path) 229 } 230 231 fn curl_command(api_socket: &str, method: &str, url: &str, http_body: Option<&str>) { 232 let mut curl_args: Vec<&str> = ["--unix-socket", api_socket, "-i", "-X", method, url].to_vec(); 233 234 if let Some(body) = http_body { 235 curl_args.push("-H"); 236 curl_args.push("Accept: application/json"); 237 curl_args.push("-H"); 238 curl_args.push("Content-Type: application/json"); 239 curl_args.push("-d"); 240 curl_args.push(body); 241 } 242 243 let status = Command::new("curl") 244 .args(curl_args) 245 .status() 246 .expect("Failed to launch curl command"); 247 248 assert!(status.success()); 249 } 250 251 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 252 let mut cmd = Command::new(clh_command("ch-remote")); 253 cmd.args([&format!("--api-socket={}", api_socket), command]); 254 255 if let Some(arg) = arg { 256 cmd.arg(arg); 257 } 258 let output = cmd.output().unwrap(); 259 if output.status.success() { 260 true 261 } else { 262 eprintln!("Error running ch-remote command: {:?}", &cmd); 263 let stderr = String::from_utf8_lossy(&output.stderr); 264 eprintln!("stderr: {}", stderr); 265 false 266 } 267 } 268 269 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 270 let mut cmd = Command::new(clh_command("ch-remote")); 271 cmd.args([&format!("--api-socket={}", api_socket), command]); 272 273 if let Some(arg) = arg { 274 cmd.arg(arg); 275 } 276 277 let output = cmd.output().expect("Failed to launch ch-remote"); 278 279 (output.status.success(), output.stdout) 280 } 281 282 fn resize_command( 283 api_socket: &str, 284 desired_vcpus: Option<u8>, 285 desired_ram: Option<usize>, 286 desired_balloon: Option<usize>, 287 event_file: Option<&str>, 288 ) -> bool { 289 let mut cmd = Command::new(clh_command("ch-remote")); 290 cmd.args([&format!("--api-socket={}", api_socket), "resize"]); 291 292 if let Some(desired_vcpus) = desired_vcpus { 293 cmd.arg(format!("--cpus={}", desired_vcpus)); 294 } 295 296 if let Some(desired_ram) = desired_ram { 297 cmd.arg(format!("--memory={}", desired_ram)); 298 } 299 300 if let Some(desired_balloon) = desired_balloon { 301 cmd.arg(format!("--balloon={}", desired_balloon)); 302 } 303 304 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 305 306 if let Some(event_path) = event_file { 307 let latest_events = [ 308 &MetaEvent { 309 event: "resizing".to_string(), 310 device_id: None, 311 }, 312 &MetaEvent { 313 event: "resized".to_string(), 314 device_id: None, 315 }, 316 ]; 317 assert!(check_latest_events_exact(&latest_events, event_path)); 318 } 319 320 ret 321 } 322 323 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 324 let mut cmd = Command::new(clh_command("ch-remote")); 325 cmd.args([ 326 &format!("--api-socket={}", api_socket), 327 "resize-zone", 328 &format!("--id={}", id), 329 &format!("--size={}", desired_size), 330 ]); 331 332 cmd.status().expect("Failed to launch ch-remote").success() 333 } 334 335 // setup OVS-DPDK bridge and ports 336 fn setup_ovs_dpdk() { 337 // setup OVS-DPDK 338 assert!(exec_host_command_status("service openvswitch-switch start").success()); 339 assert!(exec_host_command_status("ovs-vsctl init").success()); 340 assert!( 341 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 342 .success() 343 ); 344 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 345 346 // Create OVS-DPDK bridge and ports 347 assert!(exec_host_command_status( 348 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 349 ) 350 .success()); 351 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 352 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 353 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 354 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 355 } 356 fn cleanup_ovs_dpdk() { 357 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 358 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 359 } 360 // Setup two guests and ensure they are connected through ovs-dpdk 361 fn setup_ovs_dpdk_guests( 362 guest1: &Guest, 363 guest2: &Guest, 364 api_socket: &str, 365 release_binary: bool, 366 ) -> (Child, Child) { 367 setup_ovs_dpdk(); 368 369 let clh_path = if !release_binary { 370 clh_command("cloud-hypervisor") 371 } else { 372 cloud_hypervisor_release_path() 373 }; 374 375 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 376 .args(["--cpus", "boot=2"]) 377 .args(["--memory", "size=0,shared=on"]) 378 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 379 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 380 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 381 .default_disks() 382 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 383 .capture_output() 384 .spawn() 385 .unwrap(); 386 387 #[cfg(target_arch = "x86_64")] 388 let guest_net_iface = "ens5"; 389 #[cfg(target_arch = "aarch64")] 390 let guest_net_iface = "enp0s5"; 391 392 let r = std::panic::catch_unwind(|| { 393 guest1.wait_vm_boot(None).unwrap(); 394 395 guest1 396 .ssh_command(&format!( 397 "sudo ip addr add 172.100.0.1/24 dev {}", 398 guest_net_iface 399 )) 400 .unwrap(); 401 guest1 402 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 403 .unwrap(); 404 405 let guest_ip = guest1.network.guest_ip.clone(); 406 thread::spawn(move || { 407 ssh_command_ip( 408 "nc -l 12345", 409 &guest_ip, 410 DEFAULT_SSH_RETRIES, 411 DEFAULT_SSH_TIMEOUT, 412 ) 413 .unwrap(); 414 }); 415 }); 416 if r.is_err() { 417 cleanup_ovs_dpdk(); 418 419 let _ = child1.kill(); 420 let output = child1.wait_with_output().unwrap(); 421 handle_child_output(r, &output); 422 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 423 } 424 425 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 426 .args(["--api-socket", api_socket]) 427 .args(["--cpus", "boot=2"]) 428 .args(["--memory", "size=0,shared=on"]) 429 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 430 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 431 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 432 .default_disks() 433 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 434 .capture_output() 435 .spawn() 436 .unwrap(); 437 438 let r = std::panic::catch_unwind(|| { 439 guest2.wait_vm_boot(None).unwrap(); 440 441 guest2 442 .ssh_command(&format!( 443 "sudo ip addr add 172.100.0.2/24 dev {}", 444 guest_net_iface 445 )) 446 .unwrap(); 447 guest2 448 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 449 .unwrap(); 450 451 // Check the connection works properly between the two VMs 452 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 453 }); 454 if r.is_err() { 455 cleanup_ovs_dpdk(); 456 457 let _ = child1.kill(); 458 let _ = child2.kill(); 459 let output = child2.wait_with_output().unwrap(); 460 handle_child_output(r, &output); 461 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 462 } 463 464 (child1, child2) 465 } 466 467 enum FwType { 468 Ovmf, 469 RustHypervisorFirmware, 470 } 471 472 fn fw_path(_fw_type: FwType) -> String { 473 let mut workload_path = dirs::home_dir().unwrap(); 474 workload_path.push("workloads"); 475 476 let mut fw_path = workload_path; 477 #[cfg(target_arch = "aarch64")] 478 fw_path.push("CLOUDHV_EFI.fd"); 479 #[cfg(target_arch = "x86_64")] 480 { 481 match _fw_type { 482 FwType::Ovmf => fw_path.push(OVMF_NAME), 483 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 484 } 485 } 486 487 fw_path.to_str().unwrap().to_string() 488 } 489 490 struct MetaEvent { 491 event: String, 492 device_id: Option<String>, 493 } 494 495 impl MetaEvent { 496 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 497 let mut matched = false; 498 if v["event"].as_str().unwrap() == self.event { 499 if let Some(device_id) = &self.device_id { 500 if v["properties"]["id"].as_str().unwrap() == device_id { 501 matched = true 502 } 503 } else { 504 matched = true; 505 } 506 } 507 matched 508 } 509 } 510 511 // Parse the event_monitor file based on the format that each event 512 // is followed by a double newline 513 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 514 let content = fs::read(event_file).unwrap(); 515 let mut ret = Vec::new(); 516 for entry in String::from_utf8_lossy(&content) 517 .trim() 518 .split("\n\n") 519 .collect::<Vec<&str>>() 520 { 521 ret.push(serde_json::from_str(entry).unwrap()); 522 } 523 524 ret 525 } 526 527 // Return true if all events from the input 'expected_events' are matched sequentially 528 // with events from the 'event_file' 529 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 530 let json_events = parse_event_file(event_file); 531 let len = expected_events.len(); 532 let mut idx = 0; 533 for e in &json_events { 534 if idx == len { 535 break; 536 } 537 if expected_events[idx].match_with_json_event(e) { 538 idx += 1; 539 } 540 } 541 542 idx == len 543 } 544 545 // Return true if all events from the input 'expected_events' are matched exactly 546 // with events from the 'event_file' 547 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 548 let json_events = parse_event_file(event_file); 549 assert!(expected_events.len() <= json_events.len()); 550 let json_events = &json_events[..expected_events.len()]; 551 552 for (idx, e) in json_events.iter().enumerate() { 553 if !expected_events[idx].match_with_json_event(e) { 554 return false; 555 } 556 } 557 558 true 559 } 560 561 // Return true if events from the input 'expected_events' are matched exactly 562 // with the most recent events from the 'event_file' 563 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 564 let json_events = parse_event_file(event_file); 565 assert!(latest_events.len() <= json_events.len()); 566 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 567 568 for (idx, e) in json_events.iter().enumerate() { 569 if !latest_events[idx].match_with_json_event(e) { 570 return false; 571 } 572 } 573 574 true 575 } 576 577 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 578 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 579 let guest = Guest::new(Box::new(focal)); 580 let total_vcpus = threads_per_core * cores_per_package * packages; 581 let direct_kernel_boot_path = direct_kernel_boot_path(); 582 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 583 let fw_path = fw_path(FwType::RustHypervisorFirmware); 584 if use_fw { 585 kernel_path = fw_path.as_str(); 586 } 587 588 let mut child = GuestCommand::new(&guest) 589 .args([ 590 "--cpus", 591 &format!( 592 "boot={},topology={}:{}:1:{}", 593 total_vcpus, threads_per_core, cores_per_package, packages 594 ), 595 ]) 596 .args(["--memory", "size=512M"]) 597 .args(["--kernel", kernel_path]) 598 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 599 .default_disks() 600 .default_net() 601 .capture_output() 602 .spawn() 603 .unwrap(); 604 605 let r = std::panic::catch_unwind(|| { 606 guest.wait_vm_boot(None).unwrap(); 607 assert_eq!( 608 guest.get_cpu_count().unwrap_or_default(), 609 u32::from(total_vcpus) 610 ); 611 assert_eq!( 612 guest 613 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 614 .unwrap() 615 .trim() 616 .parse::<u8>() 617 .unwrap_or(0), 618 threads_per_core 619 ); 620 621 assert_eq!( 622 guest 623 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 624 .unwrap() 625 .trim() 626 .parse::<u8>() 627 .unwrap_or(0), 628 cores_per_package 629 ); 630 631 assert_eq!( 632 guest 633 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 634 .unwrap() 635 .trim() 636 .parse::<u8>() 637 .unwrap_or(0), 638 packages 639 ); 640 }); 641 642 let _ = child.kill(); 643 let output = child.wait_with_output().unwrap(); 644 645 handle_child_output(r, &output); 646 } 647 648 #[allow(unused_variables)] 649 fn _test_guest_numa_nodes(acpi: bool) { 650 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 651 let guest = Guest::new(Box::new(focal)); 652 let api_socket = temp_api_path(&guest.tmp_dir); 653 #[cfg(target_arch = "x86_64")] 654 let kernel_path = direct_kernel_boot_path(); 655 #[cfg(target_arch = "aarch64")] 656 let kernel_path = if acpi { 657 edk2_path() 658 } else { 659 direct_kernel_boot_path() 660 }; 661 662 let mut child = GuestCommand::new(&guest) 663 .args(["--cpus", "boot=6,max=12"]) 664 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 665 .args([ 666 "--memory-zone", 667 "id=mem0,size=1G,hotplug_size=3G", 668 "id=mem1,size=2G,hotplug_size=3G", 669 "id=mem2,size=3G,hotplug_size=3G", 670 ]) 671 .args([ 672 "--numa", 673 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 674 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 675 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 676 ]) 677 .args(["--kernel", kernel_path.to_str().unwrap()]) 678 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 679 .args(["--api-socket", &api_socket]) 680 .capture_output() 681 .default_disks() 682 .default_net() 683 .spawn() 684 .unwrap(); 685 686 let r = std::panic::catch_unwind(|| { 687 guest.wait_vm_boot(None).unwrap(); 688 689 guest.check_numa_common( 690 Some(&[960_000, 1_920_000, 2_880_000]), 691 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 692 Some(&["10 15 20", "20 10 25", "25 30 10"]), 693 ); 694 695 // AArch64 currently does not support hotplug, and therefore we only 696 // test hotplug-related function on x86_64 here. 697 #[cfg(target_arch = "x86_64")] 698 { 699 guest.enable_memory_hotplug(); 700 701 // Resize every memory zone and check each associated NUMA node 702 // has been assigned the right amount of memory. 703 resize_zone_command(&api_socket, "mem0", "4G"); 704 resize_zone_command(&api_socket, "mem1", "4G"); 705 resize_zone_command(&api_socket, "mem2", "4G"); 706 // Resize to the maximum amount of CPUs and check each NUMA 707 // node has been assigned the right CPUs set. 708 resize_command(&api_socket, Some(12), None, None, None); 709 thread::sleep(std::time::Duration::new(5, 0)); 710 711 guest.check_numa_common( 712 Some(&[3_840_000, 3_840_000, 3_840_000]), 713 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 714 None, 715 ); 716 } 717 }); 718 719 let _ = child.kill(); 720 let output = child.wait_with_output().unwrap(); 721 722 handle_child_output(r, &output); 723 } 724 725 #[allow(unused_variables)] 726 fn _test_power_button(acpi: bool) { 727 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 728 let guest = Guest::new(Box::new(focal)); 729 let mut cmd = GuestCommand::new(&guest); 730 let api_socket = temp_api_path(&guest.tmp_dir); 731 732 #[cfg(target_arch = "x86_64")] 733 let kernel_path = direct_kernel_boot_path(); 734 #[cfg(target_arch = "aarch64")] 735 let kernel_path = if acpi { 736 edk2_path() 737 } else { 738 direct_kernel_boot_path() 739 }; 740 741 cmd.args(["--cpus", "boot=1"]) 742 .args(["--memory", "size=512M"]) 743 .args(["--kernel", kernel_path.to_str().unwrap()]) 744 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 745 .capture_output() 746 .default_disks() 747 .default_net() 748 .args(["--api-socket", &api_socket]); 749 750 let child = cmd.spawn().unwrap(); 751 752 let r = std::panic::catch_unwind(|| { 753 guest.wait_vm_boot(None).unwrap(); 754 assert!(remote_command(&api_socket, "power-button", None)); 755 }); 756 757 let output = child.wait_with_output().unwrap(); 758 assert!(output.status.success()); 759 handle_child_output(r, &output); 760 } 761 762 type PrepareNetDaemon = dyn Fn( 763 &TempDir, 764 &str, 765 Option<&str>, 766 Option<u16>, 767 usize, 768 bool, 769 ) -> (std::process::Command, String); 770 771 fn test_vhost_user_net( 772 tap: Option<&str>, 773 num_queues: usize, 774 prepare_daemon: &PrepareNetDaemon, 775 generate_host_mac: bool, 776 client_mode_daemon: bool, 777 ) { 778 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 779 let guest = Guest::new(Box::new(focal)); 780 let api_socket = temp_api_path(&guest.tmp_dir); 781 782 let kernel_path = direct_kernel_boot_path(); 783 784 let host_mac = if generate_host_mac { 785 Some(MacAddr::local_random()) 786 } else { 787 None 788 }; 789 790 let mtu = Some(3000); 791 792 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 793 &guest.tmp_dir, 794 &guest.network.host_ip, 795 tap, 796 mtu, 797 num_queues, 798 client_mode_daemon, 799 ); 800 801 let net_params = format!( 802 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 803 guest.network.guest_mac, 804 vunet_socket_path, 805 num_queues, 806 if let Some(host_mac) = host_mac { 807 format!(",host_mac={}", host_mac) 808 } else { 809 "".to_owned() 810 }, 811 if client_mode_daemon { 812 "server" 813 } else { 814 "client" 815 }, 816 ); 817 818 let mut ch_command = GuestCommand::new(&guest); 819 ch_command 820 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 821 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 822 .args(["--kernel", kernel_path.to_str().unwrap()]) 823 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 824 .default_disks() 825 .args(["--net", net_params.as_str()]) 826 .args(["--api-socket", &api_socket]) 827 .capture_output(); 828 829 let mut daemon_child: std::process::Child; 830 let mut child: std::process::Child; 831 832 if client_mode_daemon { 833 child = ch_command.spawn().unwrap(); 834 // Make sure the VMM is waiting for the backend to connect 835 thread::sleep(std::time::Duration::new(10, 0)); 836 daemon_child = daemon_command.spawn().unwrap(); 837 } else { 838 daemon_child = daemon_command.spawn().unwrap(); 839 // Make sure the backend is waiting for the VMM to connect 840 thread::sleep(std::time::Duration::new(10, 0)); 841 child = ch_command.spawn().unwrap(); 842 } 843 844 let r = std::panic::catch_unwind(|| { 845 guest.wait_vm_boot(None).unwrap(); 846 847 if let Some(tap_name) = tap { 848 let tap_count = exec_host_command_output(&format!("ip link | grep -c {}", tap_name)); 849 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 850 } 851 852 if let Some(host_mac) = tap { 853 let mac_count = exec_host_command_output(&format!("ip link | grep -c {}", host_mac)); 854 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 855 } 856 857 #[cfg(target_arch = "aarch64")] 858 let iface = "enp0s4"; 859 #[cfg(target_arch = "x86_64")] 860 let iface = "ens4"; 861 862 assert_eq!( 863 guest 864 .ssh_command(format!("cat /sys/class/net/{}/mtu", iface).as_str()) 865 .unwrap() 866 .trim(), 867 "3000" 868 ); 869 870 // 1 network interface + default localhost ==> 2 interfaces 871 // It's important to note that this test is fully exercising the 872 // vhost-user-net implementation and the associated backend since 873 // it does not define any --net network interface. That means all 874 // the ssh communication in that test happens through the network 875 // interface backed by vhost-user-net. 876 assert_eq!( 877 guest 878 .ssh_command("ip -o link | wc -l") 879 .unwrap() 880 .trim() 881 .parse::<u32>() 882 .unwrap_or_default(), 883 2 884 ); 885 886 // The following pci devices will appear on guest with PCI-MSI 887 // interrupt vectors assigned. 888 // 1 virtio-console with 3 vectors: config, Rx, Tx 889 // 1 virtio-blk with 2 vectors: config, Request 890 // 1 virtio-blk with 2 vectors: config, Request 891 // 1 virtio-rng with 2 vectors: config, Request 892 // Since virtio-net has 2 queue pairs, its vectors is as follows: 893 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 894 // Based on the above, the total vectors should 14. 895 #[cfg(target_arch = "x86_64")] 896 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 897 #[cfg(target_arch = "aarch64")] 898 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 899 assert_eq!( 900 guest 901 .ssh_command(grep_cmd) 902 .unwrap() 903 .trim() 904 .parse::<u32>() 905 .unwrap_or_default(), 906 10 + (num_queues as u32) 907 ); 908 909 // ACPI feature is needed. 910 #[cfg(target_arch = "x86_64")] 911 { 912 guest.enable_memory_hotplug(); 913 914 // Add RAM to the VM 915 let desired_ram = 1024 << 20; 916 resize_command(&api_socket, None, Some(desired_ram), None, None); 917 918 thread::sleep(std::time::Duration::new(10, 0)); 919 920 // Here by simply checking the size (through ssh), we validate 921 // the connection is still working, which means vhost-user-net 922 // keeps working after the resize. 923 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 924 } 925 }); 926 927 let _ = child.kill(); 928 let output = child.wait_with_output().unwrap(); 929 930 thread::sleep(std::time::Duration::new(5, 0)); 931 let _ = daemon_child.kill(); 932 let _ = daemon_child.wait(); 933 934 handle_child_output(r, &output); 935 } 936 937 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 938 939 fn test_vhost_user_blk( 940 num_queues: usize, 941 readonly: bool, 942 direct: bool, 943 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 944 ) { 945 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 946 let guest = Guest::new(Box::new(focal)); 947 let api_socket = temp_api_path(&guest.tmp_dir); 948 949 let kernel_path = direct_kernel_boot_path(); 950 951 let (blk_params, daemon_child) = { 952 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 953 // Start the daemon 954 let (daemon_child, vubd_socket_path) = 955 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 956 957 ( 958 format!( 959 "vhost_user=true,socket={},num_queues={},queue_size=128", 960 vubd_socket_path, num_queues, 961 ), 962 Some(daemon_child), 963 ) 964 }; 965 966 let mut child = GuestCommand::new(&guest) 967 .args(["--cpus", format!("boot={}", num_queues).as_str()]) 968 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 969 .args(["--kernel", kernel_path.to_str().unwrap()]) 970 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 971 .args([ 972 "--disk", 973 format!( 974 "path={}", 975 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 976 ) 977 .as_str(), 978 format!( 979 "path={}", 980 guest.disk_config.disk(DiskType::CloudInit).unwrap() 981 ) 982 .as_str(), 983 blk_params.as_str(), 984 ]) 985 .default_net() 986 .args(["--api-socket", &api_socket]) 987 .capture_output() 988 .spawn() 989 .unwrap(); 990 991 let r = std::panic::catch_unwind(|| { 992 guest.wait_vm_boot(None).unwrap(); 993 994 // Check both if /dev/vdc exists and if the block size is 16M. 995 assert_eq!( 996 guest 997 .ssh_command("lsblk | grep vdc | grep -c 16M") 998 .unwrap() 999 .trim() 1000 .parse::<u32>() 1001 .unwrap_or_default(), 1002 1 1003 ); 1004 1005 // Check if this block is RO or RW. 1006 assert_eq!( 1007 guest 1008 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1009 .unwrap() 1010 .trim() 1011 .parse::<u32>() 1012 .unwrap_or_default(), 1013 readonly as u32 1014 ); 1015 1016 // Check if the number of queues in /sys/block/vdc/mq matches the 1017 // expected num_queues. 1018 assert_eq!( 1019 guest 1020 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1021 .unwrap() 1022 .trim() 1023 .parse::<u32>() 1024 .unwrap_or_default(), 1025 num_queues as u32 1026 ); 1027 1028 // Mount the device 1029 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1030 guest.ssh_command("mkdir mount_image").unwrap(); 1031 guest 1032 .ssh_command( 1033 format!( 1034 "sudo mount -o {} -t ext4 /dev/vdc mount_image/", 1035 mount_ro_rw_flag 1036 ) 1037 .as_str(), 1038 ) 1039 .unwrap(); 1040 1041 // Check the content of the block device. The file "foo" should 1042 // contain "bar". 1043 assert_eq!( 1044 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1045 "bar" 1046 ); 1047 1048 // ACPI feature is needed. 1049 #[cfg(target_arch = "x86_64")] 1050 { 1051 guest.enable_memory_hotplug(); 1052 1053 // Add RAM to the VM 1054 let desired_ram = 1024 << 20; 1055 resize_command(&api_socket, None, Some(desired_ram), None, None); 1056 1057 thread::sleep(std::time::Duration::new(10, 0)); 1058 1059 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1060 1061 // Check again the content of the block device after the resize 1062 // has been performed. 1063 assert_eq!( 1064 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1065 "bar" 1066 ); 1067 } 1068 1069 // Unmount the device 1070 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1071 guest.ssh_command("rm -r mount_image").unwrap(); 1072 }); 1073 1074 let _ = child.kill(); 1075 let output = child.wait_with_output().unwrap(); 1076 1077 if let Some(mut daemon_child) = daemon_child { 1078 thread::sleep(std::time::Duration::new(5, 0)); 1079 let _ = daemon_child.kill(); 1080 let _ = daemon_child.wait(); 1081 } 1082 1083 handle_child_output(r, &output); 1084 } 1085 1086 fn test_boot_from_vhost_user_blk( 1087 num_queues: usize, 1088 readonly: bool, 1089 direct: bool, 1090 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1091 ) { 1092 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1093 let guest = Guest::new(Box::new(focal)); 1094 1095 let kernel_path = direct_kernel_boot_path(); 1096 1097 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1098 1099 let (blk_boot_params, daemon_child) = { 1100 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1101 // Start the daemon 1102 let (daemon_child, vubd_socket_path) = prepare_daemon( 1103 &guest.tmp_dir, 1104 disk_path.as_str(), 1105 num_queues, 1106 readonly, 1107 direct, 1108 ); 1109 1110 ( 1111 format!( 1112 "vhost_user=true,socket={},num_queues={},queue_size=128", 1113 vubd_socket_path, num_queues, 1114 ), 1115 Some(daemon_child), 1116 ) 1117 }; 1118 1119 let mut child = GuestCommand::new(&guest) 1120 .args(["--cpus", format!("boot={}", num_queues).as_str()]) 1121 .args(["--memory", "size=512M,shared=on"]) 1122 .args(["--kernel", kernel_path.to_str().unwrap()]) 1123 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1124 .args([ 1125 "--disk", 1126 blk_boot_params.as_str(), 1127 format!( 1128 "path={}", 1129 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1130 ) 1131 .as_str(), 1132 ]) 1133 .default_net() 1134 .capture_output() 1135 .spawn() 1136 .unwrap(); 1137 1138 let r = std::panic::catch_unwind(|| { 1139 guest.wait_vm_boot(None).unwrap(); 1140 1141 // Just check the VM booted correctly. 1142 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1143 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1144 }); 1145 let _ = child.kill(); 1146 let output = child.wait_with_output().unwrap(); 1147 1148 if let Some(mut daemon_child) = daemon_child { 1149 thread::sleep(std::time::Duration::new(5, 0)); 1150 let _ = daemon_child.kill(); 1151 let _ = daemon_child.wait(); 1152 } 1153 1154 handle_child_output(r, &output); 1155 } 1156 1157 fn _test_virtio_fs( 1158 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1159 hotplug: bool, 1160 pci_segment: Option<u16>, 1161 ) { 1162 #[cfg(target_arch = "aarch64")] 1163 let focal_image = if hotplug { 1164 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1165 } else { 1166 FOCAL_IMAGE_NAME.to_string() 1167 }; 1168 #[cfg(target_arch = "x86_64")] 1169 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1170 let focal = UbuntuDiskConfig::new(focal_image); 1171 let guest = Guest::new(Box::new(focal)); 1172 let api_socket = temp_api_path(&guest.tmp_dir); 1173 1174 let mut workload_path = dirs::home_dir().unwrap(); 1175 workload_path.push("workloads"); 1176 1177 let mut shared_dir = workload_path; 1178 shared_dir.push("shared_dir"); 1179 1180 #[cfg(target_arch = "x86_64")] 1181 let kernel_path = direct_kernel_boot_path(); 1182 #[cfg(target_arch = "aarch64")] 1183 let kernel_path = if hotplug { 1184 edk2_path() 1185 } else { 1186 direct_kernel_boot_path() 1187 }; 1188 1189 let (mut daemon_child, virtiofsd_socket_path) = 1190 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1191 1192 let mut guest_command = GuestCommand::new(&guest); 1193 guest_command 1194 .args(["--cpus", "boot=1"]) 1195 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1196 .args(["--kernel", kernel_path.to_str().unwrap()]) 1197 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1198 .default_disks() 1199 .default_net() 1200 .args(["--api-socket", &api_socket]); 1201 if pci_segment.is_some() { 1202 guest_command.args(["--platform", "num_pci_segments=16"]); 1203 } 1204 1205 let fs_params = format!( 1206 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1207 virtiofsd_socket_path, 1208 if let Some(pci_segment) = pci_segment { 1209 format!(",pci_segment={}", pci_segment) 1210 } else { 1211 "".to_owned() 1212 } 1213 ); 1214 1215 if !hotplug { 1216 guest_command.args(["--fs", fs_params.as_str()]); 1217 } 1218 1219 let mut child = guest_command.capture_output().spawn().unwrap(); 1220 1221 let r = std::panic::catch_unwind(|| { 1222 guest.wait_vm_boot(None).unwrap(); 1223 1224 if hotplug { 1225 // Add fs to the VM 1226 let (cmd_success, cmd_output) = 1227 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1228 assert!(cmd_success); 1229 1230 if let Some(pci_segment) = pci_segment { 1231 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1232 "{{\"id\":\"myfs0\",\"bdf\":\"{:04x}:00:01.0\"}}", 1233 pci_segment 1234 ))); 1235 } else { 1236 assert!(String::from_utf8_lossy(&cmd_output) 1237 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1238 } 1239 1240 thread::sleep(std::time::Duration::new(10, 0)); 1241 } 1242 1243 // Mount shared directory through virtio_fs filesystem 1244 guest 1245 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1246 .unwrap(); 1247 1248 // Check file1 exists and its content is "foo" 1249 assert_eq!( 1250 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1251 "foo" 1252 ); 1253 // Check file2 does not exist 1254 guest 1255 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1256 .unwrap(); 1257 1258 // Check file3 exists and its content is "bar" 1259 assert_eq!( 1260 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1261 "bar" 1262 ); 1263 1264 // ACPI feature is needed. 1265 #[cfg(target_arch = "x86_64")] 1266 { 1267 guest.enable_memory_hotplug(); 1268 1269 // Add RAM to the VM 1270 let desired_ram = 1024 << 20; 1271 resize_command(&api_socket, None, Some(desired_ram), None, None); 1272 1273 thread::sleep(std::time::Duration::new(30, 0)); 1274 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1275 1276 // After the resize, check again that file1 exists and its 1277 // content is "foo". 1278 assert_eq!( 1279 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1280 "foo" 1281 ); 1282 } 1283 1284 if hotplug { 1285 // Remove from VM 1286 guest.ssh_command("sudo umount mount_dir").unwrap(); 1287 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1288 } 1289 }); 1290 1291 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1292 thread::sleep(std::time::Duration::new(10, 0)); 1293 let (daemon_child, virtiofsd_socket_path) = 1294 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1295 1296 let r = std::panic::catch_unwind(|| { 1297 thread::sleep(std::time::Duration::new(10, 0)); 1298 let fs_params = format!( 1299 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1300 virtiofsd_socket_path, 1301 if let Some(pci_segment) = pci_segment { 1302 format!(",pci_segment={}", pci_segment) 1303 } else { 1304 "".to_owned() 1305 } 1306 ); 1307 1308 // Add back and check it works 1309 let (cmd_success, cmd_output) = 1310 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1311 assert!(cmd_success); 1312 if let Some(pci_segment) = pci_segment { 1313 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1314 "{{\"id\":\"myfs0\",\"bdf\":\"{:04x}:00:01.0\"}}", 1315 pci_segment 1316 ))); 1317 } else { 1318 assert!(String::from_utf8_lossy(&cmd_output) 1319 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1320 } 1321 1322 thread::sleep(std::time::Duration::new(10, 0)); 1323 // Mount shared directory through virtio_fs filesystem 1324 guest 1325 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1326 .unwrap(); 1327 1328 // Check file1 exists and its content is "foo" 1329 assert_eq!( 1330 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1331 "foo" 1332 ); 1333 }); 1334 1335 (r, Some(daemon_child)) 1336 } else { 1337 (r, None) 1338 }; 1339 1340 let _ = child.kill(); 1341 let output = child.wait_with_output().unwrap(); 1342 1343 let _ = daemon_child.kill(); 1344 let _ = daemon_child.wait(); 1345 1346 if let Some(mut daemon_child) = hotplug_daemon_child { 1347 let _ = daemon_child.kill(); 1348 let _ = daemon_child.wait(); 1349 } 1350 1351 handle_child_output(r, &output); 1352 } 1353 1354 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1355 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1356 let guest = Guest::new(Box::new(focal)); 1357 1358 let kernel_path = direct_kernel_boot_path(); 1359 1360 let pmem_temp_file = TempFile::new().unwrap(); 1361 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1362 1363 std::process::Command::new("mkfs.ext4") 1364 .arg(pmem_temp_file.as_path()) 1365 .output() 1366 .expect("Expect creating disk image to succeed"); 1367 1368 let mut child = GuestCommand::new(&guest) 1369 .args(["--cpus", "boot=1"]) 1370 .args(["--memory", "size=512M"]) 1371 .args(["--kernel", kernel_path.to_str().unwrap()]) 1372 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1373 .default_disks() 1374 .default_net() 1375 .args([ 1376 "--pmem", 1377 format!( 1378 "file={}{}{}", 1379 pmem_temp_file.as_path().to_str().unwrap(), 1380 if specify_size { ",size=128M" } else { "" }, 1381 if discard_writes { 1382 ",discard_writes=on" 1383 } else { 1384 "" 1385 } 1386 ) 1387 .as_str(), 1388 ]) 1389 .capture_output() 1390 .spawn() 1391 .unwrap(); 1392 1393 let r = std::panic::catch_unwind(|| { 1394 guest.wait_vm_boot(None).unwrap(); 1395 1396 // Check for the presence of /dev/pmem0 1397 assert_eq!( 1398 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1399 "/dev/pmem0" 1400 ); 1401 1402 // Check changes persist after reboot 1403 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1404 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1405 guest 1406 .ssh_command("echo test123 | sudo tee /mnt/test") 1407 .unwrap(); 1408 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1409 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1410 1411 guest.reboot_linux(0, None); 1412 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1413 assert_eq!( 1414 guest 1415 .ssh_command("sudo cat /mnt/test || true") 1416 .unwrap() 1417 .trim(), 1418 if discard_writes { "" } else { "test123" } 1419 ); 1420 }); 1421 1422 let _ = child.kill(); 1423 let output = child.wait_with_output().unwrap(); 1424 1425 handle_child_output(r, &output); 1426 } 1427 1428 fn get_fd_count(pid: u32) -> usize { 1429 fs::read_dir(format!("/proc/{}/fd", pid)).unwrap().count() 1430 } 1431 1432 fn _test_virtio_vsock(hotplug: bool) { 1433 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1434 let guest = Guest::new(Box::new(focal)); 1435 1436 #[cfg(target_arch = "x86_64")] 1437 let kernel_path = direct_kernel_boot_path(); 1438 #[cfg(target_arch = "aarch64")] 1439 let kernel_path = if hotplug { 1440 edk2_path() 1441 } else { 1442 direct_kernel_boot_path() 1443 }; 1444 1445 let socket = temp_vsock_path(&guest.tmp_dir); 1446 let api_socket = temp_api_path(&guest.tmp_dir); 1447 1448 let mut cmd = GuestCommand::new(&guest); 1449 cmd.args(["--api-socket", &api_socket]); 1450 cmd.args(["--cpus", "boot=1"]); 1451 cmd.args(["--memory", "size=512M"]); 1452 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1453 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1454 cmd.default_disks(); 1455 cmd.default_net(); 1456 1457 if !hotplug { 1458 cmd.args(["--vsock", format!("cid=3,socket={}", socket).as_str()]); 1459 } 1460 1461 let mut child = cmd.capture_output().spawn().unwrap(); 1462 1463 let r = std::panic::catch_unwind(|| { 1464 guest.wait_vm_boot(None).unwrap(); 1465 1466 if hotplug { 1467 let (cmd_success, cmd_output) = remote_command_w_output( 1468 &api_socket, 1469 "add-vsock", 1470 Some(format!("cid=3,socket={},id=test0", socket).as_str()), 1471 ); 1472 assert!(cmd_success); 1473 assert!(String::from_utf8_lossy(&cmd_output) 1474 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1475 thread::sleep(std::time::Duration::new(10, 0)); 1476 // Check adding a second one fails 1477 assert!(!remote_command( 1478 &api_socket, 1479 "add-vsock", 1480 Some("cid=1234,socket=/tmp/fail") 1481 )); 1482 } 1483 1484 // Validate vsock works as expected. 1485 guest.check_vsock(socket.as_str()); 1486 guest.reboot_linux(0, None); 1487 // Validate vsock still works after a reboot. 1488 guest.check_vsock(socket.as_str()); 1489 1490 if hotplug { 1491 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1492 } 1493 }); 1494 1495 let _ = child.kill(); 1496 let output = child.wait_with_output().unwrap(); 1497 1498 handle_child_output(r, &output); 1499 } 1500 1501 fn get_ksm_pages_shared() -> u32 { 1502 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1503 .unwrap() 1504 .trim() 1505 .parse::<u32>() 1506 .unwrap() 1507 } 1508 1509 fn test_memory_mergeable(mergeable: bool) { 1510 let memory_param = if mergeable { 1511 "mergeable=on" 1512 } else { 1513 "mergeable=off" 1514 }; 1515 1516 // We are assuming the rest of the system in our CI is not using mergeable memeory 1517 let ksm_ps_init = get_ksm_pages_shared(); 1518 assert!(ksm_ps_init == 0); 1519 1520 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1521 let guest1 = Guest::new(Box::new(focal1)); 1522 let mut child1 = GuestCommand::new(&guest1) 1523 .args(["--cpus", "boot=1"]) 1524 .args(["--memory", format!("size=512M,{}", memory_param).as_str()]) 1525 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1526 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1527 .default_disks() 1528 .args(["--net", guest1.default_net_string().as_str()]) 1529 .args(["--serial", "tty", "--console", "off"]) 1530 .capture_output() 1531 .spawn() 1532 .unwrap(); 1533 1534 let r = std::panic::catch_unwind(|| { 1535 guest1.wait_vm_boot(None).unwrap(); 1536 }); 1537 if r.is_err() { 1538 let _ = child1.kill(); 1539 let output = child1.wait_with_output().unwrap(); 1540 handle_child_output(r, &output); 1541 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1542 } 1543 1544 let ksm_ps_guest1 = get_ksm_pages_shared(); 1545 1546 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1547 let guest2 = Guest::new(Box::new(focal2)); 1548 let mut child2 = GuestCommand::new(&guest2) 1549 .args(["--cpus", "boot=1"]) 1550 .args(["--memory", format!("size=512M,{}", memory_param).as_str()]) 1551 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1552 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1553 .default_disks() 1554 .args(["--net", guest2.default_net_string().as_str()]) 1555 .args(["--serial", "tty", "--console", "off"]) 1556 .capture_output() 1557 .spawn() 1558 .unwrap(); 1559 1560 let r = std::panic::catch_unwind(|| { 1561 guest2.wait_vm_boot(None).unwrap(); 1562 let ksm_ps_guest2 = get_ksm_pages_shared(); 1563 1564 if mergeable { 1565 println!( 1566 "ksm pages_shared after vm1 booted '{}', ksm pages_shared after vm2 booted '{}'", 1567 ksm_ps_guest1, ksm_ps_guest2 1568 ); 1569 // We are expecting the number of shared pages to increase as the number of VM increases 1570 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1571 } else { 1572 assert!(ksm_ps_guest1 == 0); 1573 assert!(ksm_ps_guest2 == 0); 1574 } 1575 }); 1576 1577 let _ = child1.kill(); 1578 let _ = child2.kill(); 1579 1580 let output = child1.wait_with_output().unwrap(); 1581 child2.wait().unwrap(); 1582 1583 handle_child_output(r, &output); 1584 } 1585 1586 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 1587 let smaps = fs::File::open(format!("/proc/{}/smaps", pid)).unwrap(); 1588 let reader = io::BufReader::new(smaps); 1589 1590 let mut skip_map: bool = false; 1591 let mut region_name: String = "".to_string(); 1592 let mut region_maps = HashMap::new(); 1593 for line in reader.lines() { 1594 let l = line.unwrap(); 1595 1596 if l.contains('-') { 1597 let values: Vec<&str> = l.split_whitespace().collect(); 1598 region_name = values.last().unwrap().trim().to_string(); 1599 if region_name == "0" { 1600 region_name = "anonymous".to_string() 1601 } 1602 } 1603 1604 // Each section begins with something that looks like: 1605 // Size: 2184 kB 1606 if l.starts_with("Size:") { 1607 let values: Vec<&str> = l.split_whitespace().collect(); 1608 let map_size = values[1].parse::<u32>().unwrap(); 1609 // We skip the assigned guest RAM map, its RSS is only 1610 // dependent on the guest actual memory usage. 1611 // Everything else can be added to the VMM overhead. 1612 skip_map = map_size >= guest_memory_size; 1613 continue; 1614 } 1615 1616 // If this is a map we're taking into account, then we only 1617 // count the RSS. The sum of all counted RSS is the VMM overhead. 1618 if !skip_map && l.starts_with("Rss:") { 1619 let values: Vec<&str> = l.split_whitespace().collect(); 1620 let value = values[1].trim().parse::<u32>().unwrap(); 1621 *region_maps.entry(region_name.clone()).or_insert(0) += value; 1622 } 1623 } 1624 1625 region_maps 1626 } 1627 1628 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 1629 let mut total = 0; 1630 1631 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 1632 eprintln!("{}: {}", region_name, value); 1633 total += value; 1634 } 1635 1636 total 1637 } 1638 1639 fn process_rss_kib(pid: u32) -> usize { 1640 let command = format!("ps -q {} -o rss=", pid); 1641 let rss = exec_host_command_output(&command); 1642 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 1643 } 1644 1645 // 10MB is our maximum accepted overhead. 1646 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 1647 1648 #[derive(PartialEq, Eq, PartialOrd)] 1649 struct Counters { 1650 rx_bytes: u64, 1651 rx_frames: u64, 1652 tx_bytes: u64, 1653 tx_frames: u64, 1654 read_bytes: u64, 1655 write_bytes: u64, 1656 read_ops: u64, 1657 write_ops: u64, 1658 } 1659 1660 fn get_counters(api_socket: &str) -> Counters { 1661 // Get counters 1662 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 1663 assert!(cmd_success); 1664 1665 let counters: HashMap<&str, HashMap<&str, u64>> = 1666 serde_json::from_slice(&cmd_output).unwrap_or_default(); 1667 1668 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 1669 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 1670 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 1671 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 1672 1673 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 1674 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 1675 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 1676 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 1677 1678 Counters { 1679 rx_bytes, 1680 rx_frames, 1681 tx_bytes, 1682 tx_frames, 1683 read_bytes, 1684 write_bytes, 1685 read_ops, 1686 write_ops, 1687 } 1688 } 1689 1690 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 1691 let (tx, rx) = mpsc::channel::<String>(); 1692 thread::spawn(move || loop { 1693 thread::sleep(std::time::Duration::new(1, 0)); 1694 let mut buf = [0; 512]; 1695 match pty.read(&mut buf) { 1696 Ok(_) => { 1697 let output = std::str::from_utf8(&buf).unwrap().to_string(); 1698 match tx.send(output) { 1699 Ok(_) => (), 1700 Err(_) => break, 1701 } 1702 } 1703 Err(_) => break, 1704 } 1705 }); 1706 rx 1707 } 1708 1709 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 1710 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1711 assert!(cmd_success); 1712 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1713 assert_eq!("Pty", info["config"][pty_type]["mode"]); 1714 PathBuf::from( 1715 info["config"][pty_type]["file"] 1716 .as_str() 1717 .expect("Missing pty path"), 1718 ) 1719 } 1720 1721 // VFIO test network setup. 1722 // We reserve a different IP class for it: 172.18.0.0/24. 1723 #[cfg(target_arch = "x86_64")] 1724 fn setup_vfio_network_interfaces() { 1725 // 'vfio-br0' 1726 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 1727 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 1728 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 1729 // 'vfio-tap0' 1730 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 1731 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 1732 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 1733 // 'vfio-tap1' 1734 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 1735 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 1736 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 1737 // 'vfio-tap2' 1738 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 1739 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 1740 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 1741 // 'vfio-tap3' 1742 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 1743 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 1744 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 1745 } 1746 1747 // Tear VFIO test network down 1748 #[cfg(target_arch = "x86_64")] 1749 fn cleanup_vfio_network_interfaces() { 1750 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 1751 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 1752 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 1753 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 1754 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 1755 } 1756 1757 fn balloon_size(api_socket: &str) -> u64 { 1758 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1759 assert!(cmd_success); 1760 1761 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1762 let total_mem = &info["config"]["memory"]["size"] 1763 .to_string() 1764 .parse::<u64>() 1765 .unwrap(); 1766 let actual_mem = &info["memory_actual_size"] 1767 .to_string() 1768 .parse::<u64>() 1769 .unwrap(); 1770 total_mem - actual_mem 1771 } 1772 1773 // This test validates that it can find the virtio-iommu device at first. 1774 // It also verifies that both disks and the network card are attached to 1775 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 1776 // The last interesting part of this test is that it exercises the network 1777 // interface attached to the virtual IOMMU since this is the one used to 1778 // send all commands through SSH. 1779 fn _test_virtio_iommu(acpi: bool) { 1780 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 1781 // Focal image is still old. 1782 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 1783 // the kernel binary has been updated. 1784 #[cfg(target_arch = "aarch64")] 1785 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 1786 #[cfg(target_arch = "x86_64")] 1787 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1788 let focal = UbuntuDiskConfig::new(focal_image); 1789 let guest = Guest::new(Box::new(focal)); 1790 1791 #[cfg(target_arch = "x86_64")] 1792 let kernel_path = direct_kernel_boot_path(); 1793 #[cfg(target_arch = "aarch64")] 1794 let kernel_path = if acpi { 1795 edk2_path() 1796 } else { 1797 direct_kernel_boot_path() 1798 }; 1799 1800 let mut child = GuestCommand::new(&guest) 1801 .args(["--cpus", "boot=1"]) 1802 .args(["--memory", "size=512M"]) 1803 .args(["--kernel", kernel_path.to_str().unwrap()]) 1804 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1805 .args([ 1806 "--disk", 1807 format!( 1808 "path={},iommu=on", 1809 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1810 ) 1811 .as_str(), 1812 format!( 1813 "path={},iommu=on", 1814 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1815 ) 1816 .as_str(), 1817 ]) 1818 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 1819 .capture_output() 1820 .spawn() 1821 .unwrap(); 1822 1823 let r = std::panic::catch_unwind(|| { 1824 guest.wait_vm_boot(None).unwrap(); 1825 1826 // Verify the virtio-iommu device is present. 1827 assert!(guest 1828 .does_device_vendor_pair_match("0x1057", "0x1af4") 1829 .unwrap_or_default()); 1830 1831 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 1832 // different with ACPI. 1833 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 1834 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 1835 // and appear under folder '/sys/kernel/iommu_groups/'. 1836 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 1837 // which is the console. The first disk "0000:00:02.0" is in group '1'. 1838 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 1839 // contains "0000:00:02.0" which is the first disk. 1840 // 1841 // Verify the iommu group of the first disk. 1842 let iommu_group = if acpi { 0 } else { 1 }; 1843 assert_eq!( 1844 guest 1845 .ssh_command( 1846 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1847 ) 1848 .unwrap() 1849 .trim(), 1850 "0000:00:02.0" 1851 ); 1852 1853 // Verify the iommu group of the second disk. 1854 let iommu_group = if acpi { 1 } else { 2 }; 1855 assert_eq!( 1856 guest 1857 .ssh_command( 1858 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1859 ) 1860 .unwrap() 1861 .trim(), 1862 "0000:00:03.0" 1863 ); 1864 1865 // Verify the iommu group of the network card. 1866 let iommu_group = if acpi { 2 } else { 3 }; 1867 assert_eq!( 1868 guest 1869 .ssh_command( 1870 format!("ls /sys/kernel/iommu_groups/{}/devices", iommu_group).as_str() 1871 ) 1872 .unwrap() 1873 .trim(), 1874 "0000:00:04.0" 1875 ); 1876 }); 1877 1878 let _ = child.kill(); 1879 let output = child.wait_with_output().unwrap(); 1880 1881 handle_child_output(r, &output); 1882 } 1883 1884 fn get_reboot_count(guest: &Guest) -> u32 { 1885 // Ensure that the current boot journal is written so reboot counts are valid 1886 guest.ssh_command("sudo journalctl --sync").unwrap(); 1887 1888 guest 1889 .ssh_command("sudo journalctl --list-boots | wc -l") 1890 .unwrap() 1891 .trim() 1892 .parse::<u32>() 1893 .unwrap_or_default() 1894 } 1895 1896 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 1897 // Check for PCI device 1898 assert!(guest 1899 .does_device_vendor_pair_match("0x1063", "0x1af4") 1900 .unwrap_or_default()); 1901 1902 // Enable systemd watchdog 1903 guest 1904 .ssh_command(&format!( 1905 "echo RuntimeWatchdogSec={}s | sudo tee -a /etc/systemd/system.conf", 1906 watchdog_sec 1907 )) 1908 .unwrap(); 1909 } 1910 1911 mod common_parallel { 1912 use std::{fs::OpenOptions, io::SeekFrom}; 1913 1914 use crate::*; 1915 1916 #[test] 1917 #[cfg(target_arch = "x86_64")] 1918 fn test_bionic_hypervisor_fw() { 1919 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), BIONIC_IMAGE_NAME) 1920 } 1921 1922 #[test] 1923 #[cfg(target_arch = "x86_64")] 1924 fn test_focal_hypervisor_fw() { 1925 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 1926 } 1927 1928 #[test] 1929 #[cfg(target_arch = "x86_64")] 1930 fn test_bionic_ovmf() { 1931 test_simple_launch(fw_path(FwType::Ovmf), BIONIC_IMAGE_NAME) 1932 } 1933 1934 #[test] 1935 #[cfg(target_arch = "x86_64")] 1936 fn test_focal_ovmf() { 1937 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 1938 } 1939 1940 #[cfg(target_arch = "x86_64")] 1941 fn test_simple_launch(fw_path: String, disk_path: &str) { 1942 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 1943 let guest = Guest::new(disk_config); 1944 let event_path = temp_event_monitor_path(&guest.tmp_dir); 1945 1946 let mut child = GuestCommand::new(&guest) 1947 .args(["--cpus", "boot=1"]) 1948 .args(["--memory", "size=512M"]) 1949 .args(["--kernel", fw_path.as_str()]) 1950 .default_disks() 1951 .default_net() 1952 .args(["--serial", "tty", "--console", "off"]) 1953 .args(["--event-monitor", format!("path={}", event_path).as_str()]) 1954 .capture_output() 1955 .spawn() 1956 .unwrap(); 1957 1958 let r = std::panic::catch_unwind(|| { 1959 guest.wait_vm_boot(Some(120)).unwrap(); 1960 1961 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 1962 assert_eq!(guest.get_initial_apicid().unwrap_or(1), 0); 1963 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1964 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 1965 1966 let expected_sequential_events = [ 1967 &MetaEvent { 1968 event: "starting".to_string(), 1969 device_id: None, 1970 }, 1971 &MetaEvent { 1972 event: "booting".to_string(), 1973 device_id: None, 1974 }, 1975 &MetaEvent { 1976 event: "booted".to_string(), 1977 device_id: None, 1978 }, 1979 &MetaEvent { 1980 event: "activated".to_string(), 1981 device_id: Some("_disk0".to_string()), 1982 }, 1983 &MetaEvent { 1984 event: "reset".to_string(), 1985 device_id: Some("_disk0".to_string()), 1986 }, 1987 ]; 1988 assert!(check_sequential_events( 1989 &expected_sequential_events, 1990 &event_path 1991 )); 1992 1993 // It's been observed on the Bionic image that udev and snapd 1994 // services can cause some delay in the VM's shutdown. Disabling 1995 // them improves the reliability of this test. 1996 let _ = guest.ssh_command("sudo systemctl disable udev"); 1997 let _ = guest.ssh_command("sudo systemctl stop udev"); 1998 let _ = guest.ssh_command("sudo systemctl disable snapd"); 1999 let _ = guest.ssh_command("sudo systemctl stop snapd"); 2000 2001 guest.ssh_command("sudo poweroff").unwrap(); 2002 thread::sleep(std::time::Duration::new(10, 0)); 2003 let latest_events = [ 2004 &MetaEvent { 2005 event: "shutdown".to_string(), 2006 device_id: None, 2007 }, 2008 &MetaEvent { 2009 event: "deleted".to_string(), 2010 device_id: None, 2011 }, 2012 &MetaEvent { 2013 event: "shutdown".to_string(), 2014 device_id: None, 2015 }, 2016 ]; 2017 assert!(check_latest_events_exact(&latest_events, &event_path)); 2018 }); 2019 2020 let _ = child.kill(); 2021 let output = child.wait_with_output().unwrap(); 2022 2023 handle_child_output(r, &output); 2024 } 2025 2026 #[test] 2027 fn test_multi_cpu() { 2028 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 2029 let guest = Guest::new(Box::new(bionic)); 2030 let mut cmd = GuestCommand::new(&guest); 2031 cmd.args(["--cpus", "boot=2,max=4"]) 2032 .args(["--memory", "size=512M"]) 2033 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2034 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2035 .capture_output() 2036 .default_disks() 2037 .default_net(); 2038 2039 let mut child = cmd.spawn().unwrap(); 2040 2041 let r = std::panic::catch_unwind(|| { 2042 guest.wait_vm_boot(Some(120)).unwrap(); 2043 2044 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2045 2046 #[cfg(target_arch = "x86_64")] 2047 assert_eq!( 2048 guest 2049 .ssh_command(r#"dmesg | grep "smpboot: Allowing" | sed "s/\[\ *[0-9.]*\] //""#) 2050 .unwrap() 2051 .trim(), 2052 "smpboot: Allowing 4 CPUs, 2 hotplug CPUs" 2053 ); 2054 #[cfg(target_arch = "aarch64")] 2055 assert_eq!( 2056 guest 2057 .ssh_command(r#"dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""#) 2058 .unwrap() 2059 .trim(), 2060 "smp: Brought up 1 node, 2 CPUs" 2061 ); 2062 }); 2063 2064 let _ = child.kill(); 2065 let output = child.wait_with_output().unwrap(); 2066 2067 handle_child_output(r, &output); 2068 } 2069 2070 #[test] 2071 #[cfg(not(feature = "mshv"))] 2072 fn test_cpu_topology_421() { 2073 test_cpu_topology(4, 2, 1, false); 2074 } 2075 2076 #[test] 2077 #[cfg(not(feature = "mshv"))] 2078 fn test_cpu_topology_142() { 2079 test_cpu_topology(1, 4, 2, false); 2080 } 2081 2082 #[test] 2083 #[cfg(not(feature = "mshv"))] 2084 fn test_cpu_topology_262() { 2085 test_cpu_topology(2, 6, 2, false); 2086 } 2087 2088 #[test] 2089 #[cfg(target_arch = "x86_64")] 2090 #[cfg(not(feature = "mshv"))] 2091 fn test_cpu_physical_bits() { 2092 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2093 let guest = Guest::new(Box::new(focal)); 2094 let max_phys_bits: u8 = 36; 2095 let mut child = GuestCommand::new(&guest) 2096 .args(["--cpus", &format!("max_phys_bits={}", max_phys_bits)]) 2097 .args(["--memory", "size=512M"]) 2098 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2099 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2100 .default_disks() 2101 .default_net() 2102 .capture_output() 2103 .spawn() 2104 .unwrap(); 2105 2106 let r = std::panic::catch_unwind(|| { 2107 guest.wait_vm_boot(None).unwrap(); 2108 2109 assert!( 2110 guest 2111 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2112 .unwrap() 2113 .trim() 2114 .parse::<u8>() 2115 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2116 ); 2117 }); 2118 2119 let _ = child.kill(); 2120 let output = child.wait_with_output().unwrap(); 2121 2122 handle_child_output(r, &output); 2123 } 2124 2125 #[test] 2126 fn test_cpu_affinity() { 2127 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2128 let guest = Guest::new(Box::new(focal)); 2129 2130 // We need the host to have at least 4 CPUs if we want to be able 2131 // to run this test. 2132 let host_cpus_count = exec_host_command_output("nproc"); 2133 assert!( 2134 String::from_utf8_lossy(&host_cpus_count.stdout) 2135 .trim() 2136 .parse::<u8>() 2137 .unwrap_or(0) 2138 >= 4 2139 ); 2140 2141 let mut child = GuestCommand::new(&guest) 2142 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2143 .args(["--memory", "size=512M"]) 2144 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2145 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2146 .default_disks() 2147 .default_net() 2148 .capture_output() 2149 .spawn() 2150 .unwrap(); 2151 2152 let r = std::panic::catch_unwind(|| { 2153 guest.wait_vm_boot(None).unwrap(); 2154 let pid = child.id(); 2155 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"", pid).as_str()); 2156 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2157 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"", pid).as_str()); 2158 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2159 }); 2160 2161 let _ = child.kill(); 2162 let output = child.wait_with_output().unwrap(); 2163 2164 handle_child_output(r, &output); 2165 } 2166 2167 #[test] 2168 #[cfg(not(feature = "mshv"))] 2169 fn test_large_vm() { 2170 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2171 let guest = Guest::new(Box::new(focal)); 2172 let mut cmd = GuestCommand::new(&guest); 2173 cmd.args(["--cpus", "boot=48"]) 2174 .args(["--memory", "size=5120M"]) 2175 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2176 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2177 .args(["--serial", "tty"]) 2178 .args(["--console", "off"]) 2179 .capture_output() 2180 .default_disks() 2181 .default_net(); 2182 2183 let mut child = cmd.spawn().unwrap(); 2184 2185 guest.wait_vm_boot(None).unwrap(); 2186 2187 let r = std::panic::catch_unwind(|| { 2188 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2189 assert_eq!( 2190 guest 2191 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2192 .unwrap() 2193 .trim(), 2194 "0-47" 2195 ); 2196 2197 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2198 }); 2199 2200 let _ = child.kill(); 2201 let output = child.wait_with_output().unwrap(); 2202 2203 handle_child_output(r, &output); 2204 } 2205 2206 #[test] 2207 #[cfg(not(feature = "mshv"))] 2208 fn test_huge_memory() { 2209 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2210 let guest = Guest::new(Box::new(focal)); 2211 let mut cmd = GuestCommand::new(&guest); 2212 cmd.args(["--cpus", "boot=1"]) 2213 .args(["--memory", "size=128G"]) 2214 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2215 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2216 .capture_output() 2217 .default_disks() 2218 .default_net(); 2219 2220 let mut child = cmd.spawn().unwrap(); 2221 2222 guest.wait_vm_boot(Some(120)).unwrap(); 2223 2224 let r = std::panic::catch_unwind(|| { 2225 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2226 }); 2227 2228 let _ = child.kill(); 2229 let output = child.wait_with_output().unwrap(); 2230 2231 handle_child_output(r, &output); 2232 } 2233 2234 #[test] 2235 fn test_power_button() { 2236 _test_power_button(false); 2237 } 2238 2239 #[test] 2240 #[cfg(not(feature = "mshv"))] 2241 fn test_user_defined_memory_regions() { 2242 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2243 let guest = Guest::new(Box::new(focal)); 2244 let api_socket = temp_api_path(&guest.tmp_dir); 2245 2246 let kernel_path = direct_kernel_boot_path(); 2247 2248 let mut child = GuestCommand::new(&guest) 2249 .args(["--cpus", "boot=1"]) 2250 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2251 .args([ 2252 "--memory-zone", 2253 "id=mem0,size=1G,hotplug_size=2G", 2254 "id=mem1,size=1G,file=/dev/shm", 2255 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2256 ]) 2257 .args(["--kernel", kernel_path.to_str().unwrap()]) 2258 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2259 .args(["--api-socket", &api_socket]) 2260 .capture_output() 2261 .default_disks() 2262 .default_net() 2263 .spawn() 2264 .unwrap(); 2265 2266 let r = std::panic::catch_unwind(|| { 2267 guest.wait_vm_boot(None).unwrap(); 2268 2269 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2270 2271 guest.enable_memory_hotplug(); 2272 2273 resize_zone_command(&api_socket, "mem0", "3G"); 2274 thread::sleep(std::time::Duration::new(5, 0)); 2275 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2276 resize_zone_command(&api_socket, "mem2", "3G"); 2277 thread::sleep(std::time::Duration::new(5, 0)); 2278 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2279 resize_zone_command(&api_socket, "mem0", "2G"); 2280 thread::sleep(std::time::Duration::new(5, 0)); 2281 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2282 resize_zone_command(&api_socket, "mem2", "2G"); 2283 thread::sleep(std::time::Duration::new(5, 0)); 2284 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2285 2286 guest.reboot_linux(0, None); 2287 2288 // Check the amount of RAM after reboot 2289 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2290 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2291 2292 // Check if we can still resize down to the initial 'boot'size 2293 resize_zone_command(&api_socket, "mem0", "1G"); 2294 thread::sleep(std::time::Duration::new(5, 0)); 2295 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2296 resize_zone_command(&api_socket, "mem2", "1G"); 2297 thread::sleep(std::time::Duration::new(5, 0)); 2298 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2299 }); 2300 2301 let _ = child.kill(); 2302 let output = child.wait_with_output().unwrap(); 2303 2304 handle_child_output(r, &output); 2305 } 2306 2307 #[test] 2308 #[cfg(not(feature = "mshv"))] 2309 fn test_guest_numa_nodes() { 2310 _test_guest_numa_nodes(false); 2311 } 2312 2313 #[test] 2314 #[cfg(target_arch = "x86_64")] 2315 fn test_iommu_segments() { 2316 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2317 let focal = UbuntuDiskConfig::new(focal_image); 2318 let guest = Guest::new(Box::new(focal)); 2319 2320 // Prepare another disk file for the virtio-disk device 2321 let test_disk_path = String::from( 2322 guest 2323 .tmp_dir 2324 .as_path() 2325 .join("test-disk.raw") 2326 .to_str() 2327 .unwrap(), 2328 ); 2329 assert!( 2330 exec_host_command_status(format!("truncate {} -s 4M", test_disk_path).as_str()) 2331 .success() 2332 ); 2333 assert!( 2334 exec_host_command_status(format!("mkfs.ext4 {}", test_disk_path).as_str()).success() 2335 ); 2336 2337 let api_socket = temp_api_path(&guest.tmp_dir); 2338 let mut cmd = GuestCommand::new(&guest); 2339 2340 cmd.args(["--cpus", "boot=1"]) 2341 .args(["--api-socket", &api_socket]) 2342 .args(["--memory", "size=512M"]) 2343 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2344 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2345 .args(["--platform", "num_pci_segments=16,iommu_segments=[1]"]) 2346 .default_disks() 2347 .capture_output() 2348 .default_net(); 2349 2350 let mut child = cmd.spawn().unwrap(); 2351 2352 guest.wait_vm_boot(None).unwrap(); 2353 2354 let r = std::panic::catch_unwind(|| { 2355 let (cmd_success, cmd_output) = remote_command_w_output( 2356 &api_socket, 2357 "add-disk", 2358 Some( 2359 format!( 2360 "path={},id=test0,pci_segment=1,iommu=on", 2361 test_disk_path.as_str() 2362 ) 2363 .as_str(), 2364 ), 2365 ); 2366 assert!(cmd_success); 2367 assert!(String::from_utf8_lossy(&cmd_output) 2368 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2369 2370 // Check IOMMU setup 2371 assert!(guest 2372 .does_device_vendor_pair_match("0x1057", "0x1af4") 2373 .unwrap_or_default()); 2374 assert_eq!( 2375 guest 2376 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2377 .unwrap() 2378 .trim(), 2379 "0001:00:01.0" 2380 ); 2381 }); 2382 2383 let _ = child.kill(); 2384 let output = child.wait_with_output().unwrap(); 2385 2386 handle_child_output(r, &output); 2387 } 2388 2389 #[test] 2390 fn test_pci_msi() { 2391 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2392 let guest = Guest::new(Box::new(focal)); 2393 let mut cmd = GuestCommand::new(&guest); 2394 cmd.args(["--cpus", "boot=1"]) 2395 .args(["--memory", "size=512M"]) 2396 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2397 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2398 .capture_output() 2399 .default_disks() 2400 .default_net(); 2401 2402 let mut child = cmd.spawn().unwrap(); 2403 2404 guest.wait_vm_boot(None).unwrap(); 2405 2406 #[cfg(target_arch = "x86_64")] 2407 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2408 #[cfg(target_arch = "aarch64")] 2409 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2410 2411 let r = std::panic::catch_unwind(|| { 2412 assert_eq!( 2413 guest 2414 .ssh_command(grep_cmd) 2415 .unwrap() 2416 .trim() 2417 .parse::<u32>() 2418 .unwrap_or_default(), 2419 12 2420 ); 2421 }); 2422 2423 let _ = child.kill(); 2424 let output = child.wait_with_output().unwrap(); 2425 2426 handle_child_output(r, &output); 2427 } 2428 2429 #[test] 2430 fn test_virtio_net_ctrl_queue() { 2431 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2432 let guest = Guest::new(Box::new(focal)); 2433 let mut cmd = GuestCommand::new(&guest); 2434 cmd.args(["--cpus", "boot=1"]) 2435 .args(["--memory", "size=512M"]) 2436 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2437 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2438 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2439 .capture_output() 2440 .default_disks(); 2441 2442 let mut child = cmd.spawn().unwrap(); 2443 2444 guest.wait_vm_boot(None).unwrap(); 2445 2446 #[cfg(target_arch = "aarch64")] 2447 let iface = "enp0s4"; 2448 #[cfg(target_arch = "x86_64")] 2449 let iface = "ens4"; 2450 2451 let r = std::panic::catch_unwind(|| { 2452 assert_eq!( 2453 guest 2454 .ssh_command( 2455 format!("sudo ethtool -K {} rx-gro-hw off && echo success", iface).as_str() 2456 ) 2457 .unwrap() 2458 .trim(), 2459 "success" 2460 ); 2461 assert_eq!( 2462 guest 2463 .ssh_command(format!("cat /sys/class/net/{}/mtu", iface).as_str()) 2464 .unwrap() 2465 .trim(), 2466 "3000" 2467 ); 2468 }); 2469 2470 let _ = child.kill(); 2471 let output = child.wait_with_output().unwrap(); 2472 2473 handle_child_output(r, &output); 2474 } 2475 2476 #[test] 2477 #[cfg(not(feature = "mshv"))] 2478 fn test_pci_multiple_segments() { 2479 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2480 let guest = Guest::new(Box::new(focal)); 2481 2482 // Prepare another disk file for the virtio-disk device 2483 let test_disk_path = String::from( 2484 guest 2485 .tmp_dir 2486 .as_path() 2487 .join("test-disk.raw") 2488 .to_str() 2489 .unwrap(), 2490 ); 2491 assert!( 2492 exec_host_command_status(format!("truncate {} -s 4M", test_disk_path).as_str()) 2493 .success() 2494 ); 2495 assert!( 2496 exec_host_command_status(format!("mkfs.ext4 {}", test_disk_path).as_str()).success() 2497 ); 2498 2499 let mut cmd = GuestCommand::new(&guest); 2500 cmd.args(["--cpus", "boot=1"]) 2501 .args(["--memory", "size=512M"]) 2502 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2503 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2504 .args(["--platform", "num_pci_segments=16"]) 2505 .args([ 2506 "--disk", 2507 format!( 2508 "path={}", 2509 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2510 ) 2511 .as_str(), 2512 format!( 2513 "path={}", 2514 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2515 ) 2516 .as_str(), 2517 format!("path={},pci_segment=15", test_disk_path).as_str(), 2518 ]) 2519 .capture_output() 2520 .default_net(); 2521 2522 let mut child = cmd.spawn().unwrap(); 2523 2524 guest.wait_vm_boot(None).unwrap(); 2525 2526 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2527 2528 let r = std::panic::catch_unwind(|| { 2529 // There should be 16 PCI host bridges in the guest. 2530 assert_eq!( 2531 guest 2532 .ssh_command(grep_cmd) 2533 .unwrap() 2534 .trim() 2535 .parse::<u32>() 2536 .unwrap_or_default(), 2537 16 2538 ); 2539 2540 // Check both if /dev/vdc exists and if the block size is 4M. 2541 assert_eq!( 2542 guest 2543 .ssh_command("lsblk | grep vdc | grep -c 4M") 2544 .unwrap() 2545 .trim() 2546 .parse::<u32>() 2547 .unwrap_or_default(), 2548 1 2549 ); 2550 2551 // Mount the device. 2552 guest.ssh_command("mkdir mount_image").unwrap(); 2553 guest 2554 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 2555 .unwrap(); 2556 // Grant all users with write permission. 2557 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 2558 2559 // Write something to the device. 2560 guest 2561 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 2562 .unwrap(); 2563 2564 // Check the content of the block device. The file "foo" should 2565 // contain "bar". 2566 assert_eq!( 2567 guest 2568 .ssh_command("sudo cat mount_image/foo") 2569 .unwrap() 2570 .trim(), 2571 "bar" 2572 ); 2573 }); 2574 2575 let _ = child.kill(); 2576 let output = child.wait_with_output().unwrap(); 2577 2578 handle_child_output(r, &output); 2579 } 2580 2581 #[test] 2582 fn test_direct_kernel_boot() { 2583 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2584 let guest = Guest::new(Box::new(focal)); 2585 2586 let kernel_path = direct_kernel_boot_path(); 2587 2588 let mut child = GuestCommand::new(&guest) 2589 .args(["--cpus", "boot=1"]) 2590 .args(["--memory", "size=512M"]) 2591 .args(["--kernel", kernel_path.to_str().unwrap()]) 2592 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2593 .default_disks() 2594 .default_net() 2595 .capture_output() 2596 .spawn() 2597 .unwrap(); 2598 2599 let r = std::panic::catch_unwind(|| { 2600 guest.wait_vm_boot(None).unwrap(); 2601 2602 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2603 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2604 2605 let grep_cmd = if cfg!(target_arch = "x86_64") { 2606 "grep -c PCI-MSI /proc/interrupts" 2607 } else { 2608 "grep -c ITS-MSI /proc/interrupts" 2609 }; 2610 assert_eq!( 2611 guest 2612 .ssh_command(grep_cmd) 2613 .unwrap() 2614 .trim() 2615 .parse::<u32>() 2616 .unwrap_or_default(), 2617 12 2618 ); 2619 }); 2620 2621 let _ = child.kill(); 2622 let output = child.wait_with_output().unwrap(); 2623 2624 handle_child_output(r, &output); 2625 } 2626 2627 fn _test_virtio_block(image_name: &str, disable_io_uring: bool) { 2628 let focal = UbuntuDiskConfig::new(image_name.to_string()); 2629 let guest = Guest::new(Box::new(focal)); 2630 2631 let mut workload_path = dirs::home_dir().unwrap(); 2632 workload_path.push("workloads"); 2633 2634 let mut blk_file_path = workload_path; 2635 blk_file_path.push("blk.img"); 2636 2637 let kernel_path = direct_kernel_boot_path(); 2638 2639 let mut cloud_child = GuestCommand::new(&guest) 2640 .args(["--cpus", "boot=4"]) 2641 .args(["--memory", "size=512M,shared=on"]) 2642 .args(["--kernel", kernel_path.to_str().unwrap()]) 2643 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2644 .args([ 2645 "--disk", 2646 format!( 2647 "path={}", 2648 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2649 ) 2650 .as_str(), 2651 format!( 2652 "path={}", 2653 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2654 ) 2655 .as_str(), 2656 format!( 2657 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={}", 2658 blk_file_path.to_str().unwrap(), 2659 disable_io_uring 2660 ) 2661 .as_str(), 2662 ]) 2663 .default_net() 2664 .capture_output() 2665 .spawn() 2666 .unwrap(); 2667 2668 let r = std::panic::catch_unwind(|| { 2669 guest.wait_vm_boot(None).unwrap(); 2670 2671 // Check both if /dev/vdc exists and if the block size is 16M. 2672 assert_eq!( 2673 guest 2674 .ssh_command("lsblk | grep vdc | grep -c 16M") 2675 .unwrap() 2676 .trim() 2677 .parse::<u32>() 2678 .unwrap_or_default(), 2679 1 2680 ); 2681 2682 // Check both if /dev/vdc exists and if this block is RO. 2683 assert_eq!( 2684 guest 2685 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 2686 .unwrap() 2687 .trim() 2688 .parse::<u32>() 2689 .unwrap_or_default(), 2690 1 2691 ); 2692 2693 // Check if the number of queues is 4. 2694 assert_eq!( 2695 guest 2696 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 2697 .unwrap() 2698 .trim() 2699 .parse::<u32>() 2700 .unwrap_or_default(), 2701 4 2702 ); 2703 }); 2704 2705 let _ = cloud_child.kill(); 2706 let output = cloud_child.wait_with_output().unwrap(); 2707 2708 handle_child_output(r, &output); 2709 } 2710 2711 #[test] 2712 fn test_virtio_block() { 2713 _test_virtio_block(FOCAL_IMAGE_NAME, false) 2714 } 2715 2716 #[test] 2717 fn test_virtio_block_disable_io_uring() { 2718 _test_virtio_block(FOCAL_IMAGE_NAME, true) 2719 } 2720 2721 #[test] 2722 fn test_virtio_block_qcow2() { 2723 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false) 2724 } 2725 2726 #[test] 2727 fn test_virtio_block_vhd() { 2728 let mut workload_path = dirs::home_dir().unwrap(); 2729 workload_path.push("workloads"); 2730 2731 let mut raw_file_path = workload_path.clone(); 2732 let mut vhd_file_path = workload_path; 2733 raw_file_path.push(FOCAL_IMAGE_NAME); 2734 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 2735 2736 // Generate VHD file from RAW file 2737 std::process::Command::new("qemu-img") 2738 .arg("convert") 2739 .arg("-p") 2740 .args(["-f", "raw"]) 2741 .args(["-O", "vpc"]) 2742 .args(["-o", "subformat=fixed"]) 2743 .arg(raw_file_path.to_str().unwrap()) 2744 .arg(vhd_file_path.to_str().unwrap()) 2745 .output() 2746 .expect("Expect generating VHD image from RAW image"); 2747 2748 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false) 2749 } 2750 2751 #[test] 2752 fn test_virtio_block_vhdx() { 2753 let mut workload_path = dirs::home_dir().unwrap(); 2754 workload_path.push("workloads"); 2755 2756 let mut raw_file_path = workload_path.clone(); 2757 let mut vhdx_file_path = workload_path; 2758 raw_file_path.push(FOCAL_IMAGE_NAME); 2759 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 2760 2761 // Generate dynamic VHDX file from RAW file 2762 std::process::Command::new("qemu-img") 2763 .arg("convert") 2764 .arg("-p") 2765 .args(["-f", "raw"]) 2766 .args(["-O", "vhdx"]) 2767 .arg(raw_file_path.to_str().unwrap()) 2768 .arg(vhdx_file_path.to_str().unwrap()) 2769 .output() 2770 .expect("Expect generating dynamic VHDx image from RAW image"); 2771 2772 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false) 2773 } 2774 2775 #[test] 2776 fn test_virtio_block_dynamic_vhdx_expand() { 2777 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 2778 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 2779 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 2780 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 2781 2782 let mut workload_path = dirs::home_dir().unwrap(); 2783 workload_path.push("workloads"); 2784 2785 let mut vhdx_file_path = workload_path; 2786 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 2787 let vhdx_path = vhdx_file_path.to_str().unwrap(); 2788 2789 // Generate a 100 MiB dynamic VHDX file 2790 std::process::Command::new("qemu-img") 2791 .arg("create") 2792 .args(["-f", "vhdx"]) 2793 .arg(vhdx_path) 2794 .arg(VIRTUAL_DISK_SIZE.to_string()) 2795 .output() 2796 .expect("Expect generating dynamic VHDx image from RAW image"); 2797 2798 // Check if the size matches with empty VHDx file size 2799 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 2800 2801 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2802 let guest = Guest::new(Box::new(focal)); 2803 let kernel_path = direct_kernel_boot_path(); 2804 2805 let mut cloud_child = GuestCommand::new(&guest) 2806 .args(["--cpus", "boot=1"]) 2807 .args(["--memory", "size=512M"]) 2808 .args(["--kernel", kernel_path.to_str().unwrap()]) 2809 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2810 .args([ 2811 "--disk", 2812 format!( 2813 "path={}", 2814 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2815 ) 2816 .as_str(), 2817 format!( 2818 "path={}", 2819 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2820 ) 2821 .as_str(), 2822 format!("path={}", vhdx_path).as_str(), 2823 ]) 2824 .default_net() 2825 .capture_output() 2826 .spawn() 2827 .unwrap(); 2828 2829 let r = std::panic::catch_unwind(|| { 2830 guest.wait_vm_boot(None).unwrap(); 2831 2832 // Check both if /dev/vdc exists and if the block size is 100 MiB. 2833 assert_eq!( 2834 guest 2835 .ssh_command("lsblk | grep vdc | grep -c 100M") 2836 .unwrap() 2837 .trim() 2838 .parse::<u32>() 2839 .unwrap_or_default(), 2840 1 2841 ); 2842 2843 // Write 100 MB of data to the VHDx disk 2844 guest 2845 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 2846 .unwrap(); 2847 }); 2848 2849 // Check if the size matches with expected expanded VHDx file size 2850 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 2851 2852 let _ = cloud_child.kill(); 2853 let output = cloud_child.wait_with_output().unwrap(); 2854 2855 handle_child_output(r, &output); 2856 } 2857 2858 fn vhdx_image_size(disk_name: &str) -> u64 { 2859 std::fs::File::open(disk_name) 2860 .unwrap() 2861 .seek(SeekFrom::End(0)) 2862 .unwrap() 2863 } 2864 2865 #[test] 2866 fn test_virtio_block_direct_and_firmware() { 2867 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2868 let guest = Guest::new(Box::new(focal)); 2869 2870 // The OS disk must be copied to a location that is not backed by 2871 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 2872 // with EINVAL because tmpfs doesn't support this flag. 2873 let mut workloads_path = dirs::home_dir().unwrap(); 2874 workloads_path.push("workloads"); 2875 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 2876 let mut os_path = os_dir.as_path().to_path_buf(); 2877 os_path.push("osdisk.img"); 2878 rate_limited_copy( 2879 &guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 2880 os_path.as_path(), 2881 ) 2882 .expect("copying of OS disk failed"); 2883 2884 let mut child = GuestCommand::new(&guest) 2885 .args(["--cpus", "boot=1"]) 2886 .args(["--memory", "size=512M"]) 2887 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 2888 .args([ 2889 "--disk", 2890 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 2891 format!( 2892 "path={}", 2893 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2894 ) 2895 .as_str(), 2896 ]) 2897 .default_net() 2898 .capture_output() 2899 .spawn() 2900 .unwrap(); 2901 2902 let r = std::panic::catch_unwind(|| { 2903 guest.wait_vm_boot(Some(120)).unwrap(); 2904 }); 2905 2906 let _ = child.kill(); 2907 let output = child.wait_with_output().unwrap(); 2908 2909 handle_child_output(r, &output); 2910 } 2911 2912 #[test] 2913 fn test_vhost_user_net_default() { 2914 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 2915 } 2916 2917 #[test] 2918 fn test_vhost_user_net_named_tap() { 2919 test_vhost_user_net( 2920 Some("mytap0"), 2921 2, 2922 &prepare_vhost_user_net_daemon, 2923 false, 2924 false, 2925 ) 2926 } 2927 2928 #[test] 2929 fn test_vhost_user_net_existing_tap() { 2930 test_vhost_user_net( 2931 Some("vunet-tap0"), 2932 2, 2933 &prepare_vhost_user_net_daemon, 2934 false, 2935 false, 2936 ) 2937 } 2938 2939 #[test] 2940 fn test_vhost_user_net_multiple_queues() { 2941 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 2942 } 2943 2944 #[test] 2945 fn test_vhost_user_net_tap_multiple_queues() { 2946 test_vhost_user_net( 2947 Some("vunet-tap1"), 2948 4, 2949 &prepare_vhost_user_net_daemon, 2950 false, 2951 false, 2952 ) 2953 } 2954 2955 #[test] 2956 fn test_vhost_user_net_host_mac() { 2957 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 2958 } 2959 2960 #[test] 2961 fn test_vhost_user_net_client_mode() { 2962 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 2963 } 2964 2965 #[test] 2966 fn test_vhost_user_blk_default() { 2967 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 2968 } 2969 2970 #[test] 2971 fn test_vhost_user_blk_readonly() { 2972 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 2973 } 2974 2975 #[test] 2976 fn test_vhost_user_blk_direct() { 2977 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 2978 } 2979 2980 #[test] 2981 fn test_boot_from_vhost_user_blk_default() { 2982 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 2983 } 2984 2985 #[test] 2986 #[cfg(target_arch = "x86_64")] 2987 fn test_split_irqchip() { 2988 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2989 let guest = Guest::new(Box::new(focal)); 2990 2991 let mut child = GuestCommand::new(&guest) 2992 .args(["--cpus", "boot=1"]) 2993 .args(["--memory", "size=512M"]) 2994 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2995 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2996 .default_disks() 2997 .default_net() 2998 .capture_output() 2999 .spawn() 3000 .unwrap(); 3001 3002 let r = std::panic::catch_unwind(|| { 3003 guest.wait_vm_boot(None).unwrap(); 3004 3005 assert_eq!( 3006 guest 3007 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3008 .unwrap() 3009 .trim() 3010 .parse::<u32>() 3011 .unwrap_or(1), 3012 0 3013 ); 3014 assert_eq!( 3015 guest 3016 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3017 .unwrap() 3018 .trim() 3019 .parse::<u32>() 3020 .unwrap_or(1), 3021 0 3022 ); 3023 }); 3024 3025 let _ = child.kill(); 3026 let output = child.wait_with_output().unwrap(); 3027 3028 handle_child_output(r, &output); 3029 } 3030 3031 #[test] 3032 #[cfg(target_arch = "x86_64")] 3033 fn test_dmi_serial_number() { 3034 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3035 let guest = Guest::new(Box::new(focal)); 3036 3037 let mut child = GuestCommand::new(&guest) 3038 .args(["--cpus", "boot=1"]) 3039 .args(["--memory", "size=512M"]) 3040 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3041 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3042 .args(["--platform", "serial_number=a=b;c=d"]) 3043 .default_disks() 3044 .default_net() 3045 .capture_output() 3046 .spawn() 3047 .unwrap(); 3048 3049 let r = std::panic::catch_unwind(|| { 3050 guest.wait_vm_boot(None).unwrap(); 3051 3052 assert_eq!( 3053 guest 3054 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3055 .unwrap() 3056 .trim(), 3057 "a=b;c=d" 3058 ); 3059 }); 3060 3061 let _ = child.kill(); 3062 let output = child.wait_with_output().unwrap(); 3063 3064 handle_child_output(r, &output); 3065 } 3066 3067 #[test] 3068 #[cfg(target_arch = "x86_64")] 3069 fn test_dmi_uuid() { 3070 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3071 let guest = Guest::new(Box::new(focal)); 3072 3073 let mut child = GuestCommand::new(&guest) 3074 .args(["--cpus", "boot=1"]) 3075 .args(["--memory", "size=512M"]) 3076 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3077 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3078 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3079 .default_disks() 3080 .default_net() 3081 .capture_output() 3082 .spawn() 3083 .unwrap(); 3084 3085 let r = std::panic::catch_unwind(|| { 3086 guest.wait_vm_boot(None).unwrap(); 3087 3088 assert_eq!( 3089 guest 3090 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3091 .unwrap() 3092 .trim(), 3093 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3094 ); 3095 }); 3096 3097 let _ = child.kill(); 3098 let output = child.wait_with_output().unwrap(); 3099 3100 handle_child_output(r, &output); 3101 } 3102 3103 #[test] 3104 #[cfg(target_arch = "x86_64")] 3105 fn test_dmi_oem_strings() { 3106 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3107 let guest = Guest::new(Box::new(focal)); 3108 3109 let s1 = "io.systemd.credential:xx=yy"; 3110 let s2 = "This is a test string"; 3111 3112 let oem_strings = format!("oem_strings=[{},{}]", s1, s2); 3113 3114 let mut child = GuestCommand::new(&guest) 3115 .args(["--cpus", "boot=1"]) 3116 .args(["--memory", "size=512M"]) 3117 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3118 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3119 .args(["--platform", &oem_strings]) 3120 .default_disks() 3121 .default_net() 3122 .capture_output() 3123 .spawn() 3124 .unwrap(); 3125 3126 let r = std::panic::catch_unwind(|| { 3127 guest.wait_vm_boot(None).unwrap(); 3128 3129 assert_eq!( 3130 guest 3131 .ssh_command("sudo dmidecode --oem-string count") 3132 .unwrap() 3133 .trim(), 3134 "2" 3135 ); 3136 3137 assert_eq!( 3138 guest 3139 .ssh_command("sudo dmidecode --oem-string 1") 3140 .unwrap() 3141 .trim(), 3142 s1 3143 ); 3144 3145 assert_eq!( 3146 guest 3147 .ssh_command("sudo dmidecode --oem-string 2") 3148 .unwrap() 3149 .trim(), 3150 s2 3151 ); 3152 }); 3153 3154 let _ = child.kill(); 3155 let output = child.wait_with_output().unwrap(); 3156 3157 handle_child_output(r, &output); 3158 } 3159 3160 #[test] 3161 fn test_virtio_fs() { 3162 _test_virtio_fs(&prepare_virtiofsd, false, None) 3163 } 3164 3165 #[test] 3166 fn test_virtio_fs_hotplug() { 3167 _test_virtio_fs(&prepare_virtiofsd, true, None) 3168 } 3169 3170 #[test] 3171 #[cfg(not(feature = "mshv"))] 3172 fn test_virtio_fs_multi_segment_hotplug() { 3173 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3174 } 3175 3176 #[test] 3177 #[cfg(not(feature = "mshv"))] 3178 fn test_virtio_fs_multi_segment() { 3179 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3180 } 3181 3182 #[test] 3183 fn test_virtio_pmem_persist_writes() { 3184 test_virtio_pmem(false, false) 3185 } 3186 3187 #[test] 3188 fn test_virtio_pmem_discard_writes() { 3189 test_virtio_pmem(true, false) 3190 } 3191 3192 #[test] 3193 fn test_virtio_pmem_with_size() { 3194 test_virtio_pmem(true, true) 3195 } 3196 3197 #[test] 3198 fn test_boot_from_virtio_pmem() { 3199 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3200 let guest = Guest::new(Box::new(focal)); 3201 3202 let kernel_path = direct_kernel_boot_path(); 3203 3204 let mut child = GuestCommand::new(&guest) 3205 .args(["--cpus", "boot=1"]) 3206 .args(["--memory", "size=512M"]) 3207 .args(["--kernel", kernel_path.to_str().unwrap()]) 3208 .args([ 3209 "--disk", 3210 format!( 3211 "path={}", 3212 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3213 ) 3214 .as_str(), 3215 ]) 3216 .default_net() 3217 .args([ 3218 "--pmem", 3219 format!( 3220 "file={},size={}", 3221 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3222 fs::metadata(&guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3223 .unwrap() 3224 .len() 3225 ) 3226 .as_str(), 3227 ]) 3228 .args([ 3229 "--cmdline", 3230 DIRECT_KERNEL_BOOT_CMDLINE 3231 .replace("vda1", "pmem0p1") 3232 .as_str(), 3233 ]) 3234 .capture_output() 3235 .spawn() 3236 .unwrap(); 3237 3238 let r = std::panic::catch_unwind(|| { 3239 guest.wait_vm_boot(None).unwrap(); 3240 3241 // Simple checks to validate the VM booted properly 3242 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3243 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3244 }); 3245 3246 let _ = child.kill(); 3247 let output = child.wait_with_output().unwrap(); 3248 3249 handle_child_output(r, &output); 3250 } 3251 3252 #[test] 3253 fn test_multiple_network_interfaces() { 3254 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3255 let guest = Guest::new(Box::new(focal)); 3256 3257 let kernel_path = direct_kernel_boot_path(); 3258 3259 let mut child = GuestCommand::new(&guest) 3260 .args(["--cpus", "boot=1"]) 3261 .args(["--memory", "size=512M"]) 3262 .args(["--kernel", kernel_path.to_str().unwrap()]) 3263 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3264 .default_disks() 3265 .args([ 3266 "--net", 3267 guest.default_net_string().as_str(), 3268 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3269 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3270 ]) 3271 .capture_output() 3272 .spawn() 3273 .unwrap(); 3274 3275 let r = std::panic::catch_unwind(|| { 3276 guest.wait_vm_boot(None).unwrap(); 3277 3278 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3279 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3280 3281 // 3 network interfaces + default localhost ==> 4 interfaces 3282 assert_eq!( 3283 guest 3284 .ssh_command("ip -o link | wc -l") 3285 .unwrap() 3286 .trim() 3287 .parse::<u32>() 3288 .unwrap_or_default(), 3289 4 3290 ); 3291 }); 3292 3293 let _ = child.kill(); 3294 let output = child.wait_with_output().unwrap(); 3295 3296 handle_child_output(r, &output); 3297 } 3298 3299 #[test] 3300 #[cfg(target_arch = "aarch64")] 3301 fn test_pmu_on() { 3302 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3303 let guest = Guest::new(Box::new(focal)); 3304 let mut child = GuestCommand::new(&guest) 3305 .args(["--cpus", "boot=1"]) 3306 .args(["--memory", "size=512M"]) 3307 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3308 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3309 .default_disks() 3310 .default_net() 3311 .capture_output() 3312 .spawn() 3313 .unwrap(); 3314 3315 let r = std::panic::catch_unwind(|| { 3316 guest.wait_vm_boot(None).unwrap(); 3317 3318 // Test that PMU exists. 3319 assert_eq!( 3320 guest 3321 .ssh_command(GREP_PMU_IRQ_CMD) 3322 .unwrap() 3323 .trim() 3324 .parse::<u32>() 3325 .unwrap_or_default(), 3326 1 3327 ); 3328 }); 3329 3330 let _ = child.kill(); 3331 let output = child.wait_with_output().unwrap(); 3332 3333 handle_child_output(r, &output); 3334 } 3335 3336 #[test] 3337 fn test_serial_off() { 3338 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3339 let guest = Guest::new(Box::new(focal)); 3340 let mut child = GuestCommand::new(&guest) 3341 .args(["--cpus", "boot=1"]) 3342 .args(["--memory", "size=512M"]) 3343 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3344 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3345 .default_disks() 3346 .default_net() 3347 .args(["--serial", "off"]) 3348 .capture_output() 3349 .spawn() 3350 .unwrap(); 3351 3352 let r = std::panic::catch_unwind(|| { 3353 guest.wait_vm_boot(None).unwrap(); 3354 3355 // Test that there is no ttyS0 3356 assert_eq!( 3357 guest 3358 .ssh_command(GREP_SERIAL_IRQ_CMD) 3359 .unwrap() 3360 .trim() 3361 .parse::<u32>() 3362 .unwrap_or(1), 3363 0 3364 ); 3365 }); 3366 3367 let _ = child.kill(); 3368 let output = child.wait_with_output().unwrap(); 3369 3370 handle_child_output(r, &output); 3371 } 3372 3373 #[test] 3374 fn test_serial_null() { 3375 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3376 let guest = Guest::new(Box::new(focal)); 3377 let mut cmd = GuestCommand::new(&guest); 3378 #[cfg(target_arch = "x86_64")] 3379 let console_str: &str = "console=ttyS0"; 3380 #[cfg(target_arch = "aarch64")] 3381 let console_str: &str = "console=ttyAMA0"; 3382 3383 cmd.args(["--cpus", "boot=1"]) 3384 .args(["--memory", "size=512M"]) 3385 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3386 .args([ 3387 "--cmdline", 3388 DIRECT_KERNEL_BOOT_CMDLINE 3389 .replace("console=hvc0 ", console_str) 3390 .as_str(), 3391 ]) 3392 .default_disks() 3393 .default_net() 3394 .args(["--serial", "null"]) 3395 .args(["--console", "off"]) 3396 .capture_output(); 3397 3398 let mut child = cmd.spawn().unwrap(); 3399 3400 let r = std::panic::catch_unwind(|| { 3401 guest.wait_vm_boot(None).unwrap(); 3402 3403 // Test that there is a ttyS0 3404 assert_eq!( 3405 guest 3406 .ssh_command(GREP_SERIAL_IRQ_CMD) 3407 .unwrap() 3408 .trim() 3409 .parse::<u32>() 3410 .unwrap_or_default(), 3411 1 3412 ); 3413 }); 3414 3415 let _ = child.kill(); 3416 let output = child.wait_with_output().unwrap(); 3417 handle_child_output(r, &output); 3418 3419 let r = std::panic::catch_unwind(|| { 3420 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3421 }); 3422 3423 handle_child_output(r, &output); 3424 } 3425 3426 #[test] 3427 fn test_serial_tty() { 3428 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3429 let guest = Guest::new(Box::new(focal)); 3430 3431 let kernel_path = direct_kernel_boot_path(); 3432 3433 #[cfg(target_arch = "x86_64")] 3434 let console_str: &str = "console=ttyS0"; 3435 #[cfg(target_arch = "aarch64")] 3436 let console_str: &str = "console=ttyAMA0"; 3437 3438 let mut child = GuestCommand::new(&guest) 3439 .args(["--cpus", "boot=1"]) 3440 .args(["--memory", "size=512M"]) 3441 .args(["--kernel", kernel_path.to_str().unwrap()]) 3442 .args([ 3443 "--cmdline", 3444 DIRECT_KERNEL_BOOT_CMDLINE 3445 .replace("console=hvc0 ", console_str) 3446 .as_str(), 3447 ]) 3448 .default_disks() 3449 .default_net() 3450 .args(["--serial", "tty"]) 3451 .args(["--console", "off"]) 3452 .capture_output() 3453 .spawn() 3454 .unwrap(); 3455 3456 let r = std::panic::catch_unwind(|| { 3457 guest.wait_vm_boot(None).unwrap(); 3458 3459 // Test that there is a ttyS0 3460 assert_eq!( 3461 guest 3462 .ssh_command(GREP_SERIAL_IRQ_CMD) 3463 .unwrap() 3464 .trim() 3465 .parse::<u32>() 3466 .unwrap_or_default(), 3467 1 3468 ); 3469 }); 3470 3471 // This sleep is needed to wait for the login prompt 3472 thread::sleep(std::time::Duration::new(2, 0)); 3473 3474 let _ = child.kill(); 3475 let output = child.wait_with_output().unwrap(); 3476 handle_child_output(r, &output); 3477 3478 let r = std::panic::catch_unwind(|| { 3479 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3480 }); 3481 3482 handle_child_output(r, &output); 3483 } 3484 3485 #[test] 3486 fn test_serial_file() { 3487 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3488 let guest = Guest::new(Box::new(focal)); 3489 3490 let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output"); 3491 #[cfg(target_arch = "x86_64")] 3492 let console_str: &str = "console=ttyS0"; 3493 #[cfg(target_arch = "aarch64")] 3494 let console_str: &str = "console=ttyAMA0"; 3495 3496 let mut child = GuestCommand::new(&guest) 3497 .args(["--cpus", "boot=1"]) 3498 .args(["--memory", "size=512M"]) 3499 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3500 .args([ 3501 "--cmdline", 3502 DIRECT_KERNEL_BOOT_CMDLINE 3503 .replace("console=hvc0 ", console_str) 3504 .as_str(), 3505 ]) 3506 .default_disks() 3507 .default_net() 3508 .args([ 3509 "--serial", 3510 format!("file={}", serial_path.to_str().unwrap()).as_str(), 3511 ]) 3512 .capture_output() 3513 .spawn() 3514 .unwrap(); 3515 3516 let r = std::panic::catch_unwind(|| { 3517 guest.wait_vm_boot(None).unwrap(); 3518 3519 // Test that there is a ttyS0 3520 assert_eq!( 3521 guest 3522 .ssh_command(GREP_SERIAL_IRQ_CMD) 3523 .unwrap() 3524 .trim() 3525 .parse::<u32>() 3526 .unwrap_or_default(), 3527 1 3528 ); 3529 3530 guest.ssh_command("sudo shutdown -h now").unwrap(); 3531 }); 3532 3533 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3534 let _ = child.kill(); 3535 let output = child.wait_with_output().unwrap(); 3536 handle_child_output(r, &output); 3537 3538 let r = std::panic::catch_unwind(|| { 3539 // Check that the cloud-hypervisor binary actually terminated 3540 assert!(output.status.success()); 3541 3542 // Do this check after shutdown of the VM as an easy way to ensure 3543 // all writes are flushed to disk 3544 let mut f = std::fs::File::open(serial_path).unwrap(); 3545 let mut buf = String::new(); 3546 f.read_to_string(&mut buf).unwrap(); 3547 assert!(buf.contains(CONSOLE_TEST_STRING)); 3548 }); 3549 3550 handle_child_output(r, &output); 3551 } 3552 3553 #[test] 3554 fn test_pty_interaction() { 3555 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3556 let guest = Guest::new(Box::new(focal)); 3557 let api_socket = temp_api_path(&guest.tmp_dir); 3558 let serial_option = if cfg!(target_arch = "x86_64") { 3559 " console=ttyS0" 3560 } else { 3561 " console=ttyAMA0" 3562 }; 3563 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 3564 3565 let mut child = GuestCommand::new(&guest) 3566 .args(["--cpus", "boot=1"]) 3567 .args(["--memory", "size=512M"]) 3568 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3569 .args(["--cmdline", &cmdline]) 3570 .default_disks() 3571 .default_net() 3572 .args(["--serial", "null"]) 3573 .args(["--console", "pty"]) 3574 .args(["--api-socket", &api_socket]) 3575 .spawn() 3576 .unwrap(); 3577 3578 let r = std::panic::catch_unwind(|| { 3579 guest.wait_vm_boot(None).unwrap(); 3580 // Get pty fd for console 3581 let console_path = get_pty_path(&api_socket, "console"); 3582 // TODO: Get serial pty test working 3583 let mut cf = std::fs::OpenOptions::new() 3584 .write(true) 3585 .read(true) 3586 .open(console_path) 3587 .unwrap(); 3588 3589 // Some dumb sleeps but we don't want to write 3590 // before the console is up and we don't want 3591 // to try and write the next line before the 3592 // login process is ready. 3593 thread::sleep(std::time::Duration::new(5, 0)); 3594 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 3595 thread::sleep(std::time::Duration::new(2, 0)); 3596 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 3597 thread::sleep(std::time::Duration::new(2, 0)); 3598 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 3599 thread::sleep(std::time::Duration::new(2, 0)); 3600 3601 // read pty and ensure they have a login shell 3602 // some fairly hacky workarounds to avoid looping 3603 // forever in case the channel is blocked getting output 3604 let ptyc = pty_read(cf); 3605 let mut empty = 0; 3606 let mut prev = String::new(); 3607 loop { 3608 thread::sleep(std::time::Duration::new(2, 0)); 3609 match ptyc.try_recv() { 3610 Ok(line) => { 3611 empty = 0; 3612 prev = prev + &line; 3613 if prev.contains("test_pty_console") { 3614 break; 3615 } 3616 } 3617 Err(mpsc::TryRecvError::Empty) => { 3618 empty += 1; 3619 assert!(empty <= 5, "No login on pty"); 3620 } 3621 _ => panic!("No login on pty"), 3622 } 3623 } 3624 3625 guest.ssh_command("sudo shutdown -h now").unwrap(); 3626 }); 3627 3628 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3629 let _ = child.kill(); 3630 let output = child.wait_with_output().unwrap(); 3631 handle_child_output(r, &output); 3632 3633 let r = std::panic::catch_unwind(|| { 3634 // Check that the cloud-hypervisor binary actually terminated 3635 assert!(output.status.success()) 3636 }); 3637 handle_child_output(r, &output); 3638 } 3639 3640 #[test] 3641 fn test_virtio_console() { 3642 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3643 let guest = Guest::new(Box::new(focal)); 3644 3645 let kernel_path = direct_kernel_boot_path(); 3646 3647 let mut child = GuestCommand::new(&guest) 3648 .args(["--cpus", "boot=1"]) 3649 .args(["--memory", "size=512M"]) 3650 .args(["--kernel", kernel_path.to_str().unwrap()]) 3651 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3652 .default_disks() 3653 .default_net() 3654 .args(["--console", "tty"]) 3655 .args(["--serial", "null"]) 3656 .capture_output() 3657 .spawn() 3658 .unwrap(); 3659 3660 let text = String::from("On a branch floating down river a cricket, singing."); 3661 let cmd = format!("echo {} | sudo tee /dev/hvc0", text); 3662 3663 let r = std::panic::catch_unwind(|| { 3664 guest.wait_vm_boot(None).unwrap(); 3665 3666 assert!(guest 3667 .does_device_vendor_pair_match("0x1043", "0x1af4") 3668 .unwrap_or_default()); 3669 3670 guest.ssh_command(&cmd).unwrap(); 3671 }); 3672 3673 let _ = child.kill(); 3674 let output = child.wait_with_output().unwrap(); 3675 handle_child_output(r, &output); 3676 3677 let r = std::panic::catch_unwind(|| { 3678 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 3679 }); 3680 3681 handle_child_output(r, &output); 3682 } 3683 3684 #[test] 3685 fn test_console_file() { 3686 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3687 let guest = Guest::new(Box::new(focal)); 3688 3689 let console_path = guest.tmp_dir.as_path().join("/tmp/console-output"); 3690 let mut child = GuestCommand::new(&guest) 3691 .args(["--cpus", "boot=1"]) 3692 .args(["--memory", "size=512M"]) 3693 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3694 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3695 .default_disks() 3696 .default_net() 3697 .args([ 3698 "--console", 3699 format!("file={}", console_path.to_str().unwrap()).as_str(), 3700 ]) 3701 .capture_output() 3702 .spawn() 3703 .unwrap(); 3704 3705 guest.wait_vm_boot(None).unwrap(); 3706 3707 guest.ssh_command("sudo shutdown -h now").unwrap(); 3708 3709 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3710 let _ = child.kill(); 3711 let output = child.wait_with_output().unwrap(); 3712 3713 let r = std::panic::catch_unwind(|| { 3714 // Check that the cloud-hypervisor binary actually terminated 3715 assert!(output.status.success()); 3716 3717 // Do this check after shutdown of the VM as an easy way to ensure 3718 // all writes are flushed to disk 3719 let mut f = std::fs::File::open(console_path).unwrap(); 3720 let mut buf = String::new(); 3721 f.read_to_string(&mut buf).unwrap(); 3722 3723 if !buf.contains(CONSOLE_TEST_STRING) { 3724 eprintln!( 3725 "\n\n==== Console file output ====\n\n{}\n\n==== End console file output ====", 3726 buf 3727 ); 3728 } 3729 assert!(buf.contains(CONSOLE_TEST_STRING)); 3730 }); 3731 3732 handle_child_output(r, &output); 3733 } 3734 3735 #[test] 3736 #[cfg(target_arch = "x86_64")] 3737 #[cfg(not(feature = "mshv"))] 3738 #[ignore = "See #4324"] 3739 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 3740 // backed networking interfaces, bound through a simple bridge on the host. 3741 // So if the nested cloud-hypervisor succeeds in getting a directly 3742 // assigned interface from its cloud-hypervisor host, we should be able to 3743 // ssh into it, and verify that it's running with the right kernel command 3744 // line (We tag the command line from cloud-hypervisor for that purpose). 3745 // The third device is added to validate that hotplug works correctly since 3746 // it is being added to the L2 VM through hotplugging mechanism. 3747 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 3748 // vfio device support 3749 fn test_vfio() { 3750 setup_vfio_network_interfaces(); 3751 3752 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3753 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 3754 3755 let mut workload_path = dirs::home_dir().unwrap(); 3756 workload_path.push("workloads"); 3757 3758 let kernel_path = direct_kernel_boot_path(); 3759 3760 let mut vfio_path = workload_path.clone(); 3761 vfio_path.push("vfio"); 3762 3763 let mut cloud_init_vfio_base_path = vfio_path.clone(); 3764 cloud_init_vfio_base_path.push("cloudinit.img"); 3765 3766 // We copy our cloudinit into the vfio mount point, for the nested 3767 // cloud-hypervisor guest to use. 3768 rate_limited_copy( 3769 &guest.disk_config.disk(DiskType::CloudInit).unwrap(), 3770 &cloud_init_vfio_base_path, 3771 ) 3772 .expect("copying of cloud-init disk failed"); 3773 3774 let mut vfio_disk_path = workload_path.clone(); 3775 vfio_disk_path.push("vfio.img"); 3776 3777 // Create the vfio disk image 3778 let output = Command::new("mkfs.ext4") 3779 .arg("-d") 3780 .arg(vfio_path.to_str().unwrap()) 3781 .arg(vfio_disk_path.to_str().unwrap()) 3782 .arg("2g") 3783 .output() 3784 .unwrap(); 3785 if !output.status.success() { 3786 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 3787 panic!("mkfs.ext4 command generated an error"); 3788 } 3789 3790 let mut blk_file_path = workload_path; 3791 blk_file_path.push("blk.img"); 3792 3793 let vfio_tap0 = "vfio-tap0"; 3794 let vfio_tap1 = "vfio-tap1"; 3795 let vfio_tap2 = "vfio-tap2"; 3796 let vfio_tap3 = "vfio-tap3"; 3797 3798 let mut child = GuestCommand::new(&guest) 3799 .args(["--cpus", "boot=4"]) 3800 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 3801 .args(["--kernel", kernel_path.to_str().unwrap()]) 3802 .args([ 3803 "--disk", 3804 format!( 3805 "path={}", 3806 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3807 ) 3808 .as_str(), 3809 format!( 3810 "path={}", 3811 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3812 ) 3813 .as_str(), 3814 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 3815 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 3816 ]) 3817 .args([ 3818 "--cmdline", 3819 format!( 3820 "{} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts", 3821 DIRECT_KERNEL_BOOT_CMDLINE 3822 ) 3823 .as_str(), 3824 ]) 3825 .args([ 3826 "--net", 3827 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 3828 format!( 3829 "tap={},mac={},iommu=on", 3830 vfio_tap1, guest.network.l2_guest_mac1 3831 ) 3832 .as_str(), 3833 format!( 3834 "tap={},mac={},iommu=on", 3835 vfio_tap2, guest.network.l2_guest_mac2 3836 ) 3837 .as_str(), 3838 format!( 3839 "tap={},mac={},iommu=on", 3840 vfio_tap3, guest.network.l2_guest_mac3 3841 ) 3842 .as_str(), 3843 ]) 3844 .capture_output() 3845 .spawn() 3846 .unwrap(); 3847 3848 thread::sleep(std::time::Duration::new(30, 0)); 3849 3850 let r = std::panic::catch_unwind(|| { 3851 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 3852 thread::sleep(std::time::Duration::new(120, 0)); 3853 3854 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 3855 // added to its kernel command line. 3856 // Let's ssh into it and verify that it's there. If it is it means 3857 // we're in the right guest (The L2 one) because the QEMU L1 guest 3858 // does not have this command line tag. 3859 assert_eq!( 3860 guest 3861 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 3862 .unwrap() 3863 .trim() 3864 .parse::<u32>() 3865 .unwrap_or_default(), 3866 1 3867 ); 3868 3869 // Let's also verify from the second virtio-net device passed to 3870 // the L2 VM. 3871 assert_eq!( 3872 guest 3873 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 3874 .unwrap() 3875 .trim() 3876 .parse::<u32>() 3877 .unwrap_or_default(), 3878 1 3879 ); 3880 3881 // Check the amount of PCI devices appearing in L2 VM. 3882 assert_eq!( 3883 guest 3884 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3885 .unwrap() 3886 .trim() 3887 .parse::<u32>() 3888 .unwrap_or_default(), 3889 8, 3890 ); 3891 3892 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 3893 assert_eq!( 3894 guest 3895 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 3896 .unwrap() 3897 .trim() 3898 .parse::<u32>() 3899 .unwrap_or_default(), 3900 1 3901 ); 3902 3903 // Hotplug an extra virtio-net device through L2 VM. 3904 guest 3905 .ssh_command_l1( 3906 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 3907 ) 3908 .unwrap(); 3909 guest 3910 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 3911 .unwrap(); 3912 let vfio_hotplug_output = guest 3913 .ssh_command_l1( 3914 "sudo /mnt/ch-remote \ 3915 --api-socket=/tmp/ch_api.sock \ 3916 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 3917 ) 3918 .unwrap(); 3919 assert!(vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}")); 3920 3921 thread::sleep(std::time::Duration::new(10, 0)); 3922 3923 // Let's also verify from the third virtio-net device passed to 3924 // the L2 VM. This third device has been hotplugged through the L2 3925 // VM, so this is our way to validate hotplug works for VFIO PCI. 3926 assert_eq!( 3927 guest 3928 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 3929 .unwrap() 3930 .trim() 3931 .parse::<u32>() 3932 .unwrap_or_default(), 3933 1 3934 ); 3935 3936 // Check the amount of PCI devices appearing in L2 VM. 3937 // There should be one more device than before, raising the count 3938 // up to 9 PCI devices. 3939 assert_eq!( 3940 guest 3941 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3942 .unwrap() 3943 .trim() 3944 .parse::<u32>() 3945 .unwrap_or_default(), 3946 9, 3947 ); 3948 3949 // Let's now verify that we can correctly remove the virtio-net 3950 // device through the "remove-device" command responsible for 3951 // unplugging VFIO devices. 3952 guest 3953 .ssh_command_l1( 3954 "sudo /mnt/ch-remote \ 3955 --api-socket=/tmp/ch_api.sock \ 3956 remove-device vfio123", 3957 ) 3958 .unwrap(); 3959 thread::sleep(std::time::Duration::new(10, 0)); 3960 3961 // Check the amount of PCI devices appearing in L2 VM is back down 3962 // to 8 devices. 3963 assert_eq!( 3964 guest 3965 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3966 .unwrap() 3967 .trim() 3968 .parse::<u32>() 3969 .unwrap_or_default(), 3970 8, 3971 ); 3972 3973 // Perform memory hotplug in L2 and validate the memory is showing 3974 // up as expected. In order to check, we will use the virtio-net 3975 // device already passed through L2 as a VFIO device, this will 3976 // verify that VFIO devices are functional with memory hotplug. 3977 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 3978 guest 3979 .ssh_command_l2_1( 3980 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 3981 ) 3982 .unwrap(); 3983 guest 3984 .ssh_command_l1( 3985 "sudo /mnt/ch-remote \ 3986 --api-socket=/tmp/ch_api.sock \ 3987 resize --memory=1073741824", 3988 ) 3989 .unwrap(); 3990 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 3991 }); 3992 3993 let _ = child.kill(); 3994 let output = child.wait_with_output().unwrap(); 3995 3996 cleanup_vfio_network_interfaces(); 3997 3998 handle_child_output(r, &output); 3999 } 4000 4001 #[test] 4002 fn test_direct_kernel_boot_noacpi() { 4003 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4004 let guest = Guest::new(Box::new(focal)); 4005 4006 let kernel_path = direct_kernel_boot_path(); 4007 4008 let mut child = GuestCommand::new(&guest) 4009 .args(["--cpus", "boot=1"]) 4010 .args(["--memory", "size=512M"]) 4011 .args(["--kernel", kernel_path.to_str().unwrap()]) 4012 .args([ 4013 "--cmdline", 4014 format!("{} acpi=off", DIRECT_KERNEL_BOOT_CMDLINE).as_str(), 4015 ]) 4016 .default_disks() 4017 .default_net() 4018 .capture_output() 4019 .spawn() 4020 .unwrap(); 4021 4022 let r = std::panic::catch_unwind(|| { 4023 guest.wait_vm_boot(None).unwrap(); 4024 4025 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4026 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4027 }); 4028 4029 let _ = child.kill(); 4030 let output = child.wait_with_output().unwrap(); 4031 4032 handle_child_output(r, &output); 4033 } 4034 4035 #[test] 4036 fn test_reboot() { 4037 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 4038 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4039 4040 vec![Box::new(bionic), Box::new(focal)] 4041 .drain(..) 4042 .for_each(|disk_config| { 4043 let guest = Guest::new(disk_config); 4044 4045 let mut cmd = GuestCommand::new(&guest); 4046 cmd.args(["--cpus", "boot=1"]) 4047 .args(["--memory", "size=512M"]) 4048 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4049 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4050 .default_disks() 4051 .default_net() 4052 .capture_output(); 4053 4054 let mut child = cmd.spawn().unwrap(); 4055 4056 let r = std::panic::catch_unwind(|| { 4057 guest.wait_vm_boot(Some(120)).unwrap(); 4058 4059 let fd_count_1 = get_fd_count(child.id()); 4060 guest.reboot_linux(0, Some(120)); 4061 let fd_count_2 = get_fd_count(child.id()); 4062 assert_eq!(fd_count_1, fd_count_2); 4063 4064 guest.ssh_command("sudo shutdown -h now").unwrap(); 4065 }); 4066 4067 let _ = child.wait_timeout(std::time::Duration::from_secs(40)); 4068 let _ = child.kill(); 4069 let output = child.wait_with_output().unwrap(); 4070 handle_child_output(r, &output); 4071 4072 let r = std::panic::catch_unwind(|| { 4073 // Check that the cloud-hypervisor binary actually terminated 4074 assert!(output.status.success()); 4075 }); 4076 4077 handle_child_output(r, &output); 4078 }); 4079 } 4080 4081 #[test] 4082 fn test_virtio_vsock() { 4083 _test_virtio_vsock(false) 4084 } 4085 4086 #[test] 4087 fn test_virtio_vsock_hotplug() { 4088 _test_virtio_vsock(true); 4089 } 4090 4091 #[test] 4092 // Start cloud-hypervisor with no VM parameters, only the API server running. 4093 // From the API: Create a VM, boot it and check that it looks as expected. 4094 fn test_api_create_boot() { 4095 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4096 let guest = Guest::new(Box::new(focal)); 4097 4098 let api_socket = temp_api_path(&guest.tmp_dir); 4099 4100 let mut child = GuestCommand::new(&guest) 4101 .args(["--api-socket", &api_socket]) 4102 .capture_output() 4103 .spawn() 4104 .unwrap(); 4105 4106 thread::sleep(std::time::Duration::new(1, 0)); 4107 4108 // Verify API server is running 4109 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4110 4111 // Create the VM first 4112 let cpu_count: u8 = 4; 4113 let http_body = guest.api_create_body( 4114 cpu_count, 4115 direct_kernel_boot_path().to_str().unwrap(), 4116 DIRECT_KERNEL_BOOT_CMDLINE, 4117 ); 4118 4119 let temp_config_path = guest.tmp_dir.as_path().join("config"); 4120 std::fs::write(&temp_config_path, http_body).unwrap(); 4121 4122 remote_command( 4123 &api_socket, 4124 "create", 4125 Some(temp_config_path.as_os_str().to_str().unwrap()), 4126 ); 4127 4128 // Then boot it 4129 remote_command(&api_socket, "boot", None); 4130 thread::sleep(std::time::Duration::new(20, 0)); 4131 4132 let r = std::panic::catch_unwind(|| { 4133 // Check that the VM booted as expected 4134 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4135 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4136 }); 4137 4138 let _ = child.kill(); 4139 let output = child.wait_with_output().unwrap(); 4140 4141 handle_child_output(r, &output); 4142 } 4143 4144 #[test] 4145 // Start cloud-hypervisor with no VM parameters, only the API server running. 4146 // From the API: Create a VM, boot it and check it can be shutdown and then 4147 // booted again 4148 fn test_api_shutdown() { 4149 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4150 let guest = Guest::new(Box::new(focal)); 4151 4152 let api_socket = temp_api_path(&guest.tmp_dir); 4153 4154 let mut child = GuestCommand::new(&guest) 4155 .args(["--api-socket", &api_socket]) 4156 .capture_output() 4157 .spawn() 4158 .unwrap(); 4159 4160 thread::sleep(std::time::Duration::new(1, 0)); 4161 4162 // Verify API server is running 4163 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4164 4165 // Create the VM first 4166 let cpu_count: u8 = 4; 4167 let http_body = guest.api_create_body( 4168 cpu_count, 4169 direct_kernel_boot_path().to_str().unwrap(), 4170 DIRECT_KERNEL_BOOT_CMDLINE, 4171 ); 4172 4173 let r = std::panic::catch_unwind(|| { 4174 curl_command( 4175 &api_socket, 4176 "PUT", 4177 "http://localhost/api/v1/vm.create", 4178 Some(&http_body), 4179 ); 4180 4181 // Then boot it 4182 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4183 4184 guest.wait_vm_boot(None).unwrap(); 4185 4186 // Check that the VM booted as expected 4187 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4188 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4189 4190 // Sync and shutdown without powering off to prevent filesystem 4191 // corruption. 4192 guest.ssh_command("sync").unwrap(); 4193 guest.ssh_command("sudo shutdown -H now").unwrap(); 4194 4195 // Wait for the guest to be fully shutdown 4196 thread::sleep(std::time::Duration::new(20, 0)); 4197 4198 // Then shut it down 4199 curl_command( 4200 &api_socket, 4201 "PUT", 4202 "http://localhost/api/v1/vm.shutdown", 4203 None, 4204 ); 4205 4206 // Then boot it again 4207 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4208 4209 guest.wait_vm_boot(None).unwrap(); 4210 4211 // Check that the VM booted as expected 4212 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4213 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4214 }); 4215 4216 let _ = child.kill(); 4217 let output = child.wait_with_output().unwrap(); 4218 4219 handle_child_output(r, &output); 4220 } 4221 4222 #[test] 4223 // Start cloud-hypervisor with no VM parameters, only the API server running. 4224 // From the API: Create a VM, boot it and check it can be deleted and then recreated 4225 // booted again. 4226 fn test_api_delete() { 4227 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4228 let guest = Guest::new(Box::new(focal)); 4229 4230 let api_socket = temp_api_path(&guest.tmp_dir); 4231 4232 let mut child = GuestCommand::new(&guest) 4233 .args(["--api-socket", &api_socket]) 4234 .capture_output() 4235 .spawn() 4236 .unwrap(); 4237 4238 thread::sleep(std::time::Duration::new(1, 0)); 4239 4240 // Verify API server is running 4241 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4242 4243 // Create the VM first 4244 let cpu_count: u8 = 4; 4245 let http_body = guest.api_create_body( 4246 cpu_count, 4247 direct_kernel_boot_path().to_str().unwrap(), 4248 DIRECT_KERNEL_BOOT_CMDLINE, 4249 ); 4250 4251 let r = std::panic::catch_unwind(|| { 4252 curl_command( 4253 &api_socket, 4254 "PUT", 4255 "http://localhost/api/v1/vm.create", 4256 Some(&http_body), 4257 ); 4258 4259 // Then boot it 4260 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4261 4262 guest.wait_vm_boot(None).unwrap(); 4263 4264 // Check that the VM booted as expected 4265 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4266 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4267 4268 // Sync and shutdown without powering off to prevent filesystem 4269 // corruption. 4270 guest.ssh_command("sync").unwrap(); 4271 guest.ssh_command("sudo shutdown -H now").unwrap(); 4272 4273 // Wait for the guest to be fully shutdown 4274 thread::sleep(std::time::Duration::new(20, 0)); 4275 4276 // Then delete it 4277 curl_command( 4278 &api_socket, 4279 "PUT", 4280 "http://localhost/api/v1/vm.delete", 4281 None, 4282 ); 4283 4284 curl_command( 4285 &api_socket, 4286 "PUT", 4287 "http://localhost/api/v1/vm.create", 4288 Some(&http_body), 4289 ); 4290 4291 // Then boot it again 4292 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4293 4294 guest.wait_vm_boot(None).unwrap(); 4295 4296 // Check that the VM booted as expected 4297 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4298 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4299 }); 4300 4301 let _ = child.kill(); 4302 let output = child.wait_with_output().unwrap(); 4303 4304 handle_child_output(r, &output); 4305 } 4306 4307 #[test] 4308 // Start cloud-hypervisor with no VM parameters, only the API server running. 4309 // From the API: Create a VM, boot it and check that it looks as expected. 4310 // Then we pause the VM, check that it's no longer available. 4311 // Finally we resume the VM and check that it's available. 4312 fn test_api_pause_resume() { 4313 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4314 let guest = Guest::new(Box::new(focal)); 4315 4316 let api_socket = temp_api_path(&guest.tmp_dir); 4317 4318 let mut child = GuestCommand::new(&guest) 4319 .args(["--api-socket", &api_socket]) 4320 .capture_output() 4321 .spawn() 4322 .unwrap(); 4323 4324 thread::sleep(std::time::Duration::new(1, 0)); 4325 4326 // Verify API server is running 4327 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 4328 4329 // Create the VM first 4330 let cpu_count: u8 = 4; 4331 let http_body = guest.api_create_body( 4332 cpu_count, 4333 direct_kernel_boot_path().to_str().unwrap(), 4334 DIRECT_KERNEL_BOOT_CMDLINE, 4335 ); 4336 curl_command( 4337 &api_socket, 4338 "PUT", 4339 "http://localhost/api/v1/vm.create", 4340 Some(&http_body), 4341 ); 4342 4343 // Then boot it 4344 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 4345 thread::sleep(std::time::Duration::new(20, 0)); 4346 4347 let r = std::panic::catch_unwind(|| { 4348 // Check that the VM booted as expected 4349 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4350 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4351 4352 // We now pause the VM 4353 assert!(remote_command(&api_socket, "pause", None)); 4354 4355 // Check pausing again fails 4356 assert!(!remote_command(&api_socket, "pause", None)); 4357 4358 thread::sleep(std::time::Duration::new(2, 0)); 4359 4360 // SSH into the VM should fail 4361 assert!(ssh_command_ip( 4362 "grep -c processor /proc/cpuinfo", 4363 &guest.network.guest_ip, 4364 2, 4365 5 4366 ) 4367 .is_err()); 4368 4369 // Resume the VM 4370 assert!(remote_command(&api_socket, "resume", None)); 4371 4372 // Check resuming again fails 4373 assert!(!remote_command(&api_socket, "resume", None)); 4374 4375 thread::sleep(std::time::Duration::new(2, 0)); 4376 4377 // Now we should be able to SSH back in and get the right number of CPUs 4378 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 4379 }); 4380 4381 let _ = child.kill(); 4382 let output = child.wait_with_output().unwrap(); 4383 4384 handle_child_output(r, &output); 4385 } 4386 4387 #[test] 4388 fn test_virtio_iommu() { 4389 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4390 } 4391 4392 #[test] 4393 // We cannot force the software running in the guest to reprogram the BAR 4394 // with some different addresses, but we have a reliable way of testing it 4395 // with a standard Linux kernel. 4396 // By removing a device from the PCI tree, and then rescanning the tree, 4397 // Linux consistently chooses to reorganize the PCI device BARs to other 4398 // locations in the guest address space. 4399 // This test creates a dedicated PCI network device to be checked as being 4400 // properly probed first, then removing it, and adding it again by doing a 4401 // rescan. 4402 fn test_pci_bar_reprogramming() { 4403 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4404 let guest = Guest::new(Box::new(focal)); 4405 4406 #[cfg(target_arch = "x86_64")] 4407 let kernel_path = direct_kernel_boot_path(); 4408 #[cfg(target_arch = "aarch64")] 4409 let kernel_path = edk2_path(); 4410 4411 let mut child = GuestCommand::new(&guest) 4412 .args(["--cpus", "boot=1"]) 4413 .args(["--memory", "size=512M"]) 4414 .args(["--kernel", kernel_path.to_str().unwrap()]) 4415 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4416 .default_disks() 4417 .args([ 4418 "--net", 4419 guest.default_net_string().as_str(), 4420 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4421 ]) 4422 .capture_output() 4423 .spawn() 4424 .unwrap(); 4425 4426 let r = std::panic::catch_unwind(|| { 4427 guest.wait_vm_boot(None).unwrap(); 4428 4429 // 2 network interfaces + default localhost ==> 3 interfaces 4430 assert_eq!( 4431 guest 4432 .ssh_command("ip -o link | wc -l") 4433 .unwrap() 4434 .trim() 4435 .parse::<u32>() 4436 .unwrap_or_default(), 4437 3 4438 ); 4439 4440 let init_bar_addr = guest 4441 .ssh_command( 4442 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4443 ) 4444 .unwrap(); 4445 4446 // Remove the PCI device 4447 guest 4448 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4449 .unwrap(); 4450 4451 // Only 1 network interface left + default localhost ==> 2 interfaces 4452 assert_eq!( 4453 guest 4454 .ssh_command("ip -o link | wc -l") 4455 .unwrap() 4456 .trim() 4457 .parse::<u32>() 4458 .unwrap_or_default(), 4459 2 4460 ); 4461 4462 // Remove the PCI device 4463 guest 4464 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4465 .unwrap(); 4466 4467 // Back to 2 network interface + default localhost ==> 3 interfaces 4468 assert_eq!( 4469 guest 4470 .ssh_command("ip -o link | wc -l") 4471 .unwrap() 4472 .trim() 4473 .parse::<u32>() 4474 .unwrap_or_default(), 4475 3 4476 ); 4477 4478 let new_bar_addr = guest 4479 .ssh_command( 4480 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4481 ) 4482 .unwrap(); 4483 4484 // Let's compare the BAR addresses for our virtio-net device. 4485 // They should be different as we expect the BAR reprogramming 4486 // to have happened. 4487 assert_ne!(init_bar_addr, new_bar_addr); 4488 }); 4489 4490 let _ = child.kill(); 4491 let output = child.wait_with_output().unwrap(); 4492 4493 handle_child_output(r, &output); 4494 } 4495 4496 #[test] 4497 fn test_memory_mergeable_off() { 4498 test_memory_mergeable(false) 4499 } 4500 4501 #[test] 4502 #[cfg(target_arch = "x86_64")] 4503 fn test_cpu_hotplug() { 4504 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4505 let guest = Guest::new(Box::new(focal)); 4506 let api_socket = temp_api_path(&guest.tmp_dir); 4507 4508 let kernel_path = direct_kernel_boot_path(); 4509 4510 let mut child = GuestCommand::new(&guest) 4511 .args(["--cpus", "boot=2,max=4"]) 4512 .args(["--memory", "size=512M"]) 4513 .args(["--kernel", kernel_path.to_str().unwrap()]) 4514 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4515 .default_disks() 4516 .default_net() 4517 .args(["--api-socket", &api_socket]) 4518 .capture_output() 4519 .spawn() 4520 .unwrap(); 4521 4522 let r = std::panic::catch_unwind(|| { 4523 guest.wait_vm_boot(None).unwrap(); 4524 4525 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4526 4527 // Resize the VM 4528 let desired_vcpus = 4; 4529 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4530 4531 guest 4532 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4533 .unwrap(); 4534 guest 4535 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4536 .unwrap(); 4537 thread::sleep(std::time::Duration::new(10, 0)); 4538 assert_eq!( 4539 guest.get_cpu_count().unwrap_or_default(), 4540 u32::from(desired_vcpus) 4541 ); 4542 4543 guest.reboot_linux(0, None); 4544 4545 assert_eq!( 4546 guest.get_cpu_count().unwrap_or_default(), 4547 u32::from(desired_vcpus) 4548 ); 4549 4550 // Resize the VM 4551 let desired_vcpus = 2; 4552 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4553 4554 thread::sleep(std::time::Duration::new(10, 0)); 4555 assert_eq!( 4556 guest.get_cpu_count().unwrap_or_default(), 4557 u32::from(desired_vcpus) 4558 ); 4559 4560 // Resize the VM back up to 4 4561 let desired_vcpus = 4; 4562 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4563 4564 guest 4565 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4566 .unwrap(); 4567 guest 4568 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4569 .unwrap(); 4570 thread::sleep(std::time::Duration::new(10, 0)); 4571 assert_eq!( 4572 guest.get_cpu_count().unwrap_or_default(), 4573 u32::from(desired_vcpus) 4574 ); 4575 }); 4576 4577 let _ = child.kill(); 4578 let output = child.wait_with_output().unwrap(); 4579 4580 handle_child_output(r, &output); 4581 } 4582 4583 #[test] 4584 fn test_memory_hotplug() { 4585 #[cfg(target_arch = "aarch64")] 4586 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4587 #[cfg(target_arch = "x86_64")] 4588 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4589 let focal = UbuntuDiskConfig::new(focal_image); 4590 let guest = Guest::new(Box::new(focal)); 4591 let api_socket = temp_api_path(&guest.tmp_dir); 4592 4593 #[cfg(target_arch = "aarch64")] 4594 let kernel_path = edk2_path(); 4595 #[cfg(target_arch = "x86_64")] 4596 let kernel_path = direct_kernel_boot_path(); 4597 4598 let mut child = GuestCommand::new(&guest) 4599 .args(["--cpus", "boot=2,max=4"]) 4600 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4601 .args(["--kernel", kernel_path.to_str().unwrap()]) 4602 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4603 .default_disks() 4604 .default_net() 4605 .args(["--balloon", "size=0"]) 4606 .args(["--api-socket", &api_socket]) 4607 .capture_output() 4608 .spawn() 4609 .unwrap(); 4610 4611 let r = std::panic::catch_unwind(|| { 4612 guest.wait_vm_boot(None).unwrap(); 4613 4614 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4615 4616 guest.enable_memory_hotplug(); 4617 4618 // Add RAM to the VM 4619 let desired_ram = 1024 << 20; 4620 resize_command(&api_socket, None, Some(desired_ram), None, None); 4621 4622 thread::sleep(std::time::Duration::new(10, 0)); 4623 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4624 4625 // Use balloon to remove RAM from the VM 4626 let desired_balloon = 512 << 20; 4627 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4628 4629 thread::sleep(std::time::Duration::new(10, 0)); 4630 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4631 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4632 4633 guest.reboot_linux(0, None); 4634 4635 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4636 4637 // Use balloon add RAM to the VM 4638 let desired_balloon = 0; 4639 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4640 4641 thread::sleep(std::time::Duration::new(10, 0)); 4642 4643 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4644 4645 guest.enable_memory_hotplug(); 4646 4647 // Add RAM to the VM 4648 let desired_ram = 2048 << 20; 4649 resize_command(&api_socket, None, Some(desired_ram), None, None); 4650 4651 thread::sleep(std::time::Duration::new(10, 0)); 4652 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4653 4654 // Remove RAM to the VM (only applies after reboot) 4655 let desired_ram = 1024 << 20; 4656 resize_command(&api_socket, None, Some(desired_ram), None, None); 4657 4658 guest.reboot_linux(1, None); 4659 4660 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4661 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4662 }); 4663 4664 let _ = child.kill(); 4665 let output = child.wait_with_output().unwrap(); 4666 4667 handle_child_output(r, &output); 4668 } 4669 4670 #[test] 4671 #[cfg(not(feature = "mshv"))] 4672 fn test_virtio_mem() { 4673 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4674 let guest = Guest::new(Box::new(focal)); 4675 let api_socket = temp_api_path(&guest.tmp_dir); 4676 4677 let kernel_path = direct_kernel_boot_path(); 4678 4679 let mut child = GuestCommand::new(&guest) 4680 .args(["--cpus", "boot=2,max=4"]) 4681 .args([ 4682 "--memory", 4683 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4684 ]) 4685 .args(["--kernel", kernel_path.to_str().unwrap()]) 4686 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4687 .default_disks() 4688 .default_net() 4689 .args(["--api-socket", &api_socket]) 4690 .capture_output() 4691 .spawn() 4692 .unwrap(); 4693 4694 let r = std::panic::catch_unwind(|| { 4695 guest.wait_vm_boot(None).unwrap(); 4696 4697 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4698 4699 guest.enable_memory_hotplug(); 4700 4701 // Add RAM to the VM 4702 let desired_ram = 1024 << 20; 4703 resize_command(&api_socket, None, Some(desired_ram), None, None); 4704 4705 thread::sleep(std::time::Duration::new(10, 0)); 4706 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4707 4708 // Add RAM to the VM 4709 let desired_ram = 2048 << 20; 4710 resize_command(&api_socket, None, Some(desired_ram), None, None); 4711 4712 thread::sleep(std::time::Duration::new(10, 0)); 4713 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4714 4715 // Remove RAM from the VM 4716 let desired_ram = 1024 << 20; 4717 resize_command(&api_socket, None, Some(desired_ram), None, None); 4718 4719 thread::sleep(std::time::Duration::new(10, 0)); 4720 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4721 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4722 4723 guest.reboot_linux(0, None); 4724 4725 // Check the amount of memory after reboot is 1GiB 4726 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4727 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4728 4729 // Check we can still resize to 512MiB 4730 let desired_ram = 512 << 20; 4731 resize_command(&api_socket, None, Some(desired_ram), None, None); 4732 thread::sleep(std::time::Duration::new(10, 0)); 4733 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4734 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4735 }); 4736 4737 let _ = child.kill(); 4738 let output = child.wait_with_output().unwrap(); 4739 4740 handle_child_output(r, &output); 4741 } 4742 4743 #[test] 4744 #[cfg(target_arch = "x86_64")] 4745 #[cfg(not(feature = "mshv"))] 4746 // Test both vCPU and memory resizing together 4747 fn test_resize() { 4748 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4749 let guest = Guest::new(Box::new(focal)); 4750 let api_socket = temp_api_path(&guest.tmp_dir); 4751 4752 let kernel_path = direct_kernel_boot_path(); 4753 4754 let mut child = GuestCommand::new(&guest) 4755 .args(["--cpus", "boot=2,max=4"]) 4756 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4757 .args(["--kernel", kernel_path.to_str().unwrap()]) 4758 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4759 .default_disks() 4760 .default_net() 4761 .args(["--api-socket", &api_socket]) 4762 .capture_output() 4763 .spawn() 4764 .unwrap(); 4765 4766 let r = std::panic::catch_unwind(|| { 4767 guest.wait_vm_boot(None).unwrap(); 4768 4769 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4770 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4771 4772 guest.enable_memory_hotplug(); 4773 4774 // Resize the VM 4775 let desired_vcpus = 4; 4776 let desired_ram = 1024 << 20; 4777 resize_command( 4778 &api_socket, 4779 Some(desired_vcpus), 4780 Some(desired_ram), 4781 None, 4782 None, 4783 ); 4784 4785 guest 4786 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4787 .unwrap(); 4788 guest 4789 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4790 .unwrap(); 4791 thread::sleep(std::time::Duration::new(10, 0)); 4792 assert_eq!( 4793 guest.get_cpu_count().unwrap_or_default(), 4794 u32::from(desired_vcpus) 4795 ); 4796 4797 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4798 }); 4799 4800 let _ = child.kill(); 4801 let output = child.wait_with_output().unwrap(); 4802 4803 handle_child_output(r, &output); 4804 } 4805 4806 #[test] 4807 fn test_memory_overhead() { 4808 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4809 let guest = Guest::new(Box::new(focal)); 4810 4811 let kernel_path = direct_kernel_boot_path(); 4812 4813 let guest_memory_size_kb = 512 * 1024; 4814 4815 let mut child = GuestCommand::new(&guest) 4816 .args(["--cpus", "boot=1"]) 4817 .args([ 4818 "--memory", 4819 format!("size={}K", guest_memory_size_kb).as_str(), 4820 ]) 4821 .args(["--kernel", kernel_path.to_str().unwrap()]) 4822 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4823 .default_disks() 4824 .capture_output() 4825 .spawn() 4826 .unwrap(); 4827 4828 thread::sleep(std::time::Duration::new(20, 0)); 4829 4830 let r = std::panic::catch_unwind(|| { 4831 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 4832 eprintln!( 4833 "Guest memory overhead: {} vs {}", 4834 overhead, MAXIMUM_VMM_OVERHEAD_KB 4835 ); 4836 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 4837 }); 4838 4839 let _ = child.kill(); 4840 let output = child.wait_with_output().unwrap(); 4841 4842 handle_child_output(r, &output); 4843 } 4844 4845 #[test] 4846 fn test_disk_hotplug() { 4847 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4848 let guest = Guest::new(Box::new(focal)); 4849 4850 #[cfg(target_arch = "x86_64")] 4851 let kernel_path = direct_kernel_boot_path(); 4852 #[cfg(target_arch = "aarch64")] 4853 let kernel_path = edk2_path(); 4854 4855 let api_socket = temp_api_path(&guest.tmp_dir); 4856 4857 let mut child = GuestCommand::new(&guest) 4858 .args(["--api-socket", &api_socket]) 4859 .args(["--cpus", "boot=1"]) 4860 .args(["--memory", "size=512M"]) 4861 .args(["--kernel", kernel_path.to_str().unwrap()]) 4862 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4863 .default_disks() 4864 .default_net() 4865 .capture_output() 4866 .spawn() 4867 .unwrap(); 4868 4869 let r = std::panic::catch_unwind(|| { 4870 guest.wait_vm_boot(None).unwrap(); 4871 4872 // Check /dev/vdc is not there 4873 assert_eq!( 4874 guest 4875 .ssh_command("lsblk | grep -c vdc.*16M || true") 4876 .unwrap() 4877 .trim() 4878 .parse::<u32>() 4879 .unwrap_or(1), 4880 0 4881 ); 4882 4883 // Now let's add the extra disk. 4884 let mut blk_file_path = dirs::home_dir().unwrap(); 4885 blk_file_path.push("workloads"); 4886 blk_file_path.push("blk.img"); 4887 let (cmd_success, cmd_output) = remote_command_w_output( 4888 &api_socket, 4889 "add-disk", 4890 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4891 ); 4892 assert!(cmd_success); 4893 assert!(String::from_utf8_lossy(&cmd_output) 4894 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4895 4896 thread::sleep(std::time::Duration::new(10, 0)); 4897 4898 // Check that /dev/vdc exists and the block size is 16M. 4899 assert_eq!( 4900 guest 4901 .ssh_command("lsblk | grep vdc | grep -c 16M") 4902 .unwrap() 4903 .trim() 4904 .parse::<u32>() 4905 .unwrap_or_default(), 4906 1 4907 ); 4908 // And check the block device can be read. 4909 guest 4910 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4911 .unwrap(); 4912 4913 // Let's remove it the extra disk. 4914 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4915 thread::sleep(std::time::Duration::new(5, 0)); 4916 // And check /dev/vdc is not there 4917 assert_eq!( 4918 guest 4919 .ssh_command("lsblk | grep -c vdc.*16M || true") 4920 .unwrap() 4921 .trim() 4922 .parse::<u32>() 4923 .unwrap_or(1), 4924 0 4925 ); 4926 4927 // And add it back to validate unplug did work correctly. 4928 let (cmd_success, cmd_output) = remote_command_w_output( 4929 &api_socket, 4930 "add-disk", 4931 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4932 ); 4933 assert!(cmd_success); 4934 assert!(String::from_utf8_lossy(&cmd_output) 4935 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4936 4937 thread::sleep(std::time::Duration::new(10, 0)); 4938 4939 // Check that /dev/vdc exists and the block size is 16M. 4940 assert_eq!( 4941 guest 4942 .ssh_command("lsblk | grep vdc | grep -c 16M") 4943 .unwrap() 4944 .trim() 4945 .parse::<u32>() 4946 .unwrap_or_default(), 4947 1 4948 ); 4949 // And check the block device can be read. 4950 guest 4951 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4952 .unwrap(); 4953 4954 // Reboot the VM. 4955 guest.reboot_linux(0, None); 4956 4957 // Check still there after reboot 4958 assert_eq!( 4959 guest 4960 .ssh_command("lsblk | grep vdc | grep -c 16M") 4961 .unwrap() 4962 .trim() 4963 .parse::<u32>() 4964 .unwrap_or_default(), 4965 1 4966 ); 4967 4968 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4969 4970 thread::sleep(std::time::Duration::new(20, 0)); 4971 4972 // Check device has gone away 4973 assert_eq!( 4974 guest 4975 .ssh_command("lsblk | grep -c vdc.*16M || true") 4976 .unwrap() 4977 .trim() 4978 .parse::<u32>() 4979 .unwrap_or(1), 4980 0 4981 ); 4982 4983 guest.reboot_linux(1, None); 4984 4985 // Check device still absent 4986 assert_eq!( 4987 guest 4988 .ssh_command("lsblk | grep -c vdc.*16M || true") 4989 .unwrap() 4990 .trim() 4991 .parse::<u32>() 4992 .unwrap_or(1), 4993 0 4994 ); 4995 }); 4996 4997 let _ = child.kill(); 4998 let output = child.wait_with_output().unwrap(); 4999 5000 handle_child_output(r, &output); 5001 } 5002 5003 #[allow(clippy::useless_conversion)] 5004 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 5005 const LOOP_CONFIGURE: u64 = 0x4c0a; 5006 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 5007 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 5008 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 5009 5010 #[repr(C)] 5011 struct LoopInfo64 { 5012 lo_device: u64, 5013 lo_inode: u64, 5014 lo_rdevice: u64, 5015 lo_offset: u64, 5016 lo_sizelimit: u64, 5017 lo_number: u32, 5018 lo_encrypt_type: u32, 5019 lo_encrypt_key_size: u32, 5020 lo_flags: u32, 5021 lo_file_name: [u8; 64], 5022 lo_crypt_name: [u8; 64], 5023 lo_encrypt_key: [u8; 32], 5024 lo_init: [u64; 2], 5025 } 5026 5027 impl Default for LoopInfo64 { 5028 fn default() -> Self { 5029 LoopInfo64 { 5030 lo_device: 0, 5031 lo_inode: 0, 5032 lo_rdevice: 0, 5033 lo_offset: 0, 5034 lo_sizelimit: 0, 5035 lo_number: 0, 5036 lo_encrypt_type: 0, 5037 lo_encrypt_key_size: 0, 5038 lo_flags: 0, 5039 lo_file_name: [0; 64], 5040 lo_crypt_name: [0; 64], 5041 lo_encrypt_key: [0; 32], 5042 lo_init: [0; 2], 5043 } 5044 } 5045 } 5046 5047 #[derive(Default)] 5048 #[repr(C)] 5049 struct LoopConfig { 5050 fd: u32, 5051 block_size: u32, 5052 info: LoopInfo64, 5053 _reserved: [u64; 8], 5054 } 5055 5056 // Open loop-control device 5057 let loop_ctl_file = OpenOptions::new() 5058 .read(true) 5059 .write(true) 5060 .open(LOOP_CTL_PATH) 5061 .unwrap(); 5062 5063 // Request a free loop device 5064 let loop_device_number = unsafe { 5065 libc::ioctl( 5066 loop_ctl_file.as_raw_fd(), 5067 LOOP_CTL_GET_FREE.try_into().unwrap(), 5068 ) 5069 }; 5070 if loop_device_number < 0 { 5071 panic!("Couldn't find a free loop device"); 5072 } 5073 5074 // Create loop device path 5075 let loop_device_path = format!("{}{}", LOOP_DEVICE_PREFIX, loop_device_number); 5076 5077 // Open loop device 5078 let loop_device_file = OpenOptions::new() 5079 .read(true) 5080 .write(true) 5081 .open(&loop_device_path) 5082 .unwrap(); 5083 5084 // Open backing file 5085 let backing_file = OpenOptions::new() 5086 .read(true) 5087 .write(true) 5088 .open(backing_file_path) 5089 .unwrap(); 5090 5091 let loop_config = LoopConfig { 5092 fd: backing_file.as_raw_fd() as u32, 5093 block_size, 5094 ..Default::default() 5095 }; 5096 5097 for i in 0..num_retries { 5098 let ret = unsafe { 5099 libc::ioctl( 5100 loop_device_file.as_raw_fd(), 5101 LOOP_CONFIGURE.try_into().unwrap(), 5102 &loop_config, 5103 ) 5104 }; 5105 if ret != 0 { 5106 if i < num_retries - 1 { 5107 println!( 5108 "Iteration {}: Failed to configure the loop device {}: {}", 5109 i, 5110 loop_device_path, 5111 std::io::Error::last_os_error() 5112 ); 5113 } else { 5114 panic!( 5115 "Failed {} times trying to configure the loop device {}: {}", 5116 num_retries, 5117 loop_device_path, 5118 std::io::Error::last_os_error() 5119 ); 5120 } 5121 } else { 5122 break; 5123 } 5124 5125 // Wait for a bit before retrying 5126 thread::sleep(std::time::Duration::new(5, 0)); 5127 } 5128 5129 loop_device_path 5130 } 5131 5132 #[test] 5133 fn test_virtio_block_topology() { 5134 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5135 let guest = Guest::new(Box::new(focal)); 5136 5137 let kernel_path = direct_kernel_boot_path(); 5138 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5139 5140 let output = exec_host_command_output( 5141 format!( 5142 "qemu-img create -f raw {} 16M", 5143 test_disk_path.to_str().unwrap() 5144 ) 5145 .as_str(), 5146 ); 5147 if !output.status.success() { 5148 let stdout = String::from_utf8_lossy(&output.stdout); 5149 let stderr = String::from_utf8_lossy(&output.stderr); 5150 panic!( 5151 "qemu-img command failed\nstdout\n{}\nstderr\n{}", 5152 stdout, stderr 5153 ); 5154 } 5155 5156 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5157 5158 let mut child = GuestCommand::new(&guest) 5159 .args(["--cpus", "boot=1"]) 5160 .args(["--memory", "size=512M"]) 5161 .args(["--kernel", kernel_path.to_str().unwrap()]) 5162 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5163 .args([ 5164 "--disk", 5165 format!( 5166 "path={}", 5167 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5168 ) 5169 .as_str(), 5170 format!( 5171 "path={}", 5172 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5173 ) 5174 .as_str(), 5175 format!("path={}", &loop_dev).as_str(), 5176 ]) 5177 .default_net() 5178 .capture_output() 5179 .spawn() 5180 .unwrap(); 5181 5182 let r = std::panic::catch_unwind(|| { 5183 guest.wait_vm_boot(None).unwrap(); 5184 5185 // MIN-IO column 5186 assert_eq!( 5187 guest 5188 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5189 .unwrap() 5190 .trim() 5191 .parse::<u32>() 5192 .unwrap_or_default(), 5193 4096 5194 ); 5195 // PHY-SEC column 5196 assert_eq!( 5197 guest 5198 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5199 .unwrap() 5200 .trim() 5201 .parse::<u32>() 5202 .unwrap_or_default(), 5203 4096 5204 ); 5205 // LOG-SEC column 5206 assert_eq!( 5207 guest 5208 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5209 .unwrap() 5210 .trim() 5211 .parse::<u32>() 5212 .unwrap_or_default(), 5213 4096 5214 ); 5215 }); 5216 5217 let _ = child.kill(); 5218 let output = child.wait_with_output().unwrap(); 5219 5220 handle_child_output(r, &output); 5221 5222 Command::new("losetup") 5223 .args(["-d", &loop_dev]) 5224 .output() 5225 .expect("loop device not found"); 5226 } 5227 5228 #[test] 5229 fn test_virtio_balloon_deflate_on_oom() { 5230 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5231 let guest = Guest::new(Box::new(focal)); 5232 5233 let kernel_path = direct_kernel_boot_path(); 5234 5235 let api_socket = temp_api_path(&guest.tmp_dir); 5236 5237 //Let's start a 4G guest with balloon occupied 2G memory 5238 let mut child = GuestCommand::new(&guest) 5239 .args(["--api-socket", &api_socket]) 5240 .args(["--cpus", "boot=1"]) 5241 .args(["--memory", "size=4G"]) 5242 .args(["--kernel", kernel_path.to_str().unwrap()]) 5243 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5244 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5245 .default_disks() 5246 .default_net() 5247 .capture_output() 5248 .spawn() 5249 .unwrap(); 5250 5251 let r = std::panic::catch_unwind(|| { 5252 guest.wait_vm_boot(None).unwrap(); 5253 5254 // Wait for balloon memory's initialization and check its size. 5255 // The virtio-balloon driver might take a few seconds to report the 5256 // balloon effective size back to the VMM. 5257 thread::sleep(std::time::Duration::new(20, 0)); 5258 5259 let orig_balloon = balloon_size(&api_socket); 5260 println!("The original balloon memory size is {} bytes", orig_balloon); 5261 assert!(orig_balloon == 2147483648); 5262 5263 // Two steps to verify if the 'deflate_on_oom' parameter works. 5264 // 1st: run a command to trigger an OOM in the guest. 5265 guest 5266 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5267 .unwrap(); 5268 5269 // Give some time for the OOM to happen in the guest and be reported 5270 // back to the host. 5271 thread::sleep(std::time::Duration::new(20, 0)); 5272 5273 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5274 let deflated_balloon = balloon_size(&api_socket); 5275 println!( 5276 "After deflating, balloon memory size is {} bytes", 5277 deflated_balloon 5278 ); 5279 // Verify the balloon size deflated 5280 assert!(deflated_balloon < 2147483648); 5281 }); 5282 5283 let _ = child.kill(); 5284 let output = child.wait_with_output().unwrap(); 5285 5286 handle_child_output(r, &output); 5287 } 5288 5289 #[test] 5290 fn test_virtio_balloon_free_page_reporting() { 5291 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5292 let guest = Guest::new(Box::new(focal)); 5293 5294 //Let's start a 4G guest with balloon occupied 2G memory 5295 let mut child = GuestCommand::new(&guest) 5296 .args(["--cpus", "boot=1"]) 5297 .args(["--memory", "size=4G"]) 5298 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5299 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5300 .args(["--balloon", "size=0,free_page_reporting=on"]) 5301 .default_disks() 5302 .default_net() 5303 .capture_output() 5304 .spawn() 5305 .unwrap(); 5306 5307 let pid = child.id(); 5308 let r = std::panic::catch_unwind(|| { 5309 guest.wait_vm_boot(None).unwrap(); 5310 5311 // Check the initial RSS is less than 1GiB 5312 let rss = process_rss_kib(pid); 5313 println!("RSS {} < 1048576", rss); 5314 assert!(rss < 1048576); 5315 5316 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5317 // seconds 5318 let guest_ip = guest.network.guest_ip.clone(); 5319 thread::spawn(move || { 5320 ssh_command_ip( 5321 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5322 &guest_ip, 5323 DEFAULT_SSH_RETRIES, 5324 DEFAULT_SSH_TIMEOUT, 5325 ) 5326 .unwrap(); 5327 }); 5328 5329 // Wait for 50 seconds to make sure the stress command is consuming 5330 // the expected amount of memory. 5331 thread::sleep(std::time::Duration::new(50, 0)); 5332 let rss = process_rss_kib(pid); 5333 println!("RSS {} >= 2097152", rss); 5334 assert!(rss >= 2097152); 5335 5336 // Wait for an extra minute to make sure the stress command has 5337 // completed and that the guest reported the free pages to the VMM 5338 // through the virtio-balloon device. We expect the RSS to be under 5339 // 2GiB. 5340 thread::sleep(std::time::Duration::new(60, 0)); 5341 let rss = process_rss_kib(pid); 5342 println!("RSS {} < 2097152", rss); 5343 assert!(rss < 2097152); 5344 }); 5345 5346 let _ = child.kill(); 5347 let output = child.wait_with_output().unwrap(); 5348 5349 handle_child_output(r, &output); 5350 } 5351 5352 #[test] 5353 fn test_pmem_hotplug() { 5354 _test_pmem_hotplug(None) 5355 } 5356 5357 #[test] 5358 fn test_pmem_multi_segment_hotplug() { 5359 _test_pmem_hotplug(Some(15)) 5360 } 5361 5362 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5363 #[cfg(target_arch = "aarch64")] 5364 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5365 #[cfg(target_arch = "x86_64")] 5366 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5367 let focal = UbuntuDiskConfig::new(focal_image); 5368 let guest = Guest::new(Box::new(focal)); 5369 5370 #[cfg(target_arch = "x86_64")] 5371 let kernel_path = direct_kernel_boot_path(); 5372 #[cfg(target_arch = "aarch64")] 5373 let kernel_path = edk2_path(); 5374 5375 let api_socket = temp_api_path(&guest.tmp_dir); 5376 5377 let mut cmd = GuestCommand::new(&guest); 5378 5379 cmd.args(["--api-socket", &api_socket]) 5380 .args(["--cpus", "boot=1"]) 5381 .args(["--memory", "size=512M"]) 5382 .args(["--kernel", kernel_path.to_str().unwrap()]) 5383 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5384 .default_disks() 5385 .default_net() 5386 .capture_output(); 5387 5388 if pci_segment.is_some() { 5389 cmd.args(["--platform", "num_pci_segments=16"]); 5390 } 5391 5392 let mut child = cmd.spawn().unwrap(); 5393 5394 let r = std::panic::catch_unwind(|| { 5395 guest.wait_vm_boot(None).unwrap(); 5396 5397 // Check /dev/pmem0 is not there 5398 assert_eq!( 5399 guest 5400 .ssh_command("lsblk | grep -c pmem0 || true") 5401 .unwrap() 5402 .trim() 5403 .parse::<u32>() 5404 .unwrap_or(1), 5405 0 5406 ); 5407 5408 let pmem_temp_file = TempFile::new().unwrap(); 5409 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5410 let (cmd_success, cmd_output) = remote_command_w_output( 5411 &api_socket, 5412 "add-pmem", 5413 Some(&format!( 5414 "file={},id=test0{}", 5415 pmem_temp_file.as_path().to_str().unwrap(), 5416 if let Some(pci_segment) = pci_segment { 5417 format!(",pci_segment={}", pci_segment) 5418 } else { 5419 "".to_owned() 5420 } 5421 )), 5422 ); 5423 assert!(cmd_success); 5424 if let Some(pci_segment) = pci_segment { 5425 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5426 "{{\"id\":\"test0\",\"bdf\":\"{:04x}:00:01.0\"}}", 5427 pci_segment 5428 ))); 5429 } else { 5430 assert!(String::from_utf8_lossy(&cmd_output) 5431 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5432 } 5433 5434 // Check that /dev/pmem0 exists and the block size is 128M 5435 assert_eq!( 5436 guest 5437 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5438 .unwrap() 5439 .trim() 5440 .parse::<u32>() 5441 .unwrap_or_default(), 5442 1 5443 ); 5444 5445 guest.reboot_linux(0, None); 5446 5447 // Check still there after reboot 5448 assert_eq!( 5449 guest 5450 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5451 .unwrap() 5452 .trim() 5453 .parse::<u32>() 5454 .unwrap_or_default(), 5455 1 5456 ); 5457 5458 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5459 5460 thread::sleep(std::time::Duration::new(20, 0)); 5461 5462 // Check device has gone away 5463 assert_eq!( 5464 guest 5465 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5466 .unwrap() 5467 .trim() 5468 .parse::<u32>() 5469 .unwrap_or(1), 5470 0 5471 ); 5472 5473 guest.reboot_linux(1, None); 5474 5475 // Check still absent after reboot 5476 assert_eq!( 5477 guest 5478 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5479 .unwrap() 5480 .trim() 5481 .parse::<u32>() 5482 .unwrap_or(1), 5483 0 5484 ); 5485 }); 5486 5487 let _ = child.kill(); 5488 let output = child.wait_with_output().unwrap(); 5489 5490 handle_child_output(r, &output); 5491 } 5492 5493 #[test] 5494 fn test_net_hotplug() { 5495 _test_net_hotplug(None) 5496 } 5497 5498 #[test] 5499 fn test_net_multi_segment_hotplug() { 5500 _test_net_hotplug(Some(15)) 5501 } 5502 5503 fn _test_net_hotplug(pci_segment: Option<u16>) { 5504 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5505 let guest = Guest::new(Box::new(focal)); 5506 5507 #[cfg(target_arch = "x86_64")] 5508 let kernel_path = direct_kernel_boot_path(); 5509 #[cfg(target_arch = "aarch64")] 5510 let kernel_path = edk2_path(); 5511 5512 let api_socket = temp_api_path(&guest.tmp_dir); 5513 5514 // Boot without network 5515 let mut cmd = GuestCommand::new(&guest); 5516 5517 cmd.args(["--api-socket", &api_socket]) 5518 .args(["--cpus", "boot=1"]) 5519 .args(["--memory", "size=512M"]) 5520 .args(["--kernel", kernel_path.to_str().unwrap()]) 5521 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5522 .default_disks() 5523 .capture_output(); 5524 5525 if pci_segment.is_some() { 5526 cmd.args(["--platform", "num_pci_segments=16"]); 5527 } 5528 5529 let mut child = cmd.spawn().unwrap(); 5530 5531 thread::sleep(std::time::Duration::new(20, 0)); 5532 5533 let r = std::panic::catch_unwind(|| { 5534 // Add network 5535 let (cmd_success, cmd_output) = remote_command_w_output( 5536 &api_socket, 5537 "add-net", 5538 Some( 5539 format!( 5540 "{}{},id=test0", 5541 guest.default_net_string(), 5542 if let Some(pci_segment) = pci_segment { 5543 format!(",pci_segment={}", pci_segment) 5544 } else { 5545 "".to_owned() 5546 } 5547 ) 5548 .as_str(), 5549 ), 5550 ); 5551 assert!(cmd_success); 5552 5553 if let Some(pci_segment) = pci_segment { 5554 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5555 "{{\"id\":\"test0\",\"bdf\":\"{:04x}:00:01.0\"}}", 5556 pci_segment 5557 ))); 5558 } else { 5559 assert!(String::from_utf8_lossy(&cmd_output) 5560 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5561 } 5562 5563 thread::sleep(std::time::Duration::new(5, 0)); 5564 5565 // 1 network interfaces + default localhost ==> 2 interfaces 5566 assert_eq!( 5567 guest 5568 .ssh_command("ip -o link | wc -l") 5569 .unwrap() 5570 .trim() 5571 .parse::<u32>() 5572 .unwrap_or_default(), 5573 2 5574 ); 5575 5576 // Remove network 5577 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5578 thread::sleep(std::time::Duration::new(5, 0)); 5579 5580 let (cmd_success, cmd_output) = remote_command_w_output( 5581 &api_socket, 5582 "add-net", 5583 Some( 5584 format!( 5585 "{}{},id=test1", 5586 guest.default_net_string(), 5587 if let Some(pci_segment) = pci_segment { 5588 format!(",pci_segment={}", pci_segment) 5589 } else { 5590 "".to_owned() 5591 } 5592 ) 5593 .as_str(), 5594 ), 5595 ); 5596 assert!(cmd_success); 5597 5598 if let Some(pci_segment) = pci_segment { 5599 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5600 "{{\"id\":\"test1\",\"bdf\":\"{:04x}:00:01.0\"}}", 5601 pci_segment 5602 ))); 5603 } else { 5604 assert!(String::from_utf8_lossy(&cmd_output) 5605 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5606 } 5607 5608 thread::sleep(std::time::Duration::new(5, 0)); 5609 5610 // 1 network interfaces + default localhost ==> 2 interfaces 5611 assert_eq!( 5612 guest 5613 .ssh_command("ip -o link | wc -l") 5614 .unwrap() 5615 .trim() 5616 .parse::<u32>() 5617 .unwrap_or_default(), 5618 2 5619 ); 5620 5621 guest.reboot_linux(0, None); 5622 5623 // Check still there after reboot 5624 // 1 network interfaces + default localhost ==> 2 interfaces 5625 assert_eq!( 5626 guest 5627 .ssh_command("ip -o link | wc -l") 5628 .unwrap() 5629 .trim() 5630 .parse::<u32>() 5631 .unwrap_or_default(), 5632 2 5633 ); 5634 }); 5635 5636 let _ = child.kill(); 5637 let output = child.wait_with_output().unwrap(); 5638 5639 handle_child_output(r, &output); 5640 } 5641 5642 #[test] 5643 fn test_initramfs() { 5644 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5645 let guest = Guest::new(Box::new(focal)); 5646 let mut workload_path = dirs::home_dir().unwrap(); 5647 workload_path.push("workloads"); 5648 5649 #[cfg(target_arch = "x86_64")] 5650 let mut kernels = vec![direct_kernel_boot_path()]; 5651 #[cfg(target_arch = "aarch64")] 5652 let kernels = vec![direct_kernel_boot_path()]; 5653 5654 #[cfg(target_arch = "x86_64")] 5655 { 5656 let mut pvh_kernel_path = workload_path.clone(); 5657 pvh_kernel_path.push("vmlinux"); 5658 kernels.push(pvh_kernel_path); 5659 } 5660 5661 let mut initramfs_path = workload_path; 5662 initramfs_path.push("alpine_initramfs.img"); 5663 5664 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5665 let cmdline = format!("console=hvc0 quiet TEST_STRING={}", test_string); 5666 5667 kernels.iter().for_each(|k_path| { 5668 let mut child = GuestCommand::new(&guest) 5669 .args(["--kernel", k_path.to_str().unwrap()]) 5670 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 5671 .args(["--cmdline", &cmdline]) 5672 .capture_output() 5673 .spawn() 5674 .unwrap(); 5675 5676 thread::sleep(std::time::Duration::new(20, 0)); 5677 5678 let _ = child.kill(); 5679 let output = child.wait_with_output().unwrap(); 5680 5681 let r = std::panic::catch_unwind(|| { 5682 let s = String::from_utf8_lossy(&output.stdout); 5683 5684 assert_ne!(s.lines().position(|line| line == test_string), None); 5685 }); 5686 5687 handle_child_output(r, &output); 5688 }); 5689 } 5690 5691 // One thing to note about this test. The virtio-net device is heavily used 5692 // through each ssh command. There's no need to perform a dedicated test to 5693 // verify the migration went well for virtio-net. 5694 #[test] 5695 fn test_snapshot_restore() { 5696 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5697 let guest = Guest::new(Box::new(focal)); 5698 let kernel_path = direct_kernel_boot_path(); 5699 5700 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 5701 5702 let net_id = "net123"; 5703 let net_params = format!( 5704 "id={},tap=,mac={},ip={},mask=255.255.255.0", 5705 net_id, guest.network.guest_mac, guest.network.host_ip 5706 ); 5707 5708 let cloudinit_params = format!( 5709 "path={},iommu=on", 5710 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5711 ); 5712 5713 let socket = temp_vsock_path(&guest.tmp_dir); 5714 let event_path = temp_event_monitor_path(&guest.tmp_dir); 5715 5716 let mut child = GuestCommand::new(&guest) 5717 .args(["--api-socket", &api_socket_source]) 5718 .args(["--event-monitor", format!("path={}", event_path).as_str()]) 5719 .args(["--cpus", "boot=4"]) 5720 .args([ 5721 "--memory", 5722 "size=4G,hotplug_method=virtio-mem,hotplug_size=32G", 5723 ]) 5724 .args(["--balloon", "size=0"]) 5725 .args(["--kernel", kernel_path.to_str().unwrap()]) 5726 .args([ 5727 "--disk", 5728 format!( 5729 "path={}", 5730 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5731 ) 5732 .as_str(), 5733 cloudinit_params.as_str(), 5734 ]) 5735 .args(["--net", net_params.as_str()]) 5736 .args(["--vsock", format!("cid=3,socket={}", socket).as_str()]) 5737 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5738 .capture_output() 5739 .spawn() 5740 .unwrap(); 5741 5742 let console_text = String::from("On a branch floating down river a cricket, singing."); 5743 // Create the snapshot directory 5744 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 5745 5746 let r = std::panic::catch_unwind(|| { 5747 guest.wait_vm_boot(None).unwrap(); 5748 5749 // Check the number of vCPUs 5750 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5751 // Check the guest RAM 5752 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5753 // Increase guest RAM with virtio-mem 5754 resize_command( 5755 &api_socket_source, 5756 None, 5757 Some(6 << 30), 5758 None, 5759 Some(&event_path), 5760 ); 5761 thread::sleep(std::time::Duration::new(5, 0)); 5762 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5763 // Use balloon to remove RAM from the VM 5764 resize_command( 5765 &api_socket_source, 5766 None, 5767 None, 5768 Some(1 << 30), 5769 Some(&event_path), 5770 ); 5771 thread::sleep(std::time::Duration::new(5, 0)); 5772 let total_memory = guest.get_total_memory().unwrap_or_default(); 5773 assert!(total_memory > 4_800_000); 5774 assert!(total_memory < 5_760_000); 5775 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 5776 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5777 5778 // x86_64: We check that removing and adding back the virtio-net device 5779 // does not break the snapshot/restore support for virtio-pci. 5780 // This is an important thing to test as the hotplug will 5781 // trigger a PCI BAR reprogramming, which is a good way of 5782 // checking if the stored resources are correctly restored. 5783 // Unplug the virtio-net device 5784 // AArch64: Device hotplug is currently not supported, skipping here. 5785 #[cfg(target_arch = "x86_64")] 5786 { 5787 assert!(remote_command( 5788 &api_socket_source, 5789 "remove-device", 5790 Some(net_id), 5791 )); 5792 thread::sleep(std::time::Duration::new(10, 0)); 5793 let latest_events = [&MetaEvent { 5794 event: "device-removed".to_string(), 5795 device_id: Some(net_id.to_string()), 5796 }]; 5797 assert!(check_latest_events_exact(&latest_events, &event_path)); 5798 5799 // Plug the virtio-net device again 5800 assert!(remote_command( 5801 &api_socket_source, 5802 "add-net", 5803 Some(net_params.as_str()), 5804 )); 5805 thread::sleep(std::time::Duration::new(10, 0)); 5806 } 5807 5808 // Pause the VM 5809 assert!(remote_command(&api_socket_source, "pause", None)); 5810 let latest_events = [ 5811 &MetaEvent { 5812 event: "pausing".to_string(), 5813 device_id: None, 5814 }, 5815 &MetaEvent { 5816 event: "paused".to_string(), 5817 device_id: None, 5818 }, 5819 ]; 5820 assert!(check_latest_events_exact(&latest_events, &event_path)); 5821 5822 // Take a snapshot from the VM 5823 assert!(remote_command( 5824 &api_socket_source, 5825 "snapshot", 5826 Some(format!("file://{}", snapshot_dir).as_str()), 5827 )); 5828 5829 // Wait to make sure the snapshot is completed 5830 thread::sleep(std::time::Duration::new(10, 0)); 5831 5832 let latest_events = [ 5833 &MetaEvent { 5834 event: "snapshotting".to_string(), 5835 device_id: None, 5836 }, 5837 &MetaEvent { 5838 event: "snapshotted".to_string(), 5839 device_id: None, 5840 }, 5841 ]; 5842 assert!(check_latest_events_exact(&latest_events, &event_path)); 5843 }); 5844 5845 // Shutdown the source VM and check console output 5846 let _ = child.kill(); 5847 let output = child.wait_with_output().unwrap(); 5848 handle_child_output(r, &output); 5849 5850 let r = std::panic::catch_unwind(|| { 5851 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5852 }); 5853 5854 handle_child_output(r, &output); 5855 5856 // Remove the vsock socket file. 5857 Command::new("rm") 5858 .arg("-f") 5859 .arg(socket.as_str()) 5860 .output() 5861 .unwrap(); 5862 5863 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 5864 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 5865 5866 // Restore the VM from the snapshot 5867 let mut child = GuestCommand::new(&guest) 5868 .args(["--api-socket", &api_socket_restored]) 5869 .args([ 5870 "--event-monitor", 5871 format!("path={}", event_path_restored).as_str(), 5872 ]) 5873 .args([ 5874 "--restore", 5875 format!("source_url=file://{}", snapshot_dir).as_str(), 5876 ]) 5877 .capture_output() 5878 .spawn() 5879 .unwrap(); 5880 5881 // Wait for the VM to be restored 5882 thread::sleep(std::time::Duration::new(10, 0)); 5883 let expected_events = [ 5884 &MetaEvent { 5885 event: "starting".to_string(), 5886 device_id: None, 5887 }, 5888 &MetaEvent { 5889 event: "restoring".to_string(), 5890 device_id: None, 5891 }, 5892 ]; 5893 assert!(check_sequential_events_exact( 5894 &expected_events, 5895 &event_path_restored 5896 )); 5897 let latest_events = [&MetaEvent { 5898 event: "restored".to_string(), 5899 device_id: None, 5900 }]; 5901 assert!(check_latest_events_exact( 5902 &latest_events, 5903 &event_path_restored 5904 )); 5905 5906 let r = std::panic::catch_unwind(|| { 5907 // Resume the VM 5908 assert!(remote_command(&api_socket_restored, "resume", None)); 5909 let latest_events = [ 5910 &MetaEvent { 5911 event: "resuming".to_string(), 5912 device_id: None, 5913 }, 5914 &MetaEvent { 5915 event: "resumed".to_string(), 5916 device_id: None, 5917 }, 5918 ]; 5919 assert!(check_latest_events_exact( 5920 &latest_events, 5921 &event_path_restored 5922 )); 5923 5924 // Perform same checks to validate VM has been properly restored 5925 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5926 let total_memory = guest.get_total_memory().unwrap_or_default(); 5927 assert!(total_memory > 4_800_000); 5928 assert!(total_memory < 5_760_000); 5929 // Deflate balloon to restore entire RAM to the VM 5930 resize_command(&api_socket_restored, None, None, Some(0), None); 5931 thread::sleep(std::time::Duration::new(5, 0)); 5932 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5933 // Decrease guest RAM with virtio-mem 5934 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 5935 thread::sleep(std::time::Duration::new(5, 0)); 5936 let total_memory = guest.get_total_memory().unwrap_or_default(); 5937 assert!(total_memory > 4_800_000); 5938 assert!(total_memory < 5_760_000); 5939 5940 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5941 }); 5942 // Shutdown the target VM and check console output 5943 let _ = child.kill(); 5944 let output = child.wait_with_output().unwrap(); 5945 handle_child_output(r, &output); 5946 5947 let r = std::panic::catch_unwind(|| { 5948 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 5949 }); 5950 5951 handle_child_output(r, &output); 5952 } 5953 5954 #[test] 5955 fn test_counters() { 5956 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5957 let guest = Guest::new(Box::new(focal)); 5958 let api_socket = temp_api_path(&guest.tmp_dir); 5959 5960 let mut cmd = GuestCommand::new(&guest); 5961 cmd.args(["--cpus", "boot=1"]) 5962 .args(["--memory", "size=512M"]) 5963 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5964 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5965 .default_disks() 5966 .args(["--net", guest.default_net_string().as_str()]) 5967 .args(["--api-socket", &api_socket]) 5968 .capture_output(); 5969 5970 let mut child = cmd.spawn().unwrap(); 5971 5972 let r = std::panic::catch_unwind(|| { 5973 guest.wait_vm_boot(None).unwrap(); 5974 5975 let orig_counters = get_counters(&api_socket); 5976 guest 5977 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 5978 .unwrap(); 5979 5980 let new_counters = get_counters(&api_socket); 5981 5982 // Check that all the counters have increased 5983 assert!(new_counters > orig_counters); 5984 }); 5985 5986 let _ = child.kill(); 5987 let output = child.wait_with_output().unwrap(); 5988 5989 handle_child_output(r, &output); 5990 } 5991 5992 #[test] 5993 #[cfg(feature = "guest_debug")] 5994 fn test_coredump() { 5995 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5996 let guest = Guest::new(Box::new(focal)); 5997 let api_socket = temp_api_path(&guest.tmp_dir); 5998 5999 let mut cmd = GuestCommand::new(&guest); 6000 cmd.args(["--cpus", "boot=4"]) 6001 .args(["--memory", "size=4G"]) 6002 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6003 .default_disks() 6004 .args(["--net", guest.default_net_string().as_str()]) 6005 .args(["--api-socket", &api_socket]) 6006 .capture_output(); 6007 6008 let mut child = cmd.spawn().unwrap(); 6009 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6010 6011 let r = std::panic::catch_unwind(|| { 6012 guest.wait_vm_boot(None).unwrap(); 6013 6014 assert!(remote_command(&api_socket, "pause", None)); 6015 6016 assert!(remote_command( 6017 &api_socket, 6018 "coredump", 6019 Some(format!("file://{}", vmcore_file).as_str()), 6020 )); 6021 6022 // the num of CORE notes should equals to vcpu 6023 let readelf_core_num_cmd = format!( 6024 "readelf --all {} |grep CORE |grep -v Type |wc -l", 6025 vmcore_file 6026 ); 6027 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 6028 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 6029 6030 // the num of QEMU notes should equals to vcpu 6031 let readelf_vmm_num_cmd = format!("readelf --all {} |grep QEMU |wc -l", vmcore_file); 6032 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 6033 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 6034 }); 6035 6036 let _ = child.kill(); 6037 let output = child.wait_with_output().unwrap(); 6038 6039 handle_child_output(r, &output); 6040 } 6041 6042 #[test] 6043 fn test_watchdog() { 6044 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6045 let guest = Guest::new(Box::new(focal)); 6046 let api_socket = temp_api_path(&guest.tmp_dir); 6047 6048 let kernel_path = direct_kernel_boot_path(); 6049 6050 let mut cmd = GuestCommand::new(&guest); 6051 cmd.args(["--cpus", "boot=1"]) 6052 .args(["--memory", "size=512M"]) 6053 .args(["--kernel", kernel_path.to_str().unwrap()]) 6054 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6055 .default_disks() 6056 .args(["--net", guest.default_net_string().as_str()]) 6057 .args(["--watchdog"]) 6058 .args(["--api-socket", &api_socket]) 6059 .capture_output(); 6060 6061 let mut child = cmd.spawn().unwrap(); 6062 6063 let r = std::panic::catch_unwind(|| { 6064 guest.wait_vm_boot(None).unwrap(); 6065 6066 let mut expected_reboot_count = 1; 6067 6068 // Enable the watchdog with a 15s timeout 6069 enable_guest_watchdog(&guest, 15); 6070 6071 // Reboot and check that systemd has activated the watchdog 6072 guest.ssh_command("sudo reboot").unwrap(); 6073 guest.wait_vm_boot(None).unwrap(); 6074 expected_reboot_count += 1; 6075 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6076 assert_eq!( 6077 guest 6078 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6079 .unwrap() 6080 .trim() 6081 .parse::<u32>() 6082 .unwrap_or_default(), 6083 2 6084 ); 6085 6086 // Allow some normal time to elapse to check we don't get spurious reboots 6087 thread::sleep(std::time::Duration::new(40, 0)); 6088 // Check no reboot 6089 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6090 6091 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6092 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6093 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6094 guest.wait_vm_boot(Some(50)).unwrap(); 6095 // Check a reboot is triggerred by the watchdog 6096 expected_reboot_count += 1; 6097 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6098 6099 #[cfg(target_arch = "x86_64")] 6100 { 6101 // Now pause the VM and remain offline for 30s 6102 assert!(remote_command(&api_socket, "pause", None)); 6103 thread::sleep(std::time::Duration::new(30, 0)); 6104 assert!(remote_command(&api_socket, "resume", None)); 6105 6106 // Check no reboot 6107 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6108 } 6109 }); 6110 6111 let _ = child.kill(); 6112 let output = child.wait_with_output().unwrap(); 6113 6114 handle_child_output(r, &output); 6115 } 6116 6117 #[test] 6118 fn test_tap_from_fd() { 6119 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6120 let guest = Guest::new(Box::new(focal)); 6121 let kernel_path = direct_kernel_boot_path(); 6122 6123 // Create a TAP interface with multi-queue enabled 6124 let num_queue_pairs: usize = 2; 6125 6126 use std::str::FromStr; 6127 let taps = net_util::open_tap( 6128 Some("chtap0"), 6129 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6130 None, 6131 &mut None, 6132 None, 6133 num_queue_pairs, 6134 Some(libc::O_RDWR | libc::O_NONBLOCK), 6135 ) 6136 .unwrap(); 6137 6138 let mut child = GuestCommand::new(&guest) 6139 .args(["--cpus", &format!("boot={}", num_queue_pairs)]) 6140 .args(["--memory", "size=512M"]) 6141 .args(["--kernel", kernel_path.to_str().unwrap()]) 6142 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6143 .default_disks() 6144 .args([ 6145 "--net", 6146 &format!( 6147 "fd=[{},{}],mac={},num_queues={}", 6148 taps[0].as_raw_fd(), 6149 taps[1].as_raw_fd(), 6150 guest.network.guest_mac, 6151 num_queue_pairs * 2 6152 ), 6153 ]) 6154 .capture_output() 6155 .spawn() 6156 .unwrap(); 6157 6158 let r = std::panic::catch_unwind(|| { 6159 guest.wait_vm_boot(None).unwrap(); 6160 6161 assert_eq!( 6162 guest 6163 .ssh_command("ip -o link | wc -l") 6164 .unwrap() 6165 .trim() 6166 .parse::<u32>() 6167 .unwrap_or_default(), 6168 2 6169 ); 6170 6171 guest.reboot_linux(0, None); 6172 6173 assert_eq!( 6174 guest 6175 .ssh_command("ip -o link | wc -l") 6176 .unwrap() 6177 .trim() 6178 .parse::<u32>() 6179 .unwrap_or_default(), 6180 2 6181 ); 6182 }); 6183 6184 let _ = child.kill(); 6185 let output = child.wait_with_output().unwrap(); 6186 6187 handle_child_output(r, &output); 6188 } 6189 6190 // By design, a guest VM won't be able to connect to the host 6191 // machine when using a macvtap network interface (while it can 6192 // communicate externally). As a workaround, this integration 6193 // test creates two macvtap interfaces in 'bridge' mode on the 6194 // same physical net interface, one for the guest and one for 6195 // the host. With additional setup on the IP address and the 6196 // routing table, it enables the communications between the 6197 // guest VM and the host machine. 6198 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6199 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6200 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6201 let guest = Guest::new(Box::new(focal)); 6202 let api_socket = temp_api_path(&guest.tmp_dir); 6203 6204 #[cfg(target_arch = "x86_64")] 6205 let kernel_path = direct_kernel_boot_path(); 6206 #[cfg(target_arch = "aarch64")] 6207 let kernel_path = edk2_path(); 6208 6209 let phy_net = "eth0"; 6210 6211 // Create a macvtap interface for the guest VM to use 6212 assert!(exec_host_command_status(&format!( 6213 "sudo ip link add link {} name {} type macvtap mod bridge", 6214 phy_net, guest_macvtap_name 6215 )) 6216 .success()); 6217 assert!(exec_host_command_status(&format!( 6218 "sudo ip link set {} address {} up", 6219 guest_macvtap_name, guest.network.guest_mac 6220 )) 6221 .success()); 6222 assert!( 6223 exec_host_command_status(&format!("sudo ip link show {}", guest_macvtap_name)) 6224 .success() 6225 ); 6226 6227 let tap_index = 6228 fs::read_to_string(&format!("/sys/class/net/{}/ifindex", guest_macvtap_name)).unwrap(); 6229 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6230 6231 assert!( 6232 exec_host_command_status(&format!("sudo chown $UID.$UID {}", tap_device)).success() 6233 ); 6234 6235 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6236 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6237 assert!(tap_fd1 > 0); 6238 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6239 assert!(tap_fd2 > 0); 6240 6241 // Create a macvtap on the same physical net interface for 6242 // the host machine to use 6243 assert!(exec_host_command_status(&format!( 6244 "sudo ip link add link {} name {} type macvtap mod bridge", 6245 phy_net, host_macvtap_name 6246 )) 6247 .success()); 6248 // Use default mask "255.255.255.0" 6249 assert!(exec_host_command_status(&format!( 6250 "sudo ip address add {}/24 dev {}", 6251 guest.network.host_ip, host_macvtap_name 6252 )) 6253 .success()); 6254 assert!(exec_host_command_status(&format!( 6255 "sudo ip link set dev {} up", 6256 host_macvtap_name 6257 )) 6258 .success()); 6259 6260 let mut guest_command = GuestCommand::new(&guest); 6261 guest_command 6262 .args(["--cpus", "boot=2"]) 6263 .args(["--memory", "size=512M"]) 6264 .args(["--kernel", kernel_path.to_str().unwrap()]) 6265 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6266 .default_disks() 6267 .args(["--api-socket", &api_socket]); 6268 6269 let net_params = format!( 6270 "fd=[{},{}],mac={},num_queues=4", 6271 tap_fd1, tap_fd2, guest.network.guest_mac 6272 ); 6273 6274 if !hotplug { 6275 guest_command.args(["--net", &net_params]); 6276 } 6277 6278 let mut child = guest_command.capture_output().spawn().unwrap(); 6279 6280 if hotplug { 6281 // Give some time to the VMM process to listen to the API 6282 // socket. This is the only requirement to avoid the following 6283 // call to ch-remote from failing. 6284 thread::sleep(std::time::Duration::new(10, 0)); 6285 // Hotplug the virtio-net device 6286 let (cmd_success, cmd_output) = 6287 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6288 assert!(cmd_success); 6289 #[cfg(target_arch = "x86_64")] 6290 assert!(String::from_utf8_lossy(&cmd_output) 6291 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6292 #[cfg(target_arch = "aarch64")] 6293 assert!(String::from_utf8_lossy(&cmd_output) 6294 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6295 } 6296 6297 // The functional connectivity provided by the virtio-net device 6298 // gets tested through wait_vm_boot() as it expects to receive a 6299 // HTTP request, and through the SSH command as well. 6300 let r = std::panic::catch_unwind(|| { 6301 guest.wait_vm_boot(None).unwrap(); 6302 6303 assert_eq!( 6304 guest 6305 .ssh_command("ip -o link | wc -l") 6306 .unwrap() 6307 .trim() 6308 .parse::<u32>() 6309 .unwrap_or_default(), 6310 2 6311 ); 6312 }); 6313 6314 let _ = child.kill(); 6315 6316 exec_host_command_status(&format!("sudo ip link del {}", guest_macvtap_name)); 6317 exec_host_command_status(&format!("sudo ip link del {}", host_macvtap_name)); 6318 6319 let output = child.wait_with_output().unwrap(); 6320 6321 handle_child_output(r, &output); 6322 } 6323 6324 #[test] 6325 fn test_macvtap() { 6326 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6327 } 6328 6329 #[test] 6330 fn test_macvtap_hotplug() { 6331 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6332 } 6333 6334 #[test] 6335 #[cfg(not(feature = "mshv"))] 6336 fn test_ovs_dpdk() { 6337 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6338 let guest1 = Guest::new(Box::new(focal1)); 6339 6340 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6341 let guest2 = Guest::new(Box::new(focal2)); 6342 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6343 6344 let (mut child1, mut child2) = 6345 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6346 6347 // Create the snapshot directory 6348 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6349 6350 let r = std::panic::catch_unwind(|| { 6351 // Remove one of the two ports from the OVS bridge 6352 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6353 6354 // Spawn a new netcat listener in the first VM 6355 let guest_ip = guest1.network.guest_ip.clone(); 6356 thread::spawn(move || { 6357 ssh_command_ip( 6358 "nc -l 12345", 6359 &guest_ip, 6360 DEFAULT_SSH_RETRIES, 6361 DEFAULT_SSH_TIMEOUT, 6362 ) 6363 .unwrap(); 6364 }); 6365 6366 // Wait for the server to be listening 6367 thread::sleep(std::time::Duration::new(5, 0)); 6368 6369 // Check the connection fails this time 6370 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6371 6372 // Add the OVS port back 6373 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6374 6375 // And finally check the connection is functional again 6376 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6377 6378 // Pause the VM 6379 assert!(remote_command(&api_socket_source, "pause", None)); 6380 6381 // Take a snapshot from the VM 6382 assert!(remote_command( 6383 &api_socket_source, 6384 "snapshot", 6385 Some(format!("file://{}", snapshot_dir).as_str()), 6386 )); 6387 6388 // Wait to make sure the snapshot is completed 6389 thread::sleep(std::time::Duration::new(10, 0)); 6390 }); 6391 6392 // Shutdown the source VM 6393 let _ = child2.kill(); 6394 let output = child2.wait_with_output().unwrap(); 6395 handle_child_output(r, &output); 6396 6397 // Remove the vhost-user socket file. 6398 Command::new("rm") 6399 .arg("-f") 6400 .arg("/tmp/dpdkvhostclient2") 6401 .output() 6402 .unwrap(); 6403 6404 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6405 // Restore the VM from the snapshot 6406 let mut child2 = GuestCommand::new(&guest2) 6407 .args(["--api-socket", &api_socket_restored]) 6408 .args([ 6409 "--restore", 6410 format!("source_url=file://{}", snapshot_dir).as_str(), 6411 ]) 6412 .capture_output() 6413 .spawn() 6414 .unwrap(); 6415 6416 // Wait for the VM to be restored 6417 thread::sleep(std::time::Duration::new(10, 0)); 6418 6419 let r = std::panic::catch_unwind(|| { 6420 // Resume the VM 6421 assert!(remote_command(&api_socket_restored, "resume", None)); 6422 6423 // Spawn a new netcat listener in the first VM 6424 let guest_ip = guest1.network.guest_ip.clone(); 6425 thread::spawn(move || { 6426 ssh_command_ip( 6427 "nc -l 12345", 6428 &guest_ip, 6429 DEFAULT_SSH_RETRIES, 6430 DEFAULT_SSH_TIMEOUT, 6431 ) 6432 .unwrap(); 6433 }); 6434 6435 // Wait for the server to be listening 6436 thread::sleep(std::time::Duration::new(5, 0)); 6437 6438 // And check the connection is still functional after restore 6439 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6440 }); 6441 6442 let _ = child1.kill(); 6443 let _ = child2.kill(); 6444 6445 let output = child1.wait_with_output().unwrap(); 6446 child2.wait().unwrap(); 6447 6448 cleanup_ovs_dpdk(); 6449 6450 handle_child_output(r, &output); 6451 } 6452 6453 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6454 cleanup_spdk_nvme(); 6455 6456 assert!(exec_host_command_status(&format!( 6457 "mkdir -p {}", 6458 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6459 )) 6460 .success()); 6461 assert!(exec_host_command_status(&format!( 6462 "truncate {} -s 128M", 6463 nvme_dir.join("test-disk.raw").to_str().unwrap() 6464 )) 6465 .success()); 6466 assert!(exec_host_command_status(&format!( 6467 "mkfs.ext4 {}", 6468 nvme_dir.join("test-disk.raw").to_str().unwrap() 6469 )) 6470 .success()); 6471 6472 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6473 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6474 .args(["-i", "0", "-m", "0x1"]) 6475 .spawn() 6476 .unwrap(); 6477 thread::sleep(std::time::Duration::new(2, 0)); 6478 6479 assert!(exec_host_command_status( 6480 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER" 6481 ) 6482 .success()); 6483 assert!(exec_host_command_status(&format!( 6484 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6485 nvme_dir.join("test-disk.raw").to_str().unwrap() 6486 )) 6487 .success()); 6488 assert!(exec_host_command_status( 6489 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6490 ) 6491 .success()); 6492 assert!(exec_host_command_status( 6493 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6494 ) 6495 .success()); 6496 assert!(exec_host_command_status(&format!( 6497 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6498 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6499 )) 6500 .success()); 6501 } 6502 6503 fn cleanup_spdk_nvme() { 6504 exec_host_command_status("pkill -f nvmf_tgt"); 6505 } 6506 6507 #[test] 6508 fn test_vfio_user() { 6509 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6510 let jammy = UbuntuDiskConfig::new(jammy_image); 6511 let guest = Guest::new(Box::new(jammy)); 6512 6513 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6514 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6515 6516 let api_socket = temp_api_path(&guest.tmp_dir); 6517 let mut child = GuestCommand::new(&guest) 6518 .args(["--api-socket", &api_socket]) 6519 .args(["--cpus", "boot=1"]) 6520 .args(["--memory", "size=512M,shared=on,hugepages=on"]) 6521 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6522 .args(["--serial", "tty", "--console", "off"]) 6523 .default_disks() 6524 .default_net() 6525 .capture_output() 6526 .spawn() 6527 .unwrap(); 6528 6529 let r = std::panic::catch_unwind(|| { 6530 guest.wait_vm_boot(None).unwrap(); 6531 6532 // Hotplug the SPDK-NVMe device to the VM 6533 let (cmd_success, cmd_output) = remote_command_w_output( 6534 &api_socket, 6535 "add-user-device", 6536 Some(&format!( 6537 "socket={},id=vfio_user0", 6538 spdk_nvme_dir 6539 .as_path() 6540 .join("nvme-vfio-user/cntrl") 6541 .to_str() 6542 .unwrap(), 6543 )), 6544 ); 6545 assert!(cmd_success); 6546 assert!(String::from_utf8_lossy(&cmd_output) 6547 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6548 6549 thread::sleep(std::time::Duration::new(10, 0)); 6550 6551 // Check both if /dev/nvme exists and if the block size is 128M. 6552 assert_eq!( 6553 guest 6554 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6555 .unwrap() 6556 .trim() 6557 .parse::<u32>() 6558 .unwrap_or_default(), 6559 1 6560 ); 6561 6562 // Check changes persist after reboot 6563 assert_eq!( 6564 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6565 "" 6566 ); 6567 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6568 guest 6569 .ssh_command("echo test123 | sudo tee /mnt/test") 6570 .unwrap(); 6571 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6572 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6573 6574 guest.reboot_linux(0, None); 6575 assert_eq!( 6576 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6577 "" 6578 ); 6579 assert_eq!( 6580 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6581 "test123" 6582 ); 6583 }); 6584 6585 cleanup_spdk_nvme(); 6586 6587 let _ = child.kill(); 6588 let output = child.wait_with_output().unwrap(); 6589 6590 handle_child_output(r, &output); 6591 } 6592 6593 #[test] 6594 #[cfg(target_arch = "x86_64")] 6595 fn test_vdpa_block() { 6596 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6597 if !exec_host_command_status("lsmod | grep vdpa_sim_blk").success() { 6598 return; 6599 } 6600 6601 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6602 let guest = Guest::new(Box::new(focal)); 6603 let api_socket = temp_api_path(&guest.tmp_dir); 6604 6605 let kernel_path = direct_kernel_boot_path(); 6606 6607 let mut child = GuestCommand::new(&guest) 6608 .args(["--cpus", "boot=2"]) 6609 .args(["--memory", "size=512M,hugepages=on"]) 6610 .args(["--kernel", kernel_path.to_str().unwrap()]) 6611 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6612 .default_disks() 6613 .default_net() 6614 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6615 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 6616 .args(["--api-socket", &api_socket]) 6617 .capture_output() 6618 .spawn() 6619 .unwrap(); 6620 6621 let r = std::panic::catch_unwind(|| { 6622 guest.wait_vm_boot(None).unwrap(); 6623 6624 // Check both if /dev/vdc exists and if the block size is 128M. 6625 assert_eq!( 6626 guest 6627 .ssh_command("lsblk | grep vdc | grep -c 128M") 6628 .unwrap() 6629 .trim() 6630 .parse::<u32>() 6631 .unwrap_or_default(), 6632 1 6633 ); 6634 6635 // Check the content of the block device after we wrote to it. 6636 // The vpda-sim-blk should let us read what we previously wrote. 6637 guest 6638 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6639 .unwrap(); 6640 assert_eq!( 6641 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6642 "foobar" 6643 ); 6644 6645 // Hotplug an extra vDPA block device behind the vIOMMU 6646 // Add a new vDPA device to the VM 6647 let (cmd_success, cmd_output) = remote_command_w_output( 6648 &api_socket, 6649 "add-vdpa", 6650 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6651 ); 6652 assert!(cmd_success); 6653 assert!(String::from_utf8_lossy(&cmd_output) 6654 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6655 6656 thread::sleep(std::time::Duration::new(10, 0)); 6657 6658 // Check IOMMU setup 6659 assert!(guest 6660 .does_device_vendor_pair_match("0x1057", "0x1af4") 6661 .unwrap_or_default()); 6662 assert_eq!( 6663 guest 6664 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6665 .unwrap() 6666 .trim(), 6667 "0001:00:01.0" 6668 ); 6669 6670 // Check both if /dev/vdd exists and if the block size is 128M. 6671 assert_eq!( 6672 guest 6673 .ssh_command("lsblk | grep vdd | grep -c 128M") 6674 .unwrap() 6675 .trim() 6676 .parse::<u32>() 6677 .unwrap_or_default(), 6678 1 6679 ); 6680 6681 // Write some content to the block device we've just plugged. 6682 guest 6683 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6684 .unwrap(); 6685 6686 // Check we can read the content back. 6687 assert_eq!( 6688 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6689 "foobar" 6690 ); 6691 6692 // Unplug the device 6693 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6694 assert!(cmd_success); 6695 thread::sleep(std::time::Duration::new(10, 0)); 6696 6697 // Check /dev/vdd doesn't exist anymore 6698 assert_eq!( 6699 guest 6700 .ssh_command("lsblk | grep -c vdd || true") 6701 .unwrap() 6702 .trim() 6703 .parse::<u32>() 6704 .unwrap_or(1), 6705 0 6706 ); 6707 }); 6708 6709 let _ = child.kill(); 6710 let output = child.wait_with_output().unwrap(); 6711 6712 handle_child_output(r, &output); 6713 } 6714 6715 #[test] 6716 #[cfg(target_arch = "x86_64")] 6717 fn test_vdpa_net() { 6718 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6719 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6720 return; 6721 } 6722 6723 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6724 let guest = Guest::new(Box::new(focal)); 6725 6726 let kernel_path = direct_kernel_boot_path(); 6727 6728 let mut child = GuestCommand::new(&guest) 6729 .args(["--cpus", "boot=2"]) 6730 .args(["--memory", "size=512M,hugepages=on"]) 6731 .args(["--kernel", kernel_path.to_str().unwrap()]) 6732 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6733 .default_disks() 6734 .default_net() 6735 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 6736 .capture_output() 6737 .spawn() 6738 .unwrap(); 6739 6740 let r = std::panic::catch_unwind(|| { 6741 guest.wait_vm_boot(None).unwrap(); 6742 6743 // Check we can find network interface related to vDPA device 6744 assert_eq!( 6745 guest 6746 .ssh_command("ip -o link | grep -c ens6") 6747 .unwrap() 6748 .trim() 6749 .parse::<u32>() 6750 .unwrap_or(0), 6751 1 6752 ); 6753 6754 guest 6755 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 6756 .unwrap(); 6757 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 6758 6759 // Check there is no packet yet on both TX/RX of the network interface 6760 assert_eq!( 6761 guest 6762 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 6763 .unwrap() 6764 .trim() 6765 .parse::<u32>() 6766 .unwrap_or(0), 6767 2 6768 ); 6769 6770 // Send 6 packets with ping command 6771 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 6772 6773 // Check we can find 6 packets on both TX/RX of the network interface 6774 assert_eq!( 6775 guest 6776 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 6777 .unwrap() 6778 .trim() 6779 .parse::<u32>() 6780 .unwrap_or(0), 6781 2 6782 ); 6783 6784 // No need to check for hotplug as we already tested it through 6785 // test_vdpa_block() 6786 }); 6787 6788 let _ = child.kill(); 6789 let output = child.wait_with_output().unwrap(); 6790 6791 handle_child_output(r, &output); 6792 } 6793 } 6794 6795 mod common_sequential { 6796 use crate::*; 6797 6798 #[test] 6799 fn test_memory_mergeable_on() { 6800 test_memory_mergeable(true) 6801 } 6802 } 6803 6804 mod windows { 6805 use crate::*; 6806 use once_cell::sync::Lazy; 6807 6808 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 6809 6810 struct WindowsGuest { 6811 guest: Guest, 6812 auth: PasswordAuth, 6813 } 6814 6815 trait FsType { 6816 const FS_FAT: u8; 6817 const FS_NTFS: u8; 6818 } 6819 impl FsType for WindowsGuest { 6820 const FS_FAT: u8 = 0; 6821 const FS_NTFS: u8 = 1; 6822 } 6823 6824 impl WindowsGuest { 6825 fn new() -> Self { 6826 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 6827 let guest = Guest::new(Box::new(disk)); 6828 let auth = PasswordAuth { 6829 username: String::from("administrator"), 6830 password: String::from("Admin123"), 6831 }; 6832 6833 WindowsGuest { guest, auth } 6834 } 6835 6836 fn guest(&self) -> &Guest { 6837 &self.guest 6838 } 6839 6840 fn ssh_cmd(&self, cmd: &str) -> String { 6841 ssh_command_ip_with_auth( 6842 cmd, 6843 &self.auth, 6844 &self.guest.network.guest_ip, 6845 DEFAULT_SSH_RETRIES, 6846 DEFAULT_SSH_TIMEOUT, 6847 ) 6848 .unwrap() 6849 } 6850 6851 fn cpu_count(&self) -> u8 { 6852 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 6853 .trim() 6854 .parse::<u8>() 6855 .unwrap_or(0) 6856 } 6857 6858 fn ram_size(&self) -> usize { 6859 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 6860 .trim() 6861 .parse::<usize>() 6862 .unwrap_or(0) 6863 } 6864 6865 fn netdev_count(&self) -> u8 { 6866 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6867 .trim() 6868 .parse::<u8>() 6869 .unwrap_or(0) 6870 } 6871 6872 fn disk_count(&self) -> u8 { 6873 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 6874 .trim() 6875 .parse::<u8>() 6876 .unwrap_or(0) 6877 } 6878 6879 fn reboot(&self) { 6880 let _ = self.ssh_cmd("shutdown /r /t 0"); 6881 } 6882 6883 fn shutdown(&self) { 6884 let _ = self.ssh_cmd("shutdown /s /t 0"); 6885 } 6886 6887 fn run_dnsmasq(&self) -> std::process::Child { 6888 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 6889 let dhcp_host = format!( 6890 "--dhcp-host={},{}", 6891 self.guest.network.guest_mac, self.guest.network.guest_ip 6892 ); 6893 let dhcp_range = format!( 6894 "--dhcp-range=eth,{},{}", 6895 self.guest.network.guest_ip, self.guest.network.guest_ip 6896 ); 6897 6898 Command::new("dnsmasq") 6899 .arg("--no-daemon") 6900 .arg("--log-queries") 6901 .arg(listen_address.as_str()) 6902 .arg("--except-interface=lo") 6903 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 6904 .arg("--conf-file=/dev/null") 6905 .arg(dhcp_host.as_str()) 6906 .arg(dhcp_range.as_str()) 6907 .spawn() 6908 .unwrap() 6909 } 6910 6911 // TODO Cleanup image file explicitly after test, if there's some space issues. 6912 fn disk_new(&self, fs: u8, sz: usize) -> String { 6913 let mut guard = NEXT_DISK_ID.lock().unwrap(); 6914 let id = *guard; 6915 *guard = id + 1; 6916 6917 let img = PathBuf::from(format!("/tmp/test-hotplug-{}.raw", id)); 6918 let _ = fs::remove_file(&img); 6919 6920 // Create an image file 6921 let out = Command::new("qemu-img") 6922 .args([ 6923 "create", 6924 "-f", 6925 "raw", 6926 img.to_str().unwrap(), 6927 format!("{}m", sz).as_str(), 6928 ]) 6929 .output() 6930 .expect("qemu-img command failed") 6931 .stdout; 6932 println!("{:?}", out); 6933 6934 // Associate image to a loop device 6935 let out = Command::new("losetup") 6936 .args(["--show", "-f", img.to_str().unwrap()]) 6937 .output() 6938 .expect("failed to create loop device") 6939 .stdout; 6940 let _tmp = String::from_utf8_lossy(&out); 6941 let loop_dev = _tmp.trim(); 6942 println!("{:?}", out); 6943 6944 // Create a partition table 6945 // echo 'type=7' | sudo sfdisk "${LOOP}" 6946 let mut child = Command::new("sfdisk") 6947 .args([loop_dev]) 6948 .stdin(Stdio::piped()) 6949 .spawn() 6950 .unwrap(); 6951 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 6952 stdin 6953 .write_all("type=7".as_bytes()) 6954 .expect("failed to write stdin"); 6955 let out = child.wait_with_output().expect("sfdisk failed").stdout; 6956 println!("{:?}", out); 6957 6958 // Disengage the loop device 6959 let out = Command::new("losetup") 6960 .args(["-d", loop_dev]) 6961 .output() 6962 .expect("loop device not found") 6963 .stdout; 6964 println!("{:?}", out); 6965 6966 // Re-associate loop device pointing to the partition only 6967 let out = Command::new("losetup") 6968 .args([ 6969 "--show", 6970 "--offset", 6971 (512 * 2048).to_string().as_str(), 6972 "-f", 6973 img.to_str().unwrap(), 6974 ]) 6975 .output() 6976 .expect("failed to create loop device") 6977 .stdout; 6978 let _tmp = String::from_utf8_lossy(&out); 6979 let loop_dev = _tmp.trim(); 6980 println!("{:?}", out); 6981 6982 // Create filesystem. 6983 let fs_cmd = match fs { 6984 WindowsGuest::FS_FAT => "mkfs.msdos", 6985 WindowsGuest::FS_NTFS => "mkfs.ntfs", 6986 _ => panic!("Unknown filesystem type '{}'", fs), 6987 }; 6988 let out = Command::new(fs_cmd) 6989 .args([&loop_dev]) 6990 .output() 6991 .unwrap_or_else(|_| panic!("{} failed", fs_cmd)) 6992 .stdout; 6993 println!("{:?}", out); 6994 6995 // Disengage the loop device 6996 let out = Command::new("losetup") 6997 .args(["-d", loop_dev]) 6998 .output() 6999 .unwrap_or_else(|_| panic!("loop device '{}' not found", loop_dev)) 7000 .stdout; 7001 println!("{:?}", out); 7002 7003 img.to_str().unwrap().to_string() 7004 } 7005 7006 fn disks_set_rw(&self) { 7007 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 7008 } 7009 7010 fn disks_online(&self) { 7011 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 7012 } 7013 7014 fn disk_file_put(&self, fname: &str, data: &str) { 7015 let _ = self.ssh_cmd(&format!( 7016 "powershell -Command \"'{}' | Set-Content -Path {}\"", 7017 data, fname 7018 )); 7019 } 7020 7021 fn disk_file_read(&self, fname: &str) -> String { 7022 self.ssh_cmd(&format!( 7023 "powershell -Command \"Get-Content -Path {}\"", 7024 fname 7025 )) 7026 } 7027 7028 fn wait_for_boot(&self) -> bool { 7029 let cmd = "dir /b c:\\ | find \"Windows\""; 7030 let tmo_max = 180; 7031 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 7032 // interval must be small. 7033 let tmo_int = 2; 7034 let out = ssh_command_ip_with_auth( 7035 cmd, 7036 &self.auth, 7037 &self.guest.network.guest_ip, 7038 { 7039 let mut ret = 1; 7040 let mut tmo_acc = 0; 7041 loop { 7042 tmo_acc += tmo_int * ret; 7043 if tmo_acc >= tmo_max { 7044 break; 7045 } 7046 ret += 1; 7047 } 7048 ret 7049 }, 7050 tmo_int, 7051 ) 7052 .unwrap(); 7053 7054 if "Windows" == out.trim() { 7055 return true; 7056 } 7057 7058 false 7059 } 7060 } 7061 7062 fn vcpu_threads_count(pid: u32) -> u8 { 7063 // ps -T -p 12345 | grep vcpu | wc -l 7064 let out = Command::new("ps") 7065 .args(["-T", "-p", format!("{}", pid).as_str()]) 7066 .output() 7067 .expect("ps command failed") 7068 .stdout; 7069 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 7070 } 7071 7072 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 7073 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 7074 let out = Command::new("ps") 7075 .args(["-T", "-p", format!("{}", pid).as_str()]) 7076 .output() 7077 .expect("ps command failed") 7078 .stdout; 7079 let mut n = 0; 7080 String::from_utf8_lossy(&out) 7081 .split_whitespace() 7082 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 7083 n 7084 } 7085 7086 fn disk_ctrl_threads_count(pid: u32) -> u8 { 7087 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 7088 let out = Command::new("ps") 7089 .args(["-T", "-p", format!("{}", pid).as_str()]) 7090 .output() 7091 .expect("ps command failed") 7092 .stdout; 7093 let mut n = 0; 7094 String::from_utf8_lossy(&out) 7095 .split_whitespace() 7096 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 7097 n 7098 } 7099 7100 #[test] 7101 fn test_windows_guest() { 7102 let windows_guest = WindowsGuest::new(); 7103 7104 let mut child = GuestCommand::new(windows_guest.guest()) 7105 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7106 .args(["--memory", "size=4G"]) 7107 .args(["--kernel", edk2_path().to_str().unwrap()]) 7108 .args(["--serial", "tty"]) 7109 .args(["--console", "off"]) 7110 .default_disks() 7111 .default_net() 7112 .capture_output() 7113 .spawn() 7114 .unwrap(); 7115 7116 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7117 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7118 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7119 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7120 7121 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7122 7123 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7124 7125 let r = std::panic::catch_unwind(|| { 7126 // Wait to make sure Windows boots up 7127 assert!(windows_guest.wait_for_boot()); 7128 7129 windows_guest.shutdown(); 7130 }); 7131 7132 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7133 let _ = child.kill(); 7134 let output = child.wait_with_output().unwrap(); 7135 7136 let _ = child_dnsmasq.kill(); 7137 let _ = child_dnsmasq.wait(); 7138 7139 handle_child_output(r, &output); 7140 } 7141 7142 #[test] 7143 fn test_windows_guest_multiple_queues() { 7144 let windows_guest = WindowsGuest::new(); 7145 7146 let mut ovmf_path = dirs::home_dir().unwrap(); 7147 ovmf_path.push("workloads"); 7148 ovmf_path.push(OVMF_NAME); 7149 7150 let mut child = GuestCommand::new(windows_guest.guest()) 7151 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 7152 .args(["--memory", "size=4G"]) 7153 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7154 .args(["--serial", "tty"]) 7155 .args(["--console", "off"]) 7156 .args([ 7157 "--disk", 7158 format!( 7159 "path={},num_queues=4", 7160 windows_guest 7161 .guest() 7162 .disk_config 7163 .disk(DiskType::OperatingSystem) 7164 .unwrap() 7165 ) 7166 .as_str(), 7167 ]) 7168 .args([ 7169 "--net", 7170 format!( 7171 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 7172 windows_guest.guest().network.guest_mac, 7173 windows_guest.guest().network.host_ip 7174 ) 7175 .as_str(), 7176 ]) 7177 .capture_output() 7178 .spawn() 7179 .unwrap(); 7180 7181 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7182 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7183 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7184 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7185 7186 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7187 7188 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7189 7190 let r = std::panic::catch_unwind(|| { 7191 // Wait to make sure Windows boots up 7192 assert!(windows_guest.wait_for_boot()); 7193 7194 windows_guest.shutdown(); 7195 }); 7196 7197 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7198 let _ = child.kill(); 7199 let output = child.wait_with_output().unwrap(); 7200 7201 let _ = child_dnsmasq.kill(); 7202 let _ = child_dnsmasq.wait(); 7203 7204 handle_child_output(r, &output); 7205 } 7206 7207 #[test] 7208 #[cfg(not(feature = "mshv"))] 7209 #[ignore = "See #4327"] 7210 fn test_windows_guest_snapshot_restore() { 7211 let windows_guest = WindowsGuest::new(); 7212 7213 let mut ovmf_path = dirs::home_dir().unwrap(); 7214 ovmf_path.push("workloads"); 7215 ovmf_path.push(OVMF_NAME); 7216 7217 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7218 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 7219 7220 let mut child = GuestCommand::new(windows_guest.guest()) 7221 .args(["--api-socket", &api_socket_source]) 7222 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7223 .args(["--memory", "size=4G"]) 7224 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7225 .args(["--serial", "tty"]) 7226 .args(["--console", "off"]) 7227 .default_disks() 7228 .default_net() 7229 .capture_output() 7230 .spawn() 7231 .unwrap(); 7232 7233 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7234 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7235 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7236 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7237 7238 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7239 7240 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7241 7242 // Wait to make sure Windows boots up 7243 assert!(windows_guest.wait_for_boot()); 7244 7245 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 7246 7247 // Pause the VM 7248 assert!(remote_command(&api_socket_source, "pause", None)); 7249 7250 // Take a snapshot from the VM 7251 assert!(remote_command( 7252 &api_socket_source, 7253 "snapshot", 7254 Some(format!("file://{}", snapshot_dir).as_str()), 7255 )); 7256 7257 // Wait to make sure the snapshot is completed 7258 thread::sleep(std::time::Duration::new(30, 0)); 7259 7260 let _ = child.kill(); 7261 child.wait().unwrap(); 7262 7263 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 7264 7265 // Restore the VM from the snapshot 7266 let mut child = GuestCommand::new(windows_guest.guest()) 7267 .args(["--api-socket", &api_socket_restored]) 7268 .args([ 7269 "--restore", 7270 format!("source_url=file://{}", snapshot_dir).as_str(), 7271 ]) 7272 .capture_output() 7273 .spawn() 7274 .unwrap(); 7275 7276 // Wait for the VM to be restored 7277 thread::sleep(std::time::Duration::new(20, 0)); 7278 7279 let r = std::panic::catch_unwind(|| { 7280 // Resume the VM 7281 assert!(remote_command(&api_socket_restored, "resume", None)); 7282 7283 windows_guest.shutdown(); 7284 }); 7285 7286 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7287 let _ = child.kill(); 7288 let output = child.wait_with_output().unwrap(); 7289 7290 let _ = child_dnsmasq.kill(); 7291 let _ = child_dnsmasq.wait(); 7292 7293 handle_child_output(r, &output); 7294 } 7295 7296 #[test] 7297 #[cfg(not(feature = "mshv"))] 7298 #[cfg(not(target_arch = "aarch64"))] 7299 fn test_windows_guest_cpu_hotplug() { 7300 let windows_guest = WindowsGuest::new(); 7301 7302 let mut ovmf_path = dirs::home_dir().unwrap(); 7303 ovmf_path.push("workloads"); 7304 ovmf_path.push(OVMF_NAME); 7305 7306 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7307 let api_socket = temp_api_path(&tmp_dir); 7308 7309 let mut child = GuestCommand::new(windows_guest.guest()) 7310 .args(["--api-socket", &api_socket]) 7311 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 7312 .args(["--memory", "size=4G"]) 7313 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7314 .args(["--serial", "tty"]) 7315 .args(["--console", "off"]) 7316 .default_disks() 7317 .default_net() 7318 .capture_output() 7319 .spawn() 7320 .unwrap(); 7321 7322 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7323 7324 let r = std::panic::catch_unwind(|| { 7325 // Wait to make sure Windows boots up 7326 assert!(windows_guest.wait_for_boot()); 7327 7328 let vcpu_num = 2; 7329 // Check the initial number of CPUs the guest sees 7330 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7331 // Check the initial number of vcpu threads in the CH process 7332 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7333 7334 let vcpu_num = 6; 7335 // Hotplug some CPUs 7336 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7337 // Wait to make sure CPUs are added 7338 thread::sleep(std::time::Duration::new(10, 0)); 7339 // Check the guest sees the correct number 7340 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7341 // Check the CH process has the correct number of vcpu threads 7342 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7343 7344 let vcpu_num = 4; 7345 // Remove some CPUs. Note that Windows doesn't support hot-remove. 7346 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7347 // Wait to make sure CPUs are removed 7348 thread::sleep(std::time::Duration::new(10, 0)); 7349 // Reboot to let Windows catch up 7350 windows_guest.reboot(); 7351 // Wait to make sure Windows completely rebooted 7352 thread::sleep(std::time::Duration::new(60, 0)); 7353 // Check the guest sees the correct number 7354 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7355 // Check the CH process has the correct number of vcpu threads 7356 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7357 7358 windows_guest.shutdown(); 7359 }); 7360 7361 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7362 let _ = child.kill(); 7363 let output = child.wait_with_output().unwrap(); 7364 7365 let _ = child_dnsmasq.kill(); 7366 let _ = child_dnsmasq.wait(); 7367 7368 handle_child_output(r, &output); 7369 } 7370 7371 #[test] 7372 #[cfg(not(feature = "mshv"))] 7373 #[cfg(not(target_arch = "aarch64"))] 7374 fn test_windows_guest_ram_hotplug() { 7375 let windows_guest = WindowsGuest::new(); 7376 7377 let mut ovmf_path = dirs::home_dir().unwrap(); 7378 ovmf_path.push("workloads"); 7379 ovmf_path.push(OVMF_NAME); 7380 7381 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7382 let api_socket = temp_api_path(&tmp_dir); 7383 7384 let mut child = GuestCommand::new(windows_guest.guest()) 7385 .args(["--api-socket", &api_socket]) 7386 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7387 .args(["--memory", "size=2G,hotplug_size=5G"]) 7388 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7389 .args(["--serial", "tty"]) 7390 .args(["--console", "off"]) 7391 .default_disks() 7392 .default_net() 7393 .capture_output() 7394 .spawn() 7395 .unwrap(); 7396 7397 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7398 7399 let r = std::panic::catch_unwind(|| { 7400 // Wait to make sure Windows boots up 7401 assert!(windows_guest.wait_for_boot()); 7402 7403 let ram_size = 2 * 1024 * 1024 * 1024; 7404 // Check the initial number of RAM the guest sees 7405 let current_ram_size = windows_guest.ram_size(); 7406 // This size seems to be reserved by the system and thus the 7407 // reported amount differs by this constant value. 7408 let reserved_ram_size = ram_size - current_ram_size; 7409 // Verify that there's not more than 4mb constant diff wasted 7410 // by the reserved ram. 7411 assert!(reserved_ram_size < 4 * 1024 * 1024); 7412 7413 let ram_size = 4 * 1024 * 1024 * 1024; 7414 // Hotplug some RAM 7415 resize_command(&api_socket, None, Some(ram_size), None, None); 7416 // Wait to make sure RAM has been added 7417 thread::sleep(std::time::Duration::new(10, 0)); 7418 // Check the guest sees the correct number 7419 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7420 7421 let ram_size = 3 * 1024 * 1024 * 1024; 7422 // Unplug some RAM. Note that hot-remove most likely won't work. 7423 resize_command(&api_socket, None, Some(ram_size), None, None); 7424 // Wait to make sure RAM has been added 7425 thread::sleep(std::time::Duration::new(10, 0)); 7426 // Reboot to let Windows catch up 7427 windows_guest.reboot(); 7428 // Wait to make sure guest completely rebooted 7429 thread::sleep(std::time::Duration::new(60, 0)); 7430 // Check the guest sees the correct number 7431 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7432 7433 windows_guest.shutdown(); 7434 }); 7435 7436 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7437 let _ = child.kill(); 7438 let output = child.wait_with_output().unwrap(); 7439 7440 let _ = child_dnsmasq.kill(); 7441 let _ = child_dnsmasq.wait(); 7442 7443 handle_child_output(r, &output); 7444 } 7445 7446 #[test] 7447 #[cfg(not(feature = "mshv"))] 7448 fn test_windows_guest_netdev_hotplug() { 7449 let windows_guest = WindowsGuest::new(); 7450 7451 let mut ovmf_path = dirs::home_dir().unwrap(); 7452 ovmf_path.push("workloads"); 7453 ovmf_path.push(OVMF_NAME); 7454 7455 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7456 let api_socket = temp_api_path(&tmp_dir); 7457 7458 let mut child = GuestCommand::new(windows_guest.guest()) 7459 .args(["--api-socket", &api_socket]) 7460 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7461 .args(["--memory", "size=4G"]) 7462 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7463 .args(["--serial", "tty"]) 7464 .args(["--console", "off"]) 7465 .default_disks() 7466 .default_net() 7467 .capture_output() 7468 .spawn() 7469 .unwrap(); 7470 7471 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7472 7473 let r = std::panic::catch_unwind(|| { 7474 // Wait to make sure Windows boots up 7475 assert!(windows_guest.wait_for_boot()); 7476 7477 // Initially present network device 7478 let netdev_num = 1; 7479 assert_eq!(windows_guest.netdev_count(), netdev_num); 7480 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7481 7482 // Hotplug network device 7483 let (cmd_success, cmd_output) = remote_command_w_output( 7484 &api_socket, 7485 "add-net", 7486 Some(windows_guest.guest().default_net_string().as_str()), 7487 ); 7488 assert!(cmd_success); 7489 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 7490 thread::sleep(std::time::Duration::new(5, 0)); 7491 // Verify the device is on the system 7492 let netdev_num = 2; 7493 assert_eq!(windows_guest.netdev_count(), netdev_num); 7494 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7495 7496 // Remove network device 7497 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 7498 assert!(cmd_success); 7499 thread::sleep(std::time::Duration::new(5, 0)); 7500 // Verify the device has been removed 7501 let netdev_num = 1; 7502 assert_eq!(windows_guest.netdev_count(), netdev_num); 7503 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7504 7505 windows_guest.shutdown(); 7506 }); 7507 7508 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7509 let _ = child.kill(); 7510 let output = child.wait_with_output().unwrap(); 7511 7512 let _ = child_dnsmasq.kill(); 7513 let _ = child_dnsmasq.wait(); 7514 7515 handle_child_output(r, &output); 7516 } 7517 7518 #[test] 7519 #[cfg(not(feature = "mshv"))] 7520 #[cfg(not(target_arch = "aarch64"))] 7521 fn test_windows_guest_disk_hotplug() { 7522 let windows_guest = WindowsGuest::new(); 7523 7524 let mut ovmf_path = dirs::home_dir().unwrap(); 7525 ovmf_path.push("workloads"); 7526 ovmf_path.push(OVMF_NAME); 7527 7528 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7529 let api_socket = temp_api_path(&tmp_dir); 7530 7531 let mut child = GuestCommand::new(windows_guest.guest()) 7532 .args(["--api-socket", &api_socket]) 7533 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7534 .args(["--memory", "size=4G"]) 7535 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7536 .args(["--serial", "tty"]) 7537 .args(["--console", "off"]) 7538 .default_disks() 7539 .default_net() 7540 .capture_output() 7541 .spawn() 7542 .unwrap(); 7543 7544 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7545 7546 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 7547 7548 let r = std::panic::catch_unwind(|| { 7549 // Wait to make sure Windows boots up 7550 assert!(windows_guest.wait_for_boot()); 7551 7552 // Initially present disk device 7553 let disk_num = 1; 7554 assert_eq!(windows_guest.disk_count(), disk_num); 7555 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7556 7557 // Hotplug disk device 7558 let (cmd_success, cmd_output) = remote_command_w_output( 7559 &api_socket, 7560 "add-disk", 7561 Some(format!("path={},readonly=off", disk).as_str()), 7562 ); 7563 assert!(cmd_success); 7564 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 7565 thread::sleep(std::time::Duration::new(5, 0)); 7566 // Online disk device 7567 windows_guest.disks_set_rw(); 7568 windows_guest.disks_online(); 7569 // Verify the device is on the system 7570 let disk_num = 2; 7571 assert_eq!(windows_guest.disk_count(), disk_num); 7572 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7573 7574 let data = "hello"; 7575 let fname = "d:\\world"; 7576 windows_guest.disk_file_put(fname, data); 7577 7578 // Unmount disk device 7579 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 7580 assert!(cmd_success); 7581 thread::sleep(std::time::Duration::new(5, 0)); 7582 // Verify the device has been removed 7583 let disk_num = 1; 7584 assert_eq!(windows_guest.disk_count(), disk_num); 7585 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7586 7587 // Remount and check the file exists with the expected contents 7588 let (cmd_success, _cmd_output) = remote_command_w_output( 7589 &api_socket, 7590 "add-disk", 7591 Some(format!("path={},readonly=off", disk).as_str()), 7592 ); 7593 assert!(cmd_success); 7594 thread::sleep(std::time::Duration::new(5, 0)); 7595 let out = windows_guest.disk_file_read(fname); 7596 assert_eq!(data, out.trim()); 7597 7598 // Intentionally no unmount, it'll happen at shutdown. 7599 7600 windows_guest.shutdown(); 7601 }); 7602 7603 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7604 let _ = child.kill(); 7605 let output = child.wait_with_output().unwrap(); 7606 7607 let _ = child_dnsmasq.kill(); 7608 let _ = child_dnsmasq.wait(); 7609 7610 handle_child_output(r, &output); 7611 } 7612 7613 #[test] 7614 #[cfg(not(feature = "mshv"))] 7615 #[cfg(not(target_arch = "aarch64"))] 7616 fn test_windows_guest_disk_hotplug_multi() { 7617 let windows_guest = WindowsGuest::new(); 7618 7619 let mut ovmf_path = dirs::home_dir().unwrap(); 7620 ovmf_path.push("workloads"); 7621 ovmf_path.push(OVMF_NAME); 7622 7623 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7624 let api_socket = temp_api_path(&tmp_dir); 7625 7626 let mut child = GuestCommand::new(windows_guest.guest()) 7627 .args(["--api-socket", &api_socket]) 7628 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7629 .args(["--memory", "size=2G"]) 7630 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7631 .args(["--serial", "tty"]) 7632 .args(["--console", "off"]) 7633 .default_disks() 7634 .default_net() 7635 .capture_output() 7636 .spawn() 7637 .unwrap(); 7638 7639 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7640 7641 // Predefined data to used at various test stages 7642 let disk_test_data: [[String; 4]; 2] = [ 7643 [ 7644 "_disk2".to_string(), 7645 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 7646 "d:\\world".to_string(), 7647 "hello".to_string(), 7648 ], 7649 [ 7650 "_disk3".to_string(), 7651 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 7652 "e:\\hello".to_string(), 7653 "world".to_string(), 7654 ], 7655 ]; 7656 7657 let r = std::panic::catch_unwind(|| { 7658 // Wait to make sure Windows boots up 7659 assert!(windows_guest.wait_for_boot()); 7660 7661 // Initially present disk device 7662 let disk_num = 1; 7663 assert_eq!(windows_guest.disk_count(), disk_num); 7664 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7665 7666 for it in &disk_test_data { 7667 let disk_id = it[0].as_str(); 7668 let disk = it[1].as_str(); 7669 // Hotplug disk device 7670 let (cmd_success, cmd_output) = remote_command_w_output( 7671 &api_socket, 7672 "add-disk", 7673 Some(format!("path={},readonly=off", disk).as_str()), 7674 ); 7675 assert!(cmd_success); 7676 assert!(String::from_utf8_lossy(&cmd_output) 7677 .contains(format!("\"id\":\"{}\"", disk_id).as_str())); 7678 thread::sleep(std::time::Duration::new(5, 0)); 7679 // Online disk devices 7680 windows_guest.disks_set_rw(); 7681 windows_guest.disks_online(); 7682 } 7683 // Verify the devices are on the system 7684 let disk_num = (disk_test_data.len() + 1) as u8; 7685 assert_eq!(windows_guest.disk_count(), disk_num); 7686 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7687 7688 // Put test data 7689 for it in &disk_test_data { 7690 let fname = it[2].as_str(); 7691 let data = it[3].as_str(); 7692 windows_guest.disk_file_put(fname, data); 7693 } 7694 7695 // Unmount disk devices 7696 for it in &disk_test_data { 7697 let disk_id = it[0].as_str(); 7698 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 7699 assert!(cmd_success); 7700 thread::sleep(std::time::Duration::new(5, 0)); 7701 } 7702 7703 // Verify the devices have been removed 7704 let disk_num = 1; 7705 assert_eq!(windows_guest.disk_count(), disk_num); 7706 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 7707 7708 // Remount 7709 for it in &disk_test_data { 7710 let disk = it[1].as_str(); 7711 let (cmd_success, _cmd_output) = remote_command_w_output( 7712 &api_socket, 7713 "add-disk", 7714 Some(format!("path={},readonly=off", disk).as_str()), 7715 ); 7716 assert!(cmd_success); 7717 thread::sleep(std::time::Duration::new(5, 0)); 7718 } 7719 7720 // Check the files exists with the expected contents 7721 for it in &disk_test_data { 7722 let fname = it[2].as_str(); 7723 let data = it[3].as_str(); 7724 let out = windows_guest.disk_file_read(fname); 7725 assert_eq!(data, out.trim()); 7726 } 7727 7728 // Intentionally no unmount, it'll happen at shutdown. 7729 7730 windows_guest.shutdown(); 7731 }); 7732 7733 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7734 let _ = child.kill(); 7735 let output = child.wait_with_output().unwrap(); 7736 7737 let _ = child_dnsmasq.kill(); 7738 let _ = child_dnsmasq.wait(); 7739 7740 handle_child_output(r, &output); 7741 } 7742 7743 #[test] 7744 #[cfg(not(feature = "mshv"))] 7745 #[cfg(not(target_arch = "aarch64"))] 7746 fn test_windows_guest_netdev_multi() { 7747 let windows_guest = WindowsGuest::new(); 7748 7749 let mut ovmf_path = dirs::home_dir().unwrap(); 7750 ovmf_path.push("workloads"); 7751 ovmf_path.push(OVMF_NAME); 7752 7753 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7754 let api_socket = temp_api_path(&tmp_dir); 7755 7756 let mut child = GuestCommand::new(windows_guest.guest()) 7757 .args(["--api-socket", &api_socket]) 7758 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7759 .args(["--memory", "size=4G"]) 7760 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7761 .args(["--serial", "tty"]) 7762 .args(["--console", "off"]) 7763 .default_disks() 7764 // The multi net dev config is borrowed from test_multiple_network_interfaces 7765 .args([ 7766 "--net", 7767 windows_guest.guest().default_net_string().as_str(), 7768 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 7769 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 7770 ]) 7771 .capture_output() 7772 .spawn() 7773 .unwrap(); 7774 7775 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7776 7777 let r = std::panic::catch_unwind(|| { 7778 // Wait to make sure Windows boots up 7779 assert!(windows_guest.wait_for_boot()); 7780 7781 let netdev_num = 3; 7782 assert_eq!(windows_guest.netdev_count(), netdev_num); 7783 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7784 7785 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 7786 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 7787 7788 windows_guest.shutdown(); 7789 }); 7790 7791 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7792 let _ = child.kill(); 7793 let output = child.wait_with_output().unwrap(); 7794 7795 let _ = child_dnsmasq.kill(); 7796 let _ = child_dnsmasq.wait(); 7797 7798 handle_child_output(r, &output); 7799 } 7800 } 7801 7802 #[cfg(target_arch = "x86_64")] 7803 mod sgx { 7804 use crate::*; 7805 7806 #[test] 7807 fn test_sgx() { 7808 let focal = UbuntuDiskConfig::new(FOCAL_SGX_IMAGE_NAME.to_string()); 7809 let guest = Guest::new(Box::new(focal)); 7810 let mut workload_path = dirs::home_dir().unwrap(); 7811 workload_path.push("workloads"); 7812 7813 let mut kernel_path = workload_path; 7814 kernel_path.push("vmlinux_w_sgx"); 7815 7816 let mut child = GuestCommand::new(&guest) 7817 .args(["--cpus", "boot=1"]) 7818 .args(["--memory", "size=512M"]) 7819 .args(["--kernel", kernel_path.to_str().unwrap()]) 7820 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7821 .default_disks() 7822 .default_net() 7823 .args(["--sgx-epc", "id=epc0,size=64M"]) 7824 .capture_output() 7825 .spawn() 7826 .unwrap(); 7827 7828 let r = std::panic::catch_unwind(|| { 7829 guest.wait_vm_boot(None).unwrap(); 7830 7831 // Check if SGX is correctly detected in the guest. 7832 guest.check_sgx_support().unwrap(); 7833 7834 // Validate the SGX EPC section is 64MiB. 7835 assert_eq!( 7836 guest 7837 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 7838 .unwrap() 7839 .trim(), 7840 "0x0000000004000000" 7841 ); 7842 7843 // Run a test relying on SGX enclaves and check if it runs 7844 // successfully. 7845 assert!(guest 7846 .ssh_command("cd /linux-sgx/SampleCode/LocalAttestation/bin/ && sudo ./app") 7847 .unwrap() 7848 .trim() 7849 .contains( 7850 "succeed to load enclaves.\nsucceed to \ 7851 establish secure channel.\nSucceed to exchange \ 7852 secure message...\nSucceed to close Session..." 7853 )); 7854 }); 7855 7856 let _ = child.kill(); 7857 let output = child.wait_with_output().unwrap(); 7858 7859 handle_child_output(r, &output); 7860 } 7861 } 7862 7863 #[cfg(target_arch = "x86_64")] 7864 mod vfio { 7865 use crate::*; 7866 7867 #[test] 7868 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 7869 // backed networking interfaces, bound through a simple bridge on the host. 7870 // So if the nested cloud-hypervisor succeeds in getting a directly 7871 // assigned interface from its cloud-hypervisor host, we should be able to 7872 // ssh into it, and verify that it's running with the right kernel command 7873 // line (We tag the command line from cloud-hypervisor for that purpose). 7874 // The third device is added to validate that hotplug works correctly since 7875 // it is being added to the L2 VM through hotplugging mechanism. 7876 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 7877 // vfio device support 7878 fn test_vfio() { 7879 setup_vfio_network_interfaces(); 7880 7881 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7882 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 7883 7884 let mut workload_path = dirs::home_dir().unwrap(); 7885 workload_path.push("workloads"); 7886 7887 let kernel_path = direct_kernel_boot_path(); 7888 7889 let mut vfio_path = workload_path.clone(); 7890 vfio_path.push("vfio"); 7891 7892 let mut cloud_init_vfio_base_path = vfio_path.clone(); 7893 cloud_init_vfio_base_path.push("cloudinit.img"); 7894 7895 // We copy our cloudinit into the vfio mount point, for the nested 7896 // cloud-hypervisor guest to use. 7897 rate_limited_copy( 7898 &guest.disk_config.disk(DiskType::CloudInit).unwrap(), 7899 &cloud_init_vfio_base_path, 7900 ) 7901 .expect("copying of cloud-init disk failed"); 7902 7903 let mut vfio_disk_path = workload_path.clone(); 7904 vfio_disk_path.push("vfio.img"); 7905 7906 // Create the vfio disk image 7907 let output = Command::new("mkfs.ext4") 7908 .arg("-d") 7909 .arg(vfio_path.to_str().unwrap()) 7910 .arg(vfio_disk_path.to_str().unwrap()) 7911 .arg("2g") 7912 .output() 7913 .unwrap(); 7914 if !output.status.success() { 7915 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 7916 panic!("mkfs.ext4 command generated an error"); 7917 } 7918 7919 let mut blk_file_path = workload_path; 7920 blk_file_path.push("blk.img"); 7921 7922 let vfio_tap0 = "vfio-tap0"; 7923 let vfio_tap1 = "vfio-tap1"; 7924 let vfio_tap2 = "vfio-tap2"; 7925 let vfio_tap3 = "vfio-tap3"; 7926 7927 let mut child = GuestCommand::new(&guest) 7928 .args(["--cpus", "boot=4"]) 7929 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 7930 .args(["--kernel", kernel_path.to_str().unwrap()]) 7931 .args([ 7932 "--disk", 7933 format!( 7934 "path={}", 7935 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7936 ) 7937 .as_str(), 7938 format!( 7939 "path={}", 7940 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7941 ) 7942 .as_str(), 7943 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 7944 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 7945 ]) 7946 .args([ 7947 "--cmdline", 7948 format!( 7949 "{} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts", 7950 DIRECT_KERNEL_BOOT_CMDLINE 7951 ) 7952 .as_str(), 7953 ]) 7954 .args([ 7955 "--net", 7956 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 7957 format!( 7958 "tap={},mac={},iommu=on", 7959 vfio_tap1, guest.network.l2_guest_mac1 7960 ) 7961 .as_str(), 7962 format!( 7963 "tap={},mac={},iommu=on", 7964 vfio_tap2, guest.network.l2_guest_mac2 7965 ) 7966 .as_str(), 7967 format!( 7968 "tap={},mac={},iommu=on", 7969 vfio_tap3, guest.network.l2_guest_mac3 7970 ) 7971 .as_str(), 7972 ]) 7973 .capture_output() 7974 .spawn() 7975 .unwrap(); 7976 7977 thread::sleep(std::time::Duration::new(30, 0)); 7978 7979 let r = std::panic::catch_unwind(|| { 7980 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 7981 thread::sleep(std::time::Duration::new(120, 0)); 7982 7983 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 7984 // added to its kernel command line. 7985 // Let's ssh into it and verify that it's there. If it is it means 7986 // we're in the right guest (The L2 one) because the QEMU L1 guest 7987 // does not have this command line tag. 7988 assert_eq!( 7989 guest 7990 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 7991 .unwrap() 7992 .trim() 7993 .parse::<u32>() 7994 .unwrap_or_default(), 7995 1 7996 ); 7997 7998 // Let's also verify from the second virtio-net device passed to 7999 // the L2 VM. 8000 assert_eq!( 8001 guest 8002 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 8003 .unwrap() 8004 .trim() 8005 .parse::<u32>() 8006 .unwrap_or_default(), 8007 1 8008 ); 8009 8010 // Check the amount of PCI devices appearing in L2 VM. 8011 assert_eq!( 8012 guest 8013 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 8014 .unwrap() 8015 .trim() 8016 .parse::<u32>() 8017 .unwrap_or_default(), 8018 8, 8019 ); 8020 8021 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 8022 assert_eq!( 8023 guest 8024 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 8025 .unwrap() 8026 .trim() 8027 .parse::<u32>() 8028 .unwrap_or_default(), 8029 1 8030 ); 8031 8032 // Hotplug an extra virtio-net device through L2 VM. 8033 guest 8034 .ssh_command_l1( 8035 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 8036 ) 8037 .unwrap(); 8038 guest 8039 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 8040 .unwrap(); 8041 let vfio_hotplug_output = guest 8042 .ssh_command_l1( 8043 "sudo /mnt/ch-remote \ 8044 --api-socket=/tmp/ch_api.sock \ 8045 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 8046 ) 8047 .unwrap(); 8048 assert!(vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}")); 8049 8050 thread::sleep(std::time::Duration::new(10, 0)); 8051 8052 // Let's also verify from the third virtio-net device passed to 8053 // the L2 VM. This third device has been hotplugged through the L2 8054 // VM, so this is our way to validate hotplug works for VFIO PCI. 8055 assert_eq!( 8056 guest 8057 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 8058 .unwrap() 8059 .trim() 8060 .parse::<u32>() 8061 .unwrap_or_default(), 8062 1 8063 ); 8064 8065 // Check the amount of PCI devices appearing in L2 VM. 8066 // There should be one more device than before, raising the count 8067 // up to 9 PCI devices. 8068 assert_eq!( 8069 guest 8070 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 8071 .unwrap() 8072 .trim() 8073 .parse::<u32>() 8074 .unwrap_or_default(), 8075 9, 8076 ); 8077 8078 // Let's now verify that we can correctly remove the virtio-net 8079 // device through the "remove-device" command responsible for 8080 // unplugging VFIO devices. 8081 guest 8082 .ssh_command_l1( 8083 "sudo /mnt/ch-remote \ 8084 --api-socket=/tmp/ch_api.sock \ 8085 remove-device vfio123", 8086 ) 8087 .unwrap(); 8088 thread::sleep(std::time::Duration::new(10, 0)); 8089 8090 // Check the amount of PCI devices appearing in L2 VM is back down 8091 // to 8 devices. 8092 assert_eq!( 8093 guest 8094 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 8095 .unwrap() 8096 .trim() 8097 .parse::<u32>() 8098 .unwrap_or_default(), 8099 8, 8100 ); 8101 8102 // Perform memory hotplug in L2 and validate the memory is showing 8103 // up as expected. In order to check, we will use the virtio-net 8104 // device already passed through L2 as a VFIO device, this will 8105 // verify that VFIO devices are functional with memory hotplug. 8106 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 8107 guest 8108 .ssh_command_l2_1( 8109 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 8110 ) 8111 .unwrap(); 8112 guest 8113 .ssh_command_l1( 8114 "sudo /mnt/ch-remote \ 8115 --api-socket=/tmp/ch_api.sock \ 8116 resize --memory=1073741824", 8117 ) 8118 .unwrap(); 8119 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 8120 }); 8121 8122 let _ = child.kill(); 8123 let output = child.wait_with_output().unwrap(); 8124 8125 cleanup_vfio_network_interfaces(); 8126 8127 handle_child_output(r, &output); 8128 } 8129 8130 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 8131 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 8132 let guest = Guest::new(Box::new(hirsute)); 8133 let api_socket = temp_api_path(&guest.tmp_dir); 8134 8135 let mut child = GuestCommand::new(&guest) 8136 .args(["--cpus", "boot=4"]) 8137 .args([ 8138 "--memory", 8139 format!("size=4G,hotplug_size=4G,hotplug_method={}", hotplug_method).as_str(), 8140 ]) 8141 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8142 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8143 .args(["--api-socket", &api_socket]) 8144 .default_disks() 8145 .default_net() 8146 .capture_output() 8147 .spawn() 8148 .unwrap(); 8149 8150 let r = std::panic::catch_unwind(|| { 8151 guest.wait_vm_boot(None).unwrap(); 8152 8153 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8154 8155 guest.enable_memory_hotplug(); 8156 8157 // Add RAM to the VM 8158 let desired_ram = 6 << 30; 8159 resize_command(&api_socket, None, Some(desired_ram), None, None); 8160 thread::sleep(std::time::Duration::new(30, 0)); 8161 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8162 8163 // Check the VFIO device works when RAM is increased to 6GiB 8164 guest.check_nvidia_gpu(); 8165 }); 8166 8167 let _ = child.kill(); 8168 let output = child.wait_with_output().unwrap(); 8169 8170 handle_child_output(r, &output); 8171 } 8172 8173 #[test] 8174 fn test_nvidia_card_memory_hotplug_acpi() { 8175 test_nvidia_card_memory_hotplug("acpi") 8176 } 8177 8178 #[test] 8179 fn test_nvidia_card_memory_hotplug_virtio_mem() { 8180 test_nvidia_card_memory_hotplug("virtio-mem") 8181 } 8182 8183 #[test] 8184 fn test_nvidia_card_pci_hotplug() { 8185 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 8186 let guest = Guest::new(Box::new(hirsute)); 8187 let api_socket = temp_api_path(&guest.tmp_dir); 8188 8189 let mut child = GuestCommand::new(&guest) 8190 .args(["--cpus", "boot=4"]) 8191 .args(["--memory", "size=4G"]) 8192 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8193 .args(["--api-socket", &api_socket]) 8194 .default_disks() 8195 .default_net() 8196 .capture_output() 8197 .spawn() 8198 .unwrap(); 8199 8200 let r = std::panic::catch_unwind(|| { 8201 guest.wait_vm_boot(None).unwrap(); 8202 8203 // Hotplug the card to the VM 8204 let (cmd_success, cmd_output) = remote_command_w_output( 8205 &api_socket, 8206 "add-device", 8207 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 8208 ); 8209 assert!(cmd_success); 8210 assert!(String::from_utf8_lossy(&cmd_output) 8211 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 8212 8213 thread::sleep(std::time::Duration::new(10, 0)); 8214 8215 // Check the VFIO device works after hotplug 8216 guest.check_nvidia_gpu(); 8217 }); 8218 8219 let _ = child.kill(); 8220 let output = child.wait_with_output().unwrap(); 8221 8222 handle_child_output(r, &output); 8223 } 8224 8225 #[test] 8226 fn test_nvidia_card_reboot() { 8227 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 8228 let guest = Guest::new(Box::new(hirsute)); 8229 let api_socket = temp_api_path(&guest.tmp_dir); 8230 8231 let mut child = GuestCommand::new(&guest) 8232 .args(["--cpus", "boot=4"]) 8233 .args(["--memory", "size=4G"]) 8234 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8235 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8236 .args(["--api-socket", &api_socket]) 8237 .default_disks() 8238 .default_net() 8239 .capture_output() 8240 .spawn() 8241 .unwrap(); 8242 8243 let r = std::panic::catch_unwind(|| { 8244 guest.wait_vm_boot(None).unwrap(); 8245 8246 // Check the VFIO device works after boot 8247 guest.check_nvidia_gpu(); 8248 8249 guest.reboot_linux(0, None); 8250 8251 // Check the VFIO device works after reboot 8252 guest.check_nvidia_gpu(); 8253 }); 8254 8255 let _ = child.kill(); 8256 let output = child.wait_with_output().unwrap(); 8257 8258 handle_child_output(r, &output); 8259 } 8260 } 8261 8262 mod live_migration { 8263 use crate::*; 8264 8265 fn start_live_migration( 8266 migration_socket: &str, 8267 src_api_socket: &str, 8268 dest_api_socket: &str, 8269 local: bool, 8270 ) -> bool { 8271 // Start to receive migration from the destintion VM 8272 let mut receive_migration = Command::new(clh_command("ch-remote")) 8273 .args([ 8274 &format!("--api-socket={}", dest_api_socket), 8275 "receive-migration", 8276 &format! {"unix:{}", migration_socket}, 8277 ]) 8278 .stderr(Stdio::piped()) 8279 .stdout(Stdio::piped()) 8280 .spawn() 8281 .unwrap(); 8282 // Give it '1s' to make sure the 'migration_socket' file is properly created 8283 thread::sleep(std::time::Duration::new(1, 0)); 8284 // Start to send migration from the source VM 8285 8286 let mut args = [ 8287 format!("--api-socket={}", &src_api_socket), 8288 "send-migration".to_string(), 8289 format! {"unix:{}", migration_socket}, 8290 ] 8291 .to_vec(); 8292 8293 if local { 8294 args.insert(2, "--local".to_string()); 8295 } 8296 8297 let mut send_migration = Command::new(clh_command("ch-remote")) 8298 .args(&args) 8299 .stderr(Stdio::piped()) 8300 .stdout(Stdio::piped()) 8301 .spawn() 8302 .unwrap(); 8303 8304 // The 'send-migration' command should be executed successfully within the given timeout 8305 let send_success = if let Some(status) = send_migration 8306 .wait_timeout(std::time::Duration::from_secs(30)) 8307 .unwrap() 8308 { 8309 status.success() 8310 } else { 8311 false 8312 }; 8313 8314 if !send_success { 8315 let _ = send_migration.kill(); 8316 let output = send_migration.wait_with_output().unwrap(); 8317 eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 8318 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8319 } 8320 8321 // The 'receive-migration' command should be executed successfully within the given timeout 8322 let receive_success = if let Some(status) = receive_migration 8323 .wait_timeout(std::time::Duration::from_secs(30)) 8324 .unwrap() 8325 { 8326 status.success() 8327 } else { 8328 false 8329 }; 8330 8331 if !receive_success { 8332 let _ = receive_migration.kill(); 8333 let output = receive_migration.wait_with_output().unwrap(); 8334 eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 8335 String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); 8336 } 8337 8338 send_success && receive_success 8339 } 8340 8341 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 8342 let mut src_vm = src_vm; 8343 let mut dest_vm = dest_vm; 8344 8345 let _ = src_vm.kill(); 8346 let src_output = src_vm.wait_with_output().unwrap(); 8347 eprintln!( 8348 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8349 String::from_utf8_lossy(&src_output.stdout) 8350 ); 8351 eprintln!( 8352 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8353 String::from_utf8_lossy(&src_output.stderr) 8354 ); 8355 let _ = dest_vm.kill(); 8356 let dest_output = dest_vm.wait_with_output().unwrap(); 8357 eprintln!( 8358 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8359 String::from_utf8_lossy(&dest_output.stdout) 8360 ); 8361 eprintln!( 8362 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8363 String::from_utf8_lossy(&dest_output.stderr) 8364 ); 8365 8366 if let Some(ovs_vm) = ovs_vm { 8367 let mut ovs_vm = ovs_vm; 8368 let _ = ovs_vm.kill(); 8369 let ovs_output = ovs_vm.wait_with_output().unwrap(); 8370 eprintln!( 8371 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 8372 String::from_utf8_lossy(&ovs_output.stdout) 8373 ); 8374 eprintln!( 8375 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 8376 String::from_utf8_lossy(&ovs_output.stderr) 8377 ); 8378 8379 cleanup_ovs_dpdk(); 8380 } 8381 8382 panic!("Test failed: {}", message) 8383 } 8384 8385 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 8386 // same host. It ensures the following behaviors: 8387 // 1. The source VM is up and functional (including various virtio-devices are working properly); 8388 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 8389 // 3. The source VM terminated gracefully after live migration; 8390 // 4. The destination VM is functional (including various virtio-devices are working properly) after 8391 // live migration; 8392 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 8393 fn _test_live_migration(upgrade_test: bool, local: bool) { 8394 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8395 let guest = Guest::new(Box::new(focal)); 8396 let kernel_path = direct_kernel_boot_path(); 8397 let console_text = String::from("On a branch floating down river a cricket, singing."); 8398 let net_id = "net123"; 8399 let net_params = format!( 8400 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8401 net_id, guest.network.guest_mac, guest.network.host_ip 8402 ); 8403 8404 let memory_param: &[&str] = if local { 8405 &["--memory", "size=4G,shared=on"] 8406 } else { 8407 &["--memory", "size=4G"] 8408 }; 8409 8410 let boot_vcpus = 2; 8411 let max_vcpus = 4; 8412 8413 let pmem_temp_file = TempFile::new().unwrap(); 8414 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8415 std::process::Command::new("mkfs.ext4") 8416 .arg(pmem_temp_file.as_path()) 8417 .output() 8418 .expect("Expect creating disk image to succeed"); 8419 let pmem_path = String::from("/dev/pmem0"); 8420 8421 // Start the source VM 8422 let src_vm_path = if !upgrade_test { 8423 clh_command("cloud-hypervisor") 8424 } else { 8425 cloud_hypervisor_release_path() 8426 }; 8427 let src_api_socket = temp_api_path(&guest.tmp_dir); 8428 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8429 src_vm_cmd 8430 .args([ 8431 "--cpus", 8432 format!("boot={},max={}", boot_vcpus, max_vcpus).as_str(), 8433 ]) 8434 .args(memory_param) 8435 .args(["--kernel", kernel_path.to_str().unwrap()]) 8436 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8437 .default_disks() 8438 .args(["--net", net_params.as_str()]) 8439 .args(["--api-socket", &src_api_socket]) 8440 .args([ 8441 "--pmem", 8442 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8443 ]); 8444 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8445 8446 // Start the destination VM 8447 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8448 dest_api_socket.push_str(".dest"); 8449 let mut dest_child = GuestCommand::new(&guest) 8450 .args(["--api-socket", &dest_api_socket]) 8451 .capture_output() 8452 .spawn() 8453 .unwrap(); 8454 8455 let r = std::panic::catch_unwind(|| { 8456 guest.wait_vm_boot(None).unwrap(); 8457 8458 // Make sure the source VM is functaionl 8459 // Check the number of vCPUs 8460 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8461 8462 // Check the guest RAM 8463 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8464 8465 // Check the guest virtio-devices, e.g. block, rng, console, and net 8466 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8467 8468 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8469 // to make sure that removing and adding back the virtio-net device does 8470 // not break the live-migration support for virtio-pci. 8471 #[cfg(target_arch = "x86_64")] 8472 { 8473 assert!(remote_command( 8474 &src_api_socket, 8475 "remove-device", 8476 Some(net_id), 8477 )); 8478 thread::sleep(std::time::Duration::new(10, 0)); 8479 8480 // Plug the virtio-net device again 8481 assert!(remote_command( 8482 &src_api_socket, 8483 "add-net", 8484 Some(net_params.as_str()), 8485 )); 8486 thread::sleep(std::time::Duration::new(10, 0)); 8487 } 8488 8489 // Start the live-migration 8490 let migration_socket = String::from( 8491 guest 8492 .tmp_dir 8493 .as_path() 8494 .join("live-migration.sock") 8495 .to_str() 8496 .unwrap(), 8497 ); 8498 8499 assert!( 8500 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8501 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8502 ); 8503 }); 8504 8505 // Check and report any errors occured during the live-migration 8506 if r.is_err() { 8507 print_and_panic( 8508 src_child, 8509 dest_child, 8510 None, 8511 "Error occured during live-migration", 8512 ); 8513 } 8514 8515 // Check the source vm has been terminated successful (give it '3s' to settle) 8516 thread::sleep(std::time::Duration::new(3, 0)); 8517 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8518 print_and_panic( 8519 src_child, 8520 dest_child, 8521 None, 8522 "source VM was not terminated successfully.", 8523 ); 8524 }; 8525 8526 // Post live-migration check to make sure the destination VM is funcational 8527 let r = std::panic::catch_unwind(|| { 8528 // Perform same checks to validate VM has been properly migrated 8529 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8530 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8531 8532 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8533 }); 8534 8535 // Clean-up the destination VM and make sure it terminated correctly 8536 let _ = dest_child.kill(); 8537 let dest_output = dest_child.wait_with_output().unwrap(); 8538 handle_child_output(r, &dest_output); 8539 8540 // Check the destination VM has the expected 'concole_text' from its output 8541 let r = std::panic::catch_unwind(|| { 8542 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8543 }); 8544 handle_child_output(r, &dest_output); 8545 } 8546 8547 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 8548 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8549 let guest = Guest::new(Box::new(focal)); 8550 let kernel_path = direct_kernel_boot_path(); 8551 let console_text = String::from("On a branch floating down river a cricket, singing."); 8552 let net_id = "net123"; 8553 let net_params = format!( 8554 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8555 net_id, guest.network.guest_mac, guest.network.host_ip 8556 ); 8557 8558 let memory_param: &[&str] = if local { 8559 &[ 8560 "--memory", 8561 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 8562 "--balloon", 8563 "size=0", 8564 ] 8565 } else { 8566 &[ 8567 "--memory", 8568 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 8569 "--balloon", 8570 "size=0", 8571 ] 8572 }; 8573 8574 let boot_vcpus = 2; 8575 let max_vcpus = 4; 8576 8577 let pmem_temp_file = TempFile::new().unwrap(); 8578 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8579 std::process::Command::new("mkfs.ext4") 8580 .arg(pmem_temp_file.as_path()) 8581 .output() 8582 .expect("Expect creating disk image to succeed"); 8583 let pmem_path = String::from("/dev/pmem0"); 8584 8585 // Start the source VM 8586 let src_vm_path = if !upgrade_test { 8587 clh_command("cloud-hypervisor") 8588 } else { 8589 cloud_hypervisor_release_path() 8590 }; 8591 let src_api_socket = temp_api_path(&guest.tmp_dir); 8592 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8593 src_vm_cmd 8594 .args([ 8595 "--cpus", 8596 format!("boot={},max={}", boot_vcpus, max_vcpus).as_str(), 8597 ]) 8598 .args(memory_param) 8599 .args(["--kernel", kernel_path.to_str().unwrap()]) 8600 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8601 .default_disks() 8602 .args(["--net", net_params.as_str()]) 8603 .args(["--api-socket", &src_api_socket]) 8604 .args([ 8605 "--pmem", 8606 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8607 ]); 8608 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8609 8610 // Start the destination VM 8611 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8612 dest_api_socket.push_str(".dest"); 8613 let mut dest_child = GuestCommand::new(&guest) 8614 .args(["--api-socket", &dest_api_socket]) 8615 .capture_output() 8616 .spawn() 8617 .unwrap(); 8618 8619 let r = std::panic::catch_unwind(|| { 8620 guest.wait_vm_boot(None).unwrap(); 8621 8622 // Make sure the source VM is functaionl 8623 // Check the number of vCPUs 8624 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8625 8626 // Check the guest RAM 8627 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8628 // Increase the guest RAM 8629 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 8630 thread::sleep(std::time::Duration::new(5, 0)); 8631 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8632 // Use balloon to remove RAM from the VM 8633 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 8634 thread::sleep(std::time::Duration::new(5, 0)); 8635 let total_memory = guest.get_total_memory().unwrap_or_default(); 8636 assert!(total_memory > 4_800_000); 8637 assert!(total_memory < 5_760_000); 8638 8639 // Check the guest virtio-devices, e.g. block, rng, console, and net 8640 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8641 8642 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8643 // to make sure that removing and adding back the virtio-net device does 8644 // not break the live-migration support for virtio-pci. 8645 #[cfg(target_arch = "x86_64")] 8646 { 8647 assert!(remote_command( 8648 &src_api_socket, 8649 "remove-device", 8650 Some(net_id), 8651 )); 8652 thread::sleep(std::time::Duration::new(10, 0)); 8653 8654 // Plug the virtio-net device again 8655 assert!(remote_command( 8656 &src_api_socket, 8657 "add-net", 8658 Some(net_params.as_str()), 8659 )); 8660 thread::sleep(std::time::Duration::new(10, 0)); 8661 } 8662 8663 // Start the live-migration 8664 let migration_socket = String::from( 8665 guest 8666 .tmp_dir 8667 .as_path() 8668 .join("live-migration.sock") 8669 .to_str() 8670 .unwrap(), 8671 ); 8672 8673 assert!( 8674 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8675 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8676 ); 8677 }); 8678 8679 // Check and report any errors occured during the live-migration 8680 if r.is_err() { 8681 print_and_panic( 8682 src_child, 8683 dest_child, 8684 None, 8685 "Error occured during live-migration", 8686 ); 8687 } 8688 8689 // Check the source vm has been terminated successful (give it '3s' to settle) 8690 thread::sleep(std::time::Duration::new(3, 0)); 8691 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8692 print_and_panic( 8693 src_child, 8694 dest_child, 8695 None, 8696 "source VM was not terminated successfully.", 8697 ); 8698 }; 8699 8700 // Post live-migration check to make sure the destination VM is funcational 8701 let r = std::panic::catch_unwind(|| { 8702 // Perform same checks to validate VM has been properly migrated 8703 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8704 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8705 8706 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8707 8708 // Perform checks on guest RAM using balloon 8709 let total_memory = guest.get_total_memory().unwrap_or_default(); 8710 assert!(total_memory > 4_800_000); 8711 assert!(total_memory < 5_760_000); 8712 // Deflate balloon to restore entire RAM to the VM 8713 resize_command(&dest_api_socket, None, None, Some(0), None); 8714 thread::sleep(std::time::Duration::new(5, 0)); 8715 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8716 // Decrease guest RAM with virtio-mem 8717 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 8718 thread::sleep(std::time::Duration::new(5, 0)); 8719 let total_memory = guest.get_total_memory().unwrap_or_default(); 8720 assert!(total_memory > 4_800_000); 8721 assert!(total_memory < 5_760_000); 8722 }); 8723 8724 // Clean-up the destination VM and make sure it terminated correctly 8725 let _ = dest_child.kill(); 8726 let dest_output = dest_child.wait_with_output().unwrap(); 8727 handle_child_output(r, &dest_output); 8728 8729 // Check the destination VM has the expected 'concole_text' from its output 8730 let r = std::panic::catch_unwind(|| { 8731 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8732 }); 8733 handle_child_output(r, &dest_output); 8734 } 8735 8736 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 8737 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8738 let guest = Guest::new(Box::new(focal)); 8739 let kernel_path = direct_kernel_boot_path(); 8740 let console_text = String::from("On a branch floating down river a cricket, singing."); 8741 let net_id = "net123"; 8742 let net_params = format!( 8743 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8744 net_id, guest.network.guest_mac, guest.network.host_ip 8745 ); 8746 8747 let memory_param: &[&str] = if local { 8748 &[ 8749 "--memory", 8750 "size=0,hotplug_method=virtio-mem,shared=on", 8751 "--memory-zone", 8752 "id=mem0,size=1G,hotplug_size=4G,shared=on", 8753 "id=mem1,size=1G,hotplug_size=4G,shared=on", 8754 "id=mem2,size=2G,hotplug_size=4G,shared=on", 8755 "--numa", 8756 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 8757 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 8758 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 8759 ] 8760 } else { 8761 &[ 8762 "--memory", 8763 "size=0,hotplug_method=virtio-mem", 8764 "--memory-zone", 8765 "id=mem0,size=1G,hotplug_size=4G", 8766 "id=mem1,size=1G,hotplug_size=4G", 8767 "id=mem2,size=2G,hotplug_size=4G", 8768 "--numa", 8769 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 8770 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 8771 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 8772 ] 8773 }; 8774 8775 let boot_vcpus = 6; 8776 let max_vcpus = 12; 8777 8778 let pmem_temp_file = TempFile::new().unwrap(); 8779 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8780 std::process::Command::new("mkfs.ext4") 8781 .arg(pmem_temp_file.as_path()) 8782 .output() 8783 .expect("Expect creating disk image to succeed"); 8784 let pmem_path = String::from("/dev/pmem0"); 8785 8786 // Start the source VM 8787 let src_vm_path = if !upgrade_test { 8788 clh_command("cloud-hypervisor") 8789 } else { 8790 cloud_hypervisor_release_path() 8791 }; 8792 let src_api_socket = temp_api_path(&guest.tmp_dir); 8793 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8794 src_vm_cmd 8795 .args([ 8796 "--cpus", 8797 format!("boot={},max={}", boot_vcpus, max_vcpus).as_str(), 8798 ]) 8799 .args(memory_param) 8800 .args(["--kernel", kernel_path.to_str().unwrap()]) 8801 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8802 .default_disks() 8803 .args(["--net", net_params.as_str()]) 8804 .args(["--api-socket", &src_api_socket]) 8805 .args([ 8806 "--pmem", 8807 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8808 ]); 8809 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8810 8811 // Start the destination VM 8812 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8813 dest_api_socket.push_str(".dest"); 8814 let mut dest_child = GuestCommand::new(&guest) 8815 .args(["--api-socket", &dest_api_socket]) 8816 .capture_output() 8817 .spawn() 8818 .unwrap(); 8819 8820 let r = std::panic::catch_unwind(|| { 8821 guest.wait_vm_boot(None).unwrap(); 8822 8823 // Make sure the source VM is functaionl 8824 // Check the number of vCPUs 8825 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8826 8827 // Check the guest RAM 8828 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 8829 8830 // Check the guest virtio-devices, e.g. block, rng, console, and net 8831 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8832 8833 // Check the NUMA parameters are applied correctly and resize 8834 // each zone to test the case where we migrate a VM with the 8835 // virtio-mem regions being used. 8836 { 8837 guest.check_numa_common( 8838 Some(&[960_000, 960_000, 1_920_000]), 8839 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8840 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8841 ); 8842 8843 // AArch64 currently does not support hotplug, and therefore we only 8844 // test hotplug-related function on x86_64 here. 8845 #[cfg(target_arch = "x86_64")] 8846 { 8847 guest.enable_memory_hotplug(); 8848 8849 // Resize every memory zone and check each associated NUMA node 8850 // has been assigned the right amount of memory. 8851 resize_zone_command(&src_api_socket, "mem0", "2G"); 8852 resize_zone_command(&src_api_socket, "mem1", "2G"); 8853 resize_zone_command(&src_api_socket, "mem2", "3G"); 8854 thread::sleep(std::time::Duration::new(5, 0)); 8855 8856 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 8857 } 8858 } 8859 8860 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8861 // to make sure that removing and adding back the virtio-net device does 8862 // not break the live-migration support for virtio-pci. 8863 #[cfg(target_arch = "x86_64")] 8864 { 8865 assert!(remote_command( 8866 &src_api_socket, 8867 "remove-device", 8868 Some(net_id), 8869 )); 8870 thread::sleep(std::time::Duration::new(10, 0)); 8871 8872 // Plug the virtio-net device again 8873 assert!(remote_command( 8874 &src_api_socket, 8875 "add-net", 8876 Some(net_params.as_str()), 8877 )); 8878 thread::sleep(std::time::Duration::new(10, 0)); 8879 } 8880 8881 // Start the live-migration 8882 let migration_socket = String::from( 8883 guest 8884 .tmp_dir 8885 .as_path() 8886 .join("live-migration.sock") 8887 .to_str() 8888 .unwrap(), 8889 ); 8890 8891 assert!( 8892 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8893 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8894 ); 8895 }); 8896 8897 // Check and report any errors occured during the live-migration 8898 if r.is_err() { 8899 print_and_panic( 8900 src_child, 8901 dest_child, 8902 None, 8903 "Error occured during live-migration", 8904 ); 8905 } 8906 8907 // Check the source vm has been terminated successful (give it '3s' to settle) 8908 thread::sleep(std::time::Duration::new(3, 0)); 8909 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8910 print_and_panic( 8911 src_child, 8912 dest_child, 8913 None, 8914 "source VM was not terminated successfully.", 8915 ); 8916 }; 8917 8918 // Post live-migration check to make sure the destination VM is funcational 8919 let r = std::panic::catch_unwind(|| { 8920 // Perform same checks to validate VM has been properly migrated 8921 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8922 #[cfg(target_arch = "x86_64")] 8923 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 8924 #[cfg(target_arch = "aarch64")] 8925 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8926 8927 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8928 8929 // Perform NUMA related checks 8930 { 8931 #[cfg(target_arch = "aarch64")] 8932 { 8933 guest.check_numa_common( 8934 Some(&[960_000, 960_000, 1_920_000]), 8935 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8936 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8937 ); 8938 } 8939 8940 // AArch64 currently does not support hotplug, and therefore we only 8941 // test hotplug-related function on x86_64 here. 8942 #[cfg(target_arch = "x86_64")] 8943 { 8944 guest.check_numa_common( 8945 Some(&[1_920_000, 1_920_000, 2_880_000]), 8946 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 8947 Some(&["10 15 20", "20 10 25", "25 30 10"]), 8948 ); 8949 8950 guest.enable_memory_hotplug(); 8951 8952 // Resize every memory zone and check each associated NUMA node 8953 // has been assigned the right amount of memory. 8954 resize_zone_command(&dest_api_socket, "mem0", "4G"); 8955 resize_zone_command(&dest_api_socket, "mem1", "4G"); 8956 resize_zone_command(&dest_api_socket, "mem2", "4G"); 8957 // Resize to the maximum amount of CPUs and check each NUMA 8958 // node has been assigned the right CPUs set. 8959 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 8960 thread::sleep(std::time::Duration::new(5, 0)); 8961 8962 guest.check_numa_common( 8963 Some(&[3_840_000, 3_840_000, 3_840_000]), 8964 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 8965 None, 8966 ); 8967 } 8968 } 8969 }); 8970 8971 // Clean-up the destination VM and make sure it terminated correctly 8972 let _ = dest_child.kill(); 8973 let dest_output = dest_child.wait_with_output().unwrap(); 8974 handle_child_output(r, &dest_output); 8975 8976 // Check the destination VM has the expected 'concole_text' from its output 8977 let r = std::panic::catch_unwind(|| { 8978 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8979 }); 8980 handle_child_output(r, &dest_output); 8981 } 8982 8983 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 8984 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8985 let guest = Guest::new(Box::new(focal)); 8986 let kernel_path = direct_kernel_boot_path(); 8987 let console_text = String::from("On a branch floating down river a cricket, singing."); 8988 let net_id = "net123"; 8989 let net_params = format!( 8990 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8991 net_id, guest.network.guest_mac, guest.network.host_ip 8992 ); 8993 8994 let memory_param: &[&str] = if local { 8995 &["--memory", "size=4G,shared=on"] 8996 } else { 8997 &["--memory", "size=4G"] 8998 }; 8999 9000 let boot_vcpus = 2; 9001 let max_vcpus = 4; 9002 9003 let pmem_temp_file = TempFile::new().unwrap(); 9004 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9005 std::process::Command::new("mkfs.ext4") 9006 .arg(pmem_temp_file.as_path()) 9007 .output() 9008 .expect("Expect creating disk image to succeed"); 9009 let pmem_path = String::from("/dev/pmem0"); 9010 9011 // Start the source VM 9012 let src_vm_path = if !upgrade_test { 9013 clh_command("cloud-hypervisor") 9014 } else { 9015 cloud_hypervisor_release_path() 9016 }; 9017 let src_api_socket = temp_api_path(&guest.tmp_dir); 9018 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9019 src_vm_cmd 9020 .args([ 9021 "--cpus", 9022 format!("boot={},max={}", boot_vcpus, max_vcpus).as_str(), 9023 ]) 9024 .args(memory_param) 9025 .args(["--kernel", kernel_path.to_str().unwrap()]) 9026 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9027 .default_disks() 9028 .args(["--net", net_params.as_str()]) 9029 .args(["--api-socket", &src_api_socket]) 9030 .args([ 9031 "--pmem", 9032 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9033 ]) 9034 .args(["--watchdog"]); 9035 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9036 9037 // Start the destination VM 9038 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9039 dest_api_socket.push_str(".dest"); 9040 let mut dest_child = GuestCommand::new(&guest) 9041 .args(["--api-socket", &dest_api_socket]) 9042 .capture_output() 9043 .spawn() 9044 .unwrap(); 9045 9046 let r = std::panic::catch_unwind(|| { 9047 guest.wait_vm_boot(None).unwrap(); 9048 9049 // Make sure the source VM is functaionl 9050 // Check the number of vCPUs 9051 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9052 // Check the guest RAM 9053 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9054 // Check the guest virtio-devices, e.g. block, rng, console, and net 9055 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9056 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9057 // to make sure that removing and adding back the virtio-net device does 9058 // not break the live-migration support for virtio-pci. 9059 #[cfg(target_arch = "x86_64")] 9060 { 9061 assert!(remote_command( 9062 &src_api_socket, 9063 "remove-device", 9064 Some(net_id), 9065 )); 9066 thread::sleep(std::time::Duration::new(10, 0)); 9067 9068 // Plug the virtio-net device again 9069 assert!(remote_command( 9070 &src_api_socket, 9071 "add-net", 9072 Some(net_params.as_str()), 9073 )); 9074 thread::sleep(std::time::Duration::new(10, 0)); 9075 } 9076 9077 // Enable watchdog and ensure its functional 9078 let mut expected_reboot_count = 1; 9079 // Enable the watchdog with a 15s timeout 9080 enable_guest_watchdog(&guest, 15); 9081 // Reboot and check that systemd has activated the watchdog 9082 guest.ssh_command("sudo reboot").unwrap(); 9083 guest.wait_vm_boot(None).unwrap(); 9084 expected_reboot_count += 1; 9085 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9086 assert_eq!( 9087 guest 9088 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 9089 .unwrap() 9090 .trim() 9091 .parse::<u32>() 9092 .unwrap_or_default(), 9093 2 9094 ); 9095 // Allow some normal time to elapse to check we don't get spurious reboots 9096 thread::sleep(std::time::Duration::new(40, 0)); 9097 // Check no reboot 9098 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9099 9100 // Start the live-migration 9101 let migration_socket = String::from( 9102 guest 9103 .tmp_dir 9104 .as_path() 9105 .join("live-migration.sock") 9106 .to_str() 9107 .unwrap(), 9108 ); 9109 9110 assert!( 9111 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9112 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9113 ); 9114 }); 9115 9116 // Check and report any errors occured during the live-migration 9117 if r.is_err() { 9118 print_and_panic( 9119 src_child, 9120 dest_child, 9121 None, 9122 "Error occured during live-migration", 9123 ); 9124 } 9125 9126 // Check the source vm has been terminated successful (give it '3s' to settle) 9127 thread::sleep(std::time::Duration::new(3, 0)); 9128 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9129 print_and_panic( 9130 src_child, 9131 dest_child, 9132 None, 9133 "source VM was not terminated successfully.", 9134 ); 9135 }; 9136 9137 // Post live-migration check to make sure the destination VM is funcational 9138 let r = std::panic::catch_unwind(|| { 9139 // Perform same checks to validate VM has been properly migrated 9140 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9141 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9142 9143 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9144 9145 // Perform checks on watchdog 9146 let mut expected_reboot_count = 2; 9147 9148 // Allow some normal time to elapse to check we don't get spurious reboots 9149 thread::sleep(std::time::Duration::new(40, 0)); 9150 // Check no reboot 9151 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9152 9153 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 9154 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 9155 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 9156 guest.wait_vm_boot(Some(50)).unwrap(); 9157 // Check a reboot is triggerred by the watchdog 9158 expected_reboot_count += 1; 9159 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9160 9161 #[cfg(target_arch = "x86_64")] 9162 { 9163 // Now pause the VM and remain offline for 30s 9164 assert!(remote_command(&dest_api_socket, "pause", None)); 9165 thread::sleep(std::time::Duration::new(30, 0)); 9166 assert!(remote_command(&dest_api_socket, "resume", None)); 9167 9168 // Check no reboot 9169 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9170 } 9171 }); 9172 9173 // Clean-up the destination VM and make sure it terminated correctly 9174 let _ = dest_child.kill(); 9175 let dest_output = dest_child.wait_with_output().unwrap(); 9176 handle_child_output(r, &dest_output); 9177 9178 // Check the destination VM has the expected 'concole_text' from its output 9179 let r = std::panic::catch_unwind(|| { 9180 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9181 }); 9182 handle_child_output(r, &dest_output); 9183 } 9184 9185 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 9186 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9187 let ovs_guest = Guest::new(Box::new(ovs_focal)); 9188 9189 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9190 let migration_guest = Guest::new(Box::new(migration_focal)); 9191 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 9192 9193 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 9194 let (mut ovs_child, mut src_child) = 9195 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 9196 9197 // Start the destination VM 9198 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 9199 dest_api_socket.push_str(".dest"); 9200 let mut dest_child = GuestCommand::new(&migration_guest) 9201 .args(["--api-socket", &dest_api_socket]) 9202 .capture_output() 9203 .spawn() 9204 .unwrap(); 9205 9206 let r = std::panic::catch_unwind(|| { 9207 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 9208 thread::sleep(std::time::Duration::new(1, 0)); 9209 9210 // Start the live-migration 9211 let migration_socket = String::from( 9212 migration_guest 9213 .tmp_dir 9214 .as_path() 9215 .join("live-migration.sock") 9216 .to_str() 9217 .unwrap(), 9218 ); 9219 9220 assert!( 9221 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9222 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9223 ); 9224 }); 9225 9226 // Check and report any errors occured during the live-migration 9227 if r.is_err() { 9228 print_and_panic( 9229 src_child, 9230 dest_child, 9231 Some(ovs_child), 9232 "Error occured during live-migration", 9233 ); 9234 } 9235 9236 // Check the source vm has been terminated successful (give it '3s' to settle) 9237 thread::sleep(std::time::Duration::new(3, 0)); 9238 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9239 print_and_panic( 9240 src_child, 9241 dest_child, 9242 Some(ovs_child), 9243 "source VM was not terminated successfully.", 9244 ); 9245 }; 9246 9247 // Post live-migration check to make sure the destination VM is funcational 9248 let r = std::panic::catch_unwind(|| { 9249 // Perform same checks to validate VM has been properly migrated 9250 // Spawn a new netcat listener in the OVS VM 9251 let guest_ip = ovs_guest.network.guest_ip.clone(); 9252 thread::spawn(move || { 9253 ssh_command_ip( 9254 "nc -l 12345", 9255 &guest_ip, 9256 DEFAULT_SSH_RETRIES, 9257 DEFAULT_SSH_TIMEOUT, 9258 ) 9259 .unwrap(); 9260 }); 9261 9262 // Wait for the server to be listening 9263 thread::sleep(std::time::Duration::new(5, 0)); 9264 9265 // And check the connection is still functional after live-migration 9266 migration_guest 9267 .ssh_command("nc -vz 172.100.0.1 12345") 9268 .unwrap(); 9269 }); 9270 9271 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 9272 let _ = dest_child.kill(); 9273 let _ = ovs_child.kill(); 9274 let dest_output = dest_child.wait_with_output().unwrap(); 9275 let ovs_output = ovs_child.wait_with_output().unwrap(); 9276 9277 cleanup_ovs_dpdk(); 9278 9279 handle_child_output(r, &dest_output); 9280 handle_child_output(Ok(()), &ovs_output); 9281 } 9282 9283 mod live_migration_parallel { 9284 use super::*; 9285 #[test] 9286 fn test_live_migration_basic() { 9287 _test_live_migration(false, false) 9288 } 9289 9290 #[test] 9291 fn test_live_migration_local() { 9292 _test_live_migration(false, true) 9293 } 9294 9295 #[test] 9296 #[cfg(not(feature = "mshv"))] 9297 fn test_live_migration_numa() { 9298 _test_live_migration_numa(false, false) 9299 } 9300 9301 #[test] 9302 #[cfg(not(feature = "mshv"))] 9303 fn test_live_migration_numa_local() { 9304 _test_live_migration_numa(false, true) 9305 } 9306 9307 #[test] 9308 fn test_live_migration_watchdog() { 9309 _test_live_migration_watchdog(false, false) 9310 } 9311 9312 #[test] 9313 fn test_live_migration_watchdog_local() { 9314 _test_live_migration_watchdog(false, true) 9315 } 9316 9317 #[test] 9318 fn test_live_migration_balloon() { 9319 _test_live_migration_balloon(false, false) 9320 } 9321 9322 #[test] 9323 fn test_live_migration_balloon_local() { 9324 _test_live_migration_balloon(false, true) 9325 } 9326 9327 #[test] 9328 fn test_live_upgrade_basic() { 9329 _test_live_migration(true, false) 9330 } 9331 9332 #[test] 9333 fn test_live_upgrade_local() { 9334 _test_live_migration(true, true) 9335 } 9336 9337 #[test] 9338 #[cfg(not(feature = "mshv"))] 9339 fn test_live_upgrade_numa() { 9340 _test_live_migration_numa(true, false) 9341 } 9342 9343 #[test] 9344 #[cfg(not(feature = "mshv"))] 9345 fn test_live_upgrade_numa_local() { 9346 _test_live_migration_numa(true, true) 9347 } 9348 9349 #[test] 9350 fn test_live_upgrade_watchdog() { 9351 _test_live_migration_watchdog(true, false) 9352 } 9353 9354 #[test] 9355 fn test_live_upgrade_watchdog_local() { 9356 _test_live_migration_watchdog(true, true) 9357 } 9358 9359 #[test] 9360 fn test_live_upgrade_balloon() { 9361 _test_live_migration_balloon(true, false) 9362 } 9363 9364 #[test] 9365 fn test_live_upgrade_balloon_local() { 9366 _test_live_migration_balloon(true, true) 9367 } 9368 } 9369 9370 mod live_migration_sequential { 9371 #[cfg(target_arch = "x86_64")] 9372 #[cfg(not(feature = "mshv"))] 9373 use super::*; 9374 9375 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 9376 #[test] 9377 #[cfg(target_arch = "x86_64")] 9378 #[cfg(not(feature = "mshv"))] 9379 fn test_live_migration_ovs_dpdk() { 9380 _test_live_migration_ovs_dpdk(false, false); 9381 } 9382 9383 #[test] 9384 #[cfg(target_arch = "x86_64")] 9385 #[cfg(not(feature = "mshv"))] 9386 fn test_live_migration_ovs_dpdk_local() { 9387 _test_live_migration_ovs_dpdk(false, true); 9388 } 9389 9390 #[test] 9391 #[cfg(target_arch = "x86_64")] 9392 #[cfg(not(feature = "mshv"))] 9393 fn test_live_upgrade_ovs_dpdk() { 9394 _test_live_migration_ovs_dpdk(true, false); 9395 } 9396 9397 #[test] 9398 #[cfg(target_arch = "x86_64")] 9399 #[cfg(not(feature = "mshv"))] 9400 fn test_live_upgrade_ovs_dpdk_local() { 9401 _test_live_migration_ovs_dpdk(true, true); 9402 } 9403 } 9404 } 9405 9406 #[cfg(target_arch = "aarch64")] 9407 mod aarch64_acpi { 9408 use crate::*; 9409 9410 #[test] 9411 fn test_simple_launch_acpi() { 9412 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9413 9414 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 9415 let guest = Guest::new(disk_config); 9416 9417 let mut child = GuestCommand::new(&guest) 9418 .args(["--cpus", "boot=1"]) 9419 .args(["--memory", "size=512M"]) 9420 .args(["--kernel", edk2_path().to_str().unwrap()]) 9421 .default_disks() 9422 .default_net() 9423 .args(["--serial", "tty", "--console", "off"]) 9424 .capture_output() 9425 .spawn() 9426 .unwrap(); 9427 9428 let r = std::panic::catch_unwind(|| { 9429 guest.wait_vm_boot(Some(120)).unwrap(); 9430 9431 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 9432 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 9433 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 9434 }); 9435 9436 let _ = child.kill(); 9437 let output = child.wait_with_output().unwrap(); 9438 9439 handle_child_output(r, &output); 9440 }); 9441 } 9442 9443 #[test] 9444 fn test_guest_numa_nodes_acpi() { 9445 _test_guest_numa_nodes(true); 9446 } 9447 9448 #[test] 9449 fn test_cpu_topology_421_acpi() { 9450 test_cpu_topology(4, 2, 1, true); 9451 } 9452 9453 #[test] 9454 fn test_cpu_topology_142_acpi() { 9455 test_cpu_topology(1, 4, 2, true); 9456 } 9457 9458 #[test] 9459 fn test_cpu_topology_262_acpi() { 9460 test_cpu_topology(2, 6, 2, true); 9461 } 9462 9463 #[test] 9464 fn test_power_button_acpi() { 9465 _test_power_button(true); 9466 } 9467 9468 #[test] 9469 fn test_virtio_iommu() { 9470 _test_virtio_iommu(true) 9471 } 9472 } 9473 9474 mod rate_limiter { 9475 use super::*; 9476 9477 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 9478 // compared to given 'limit' rate. 9479 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 9480 let upper_limit = limit * (1_f64 + difference); 9481 let lower_limit = limit * (1_f64 - difference); 9482 9483 if measured > lower_limit && measured < upper_limit { 9484 return true; 9485 } 9486 9487 eprintln!( 9488 "\n\n==== check_rate_limit failed! ====\n\nmeasured={}, , lower_limit={}, upper_limit={}\n\n", 9489 measured, lower_limit, upper_limit 9490 ); 9491 9492 false 9493 } 9494 9495 fn _test_rate_limiter_net(rx: bool) { 9496 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9497 let guest = Guest::new(Box::new(focal)); 9498 9499 let test_timeout = 10; 9500 let num_queues = 2; 9501 let queue_size = 256; 9502 let bw_size = 10485760_u64; // bytes 9503 let bw_refill_time = 100; // ms 9504 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 9505 9506 let net_params = format!( 9507 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 9508 guest.network.guest_mac, 9509 guest.network.host_ip, 9510 num_queues, 9511 queue_size, 9512 bw_size, 9513 bw_refill_time, 9514 ); 9515 9516 let mut child = GuestCommand::new(&guest) 9517 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 9518 .args(["--memory", "size=4G"]) 9519 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9520 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9521 .default_disks() 9522 .args(["--net", net_params.as_str()]) 9523 .capture_output() 9524 .spawn() 9525 .unwrap(); 9526 9527 let r = std::panic::catch_unwind(|| { 9528 guest.wait_vm_boot(None).unwrap(); 9529 let measured_bps = 9530 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx).unwrap(); 9531 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 9532 }); 9533 9534 let _ = child.kill(); 9535 let output = child.wait_with_output().unwrap(); 9536 handle_child_output(r, &output); 9537 } 9538 9539 #[test] 9540 fn test_rate_limiter_net_rx() { 9541 _test_rate_limiter_net(true); 9542 } 9543 9544 #[test] 9545 fn test_rate_limiter_net_tx() { 9546 _test_rate_limiter_net(false); 9547 } 9548 9549 fn _test_rate_limiter_block(bandwidth: bool) { 9550 let test_timeout = 10; 9551 let num_queues = 1; 9552 let fio_ops = FioOps::RandRW; 9553 9554 let bw_size = if bandwidth { 9555 10485760_u64 // bytes 9556 } else { 9557 100_u64 // I/O 9558 }; 9559 let bw_refill_time = 100; // ms 9560 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 9561 9562 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9563 let guest = Guest::new(Box::new(focal)); 9564 let api_socket = temp_api_path(&guest.tmp_dir); 9565 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 9566 let blk_rate_limiter_test_img = 9567 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 9568 9569 // Create the test block image 9570 assert!(exec_host_command_output(&format!( 9571 "dd if=/dev/zero of={} bs=1M count=1024", 9572 blk_rate_limiter_test_img 9573 )) 9574 .status 9575 .success()); 9576 9577 let test_blk_params = if bandwidth { 9578 format!( 9579 "path={},bw_size={},bw_refill_time={}", 9580 blk_rate_limiter_test_img, bw_size, bw_refill_time 9581 ) 9582 } else { 9583 format!( 9584 "path={},ops_size={},ops_refill_time={}", 9585 blk_rate_limiter_test_img, bw_size, bw_refill_time 9586 ) 9587 }; 9588 9589 let mut child = GuestCommand::new(&guest) 9590 .args(["--cpus", &format!("boot={}", num_queues)]) 9591 .args(["--memory", "size=4G"]) 9592 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9593 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9594 .args([ 9595 "--disk", 9596 format!( 9597 "path={}", 9598 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 9599 ) 9600 .as_str(), 9601 format!( 9602 "path={}", 9603 guest.disk_config.disk(DiskType::CloudInit).unwrap() 9604 ) 9605 .as_str(), 9606 test_blk_params.as_str(), 9607 ]) 9608 .default_net() 9609 .args(["--api-socket", &api_socket]) 9610 .capture_output() 9611 .spawn() 9612 .unwrap(); 9613 9614 let r = std::panic::catch_unwind(|| { 9615 guest.wait_vm_boot(None).unwrap(); 9616 9617 let fio_command = format!( 9618 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 9619 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 9620 --rw={} --runtime={} --numjobs={}", 9621 fio_ops, test_timeout, num_queues 9622 ); 9623 let output = guest.ssh_command(&fio_command).unwrap(); 9624 9625 // Parse fio output 9626 let measured_rate = if bandwidth { 9627 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 9628 } else { 9629 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 9630 }; 9631 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 9632 }); 9633 9634 let _ = child.kill(); 9635 let output = child.wait_with_output().unwrap(); 9636 handle_child_output(r, &output); 9637 } 9638 9639 #[test] 9640 fn test_rate_limiter_block_bandwidth() { 9641 _test_rate_limiter_block(true) 9642 } 9643 9644 #[test] 9645 fn test_rate_limiter_block_iops() { 9646 _test_rate_limiter_block(false) 9647 } 9648 } 9649