1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 use std::sync::Mutex; 28 use std::thread; 29 use test_infra::*; 30 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 31 use wait_timeout::ChildExt; 32 33 // Constant taken from the VMM crate. 34 const MAX_NUM_PCI_SEGMENTS: u16 = 96; 35 36 #[cfg(target_arch = "x86_64")] 37 mod x86_64 { 38 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 39 pub const JAMMY_NVIDIA_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-nvidia.raw"; 40 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 41 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 42 "focal-server-cloudimg-amd64-custom-20210609-0-backing.qcow2"; 43 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 44 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 45 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20230119-0.raw"; 46 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2022-amd64-2.raw"; 47 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 48 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 49 } 50 51 #[cfg(target_arch = "x86_64")] 52 use x86_64::*; 53 54 #[cfg(target_arch = "aarch64")] 55 mod aarch64 { 56 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 57 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 58 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 59 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 60 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 61 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 62 "focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2"; 63 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 64 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 65 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 66 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 67 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 68 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 69 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 70 } 71 72 #[cfg(target_arch = "aarch64")] 73 use aarch64::*; 74 75 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 76 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 77 78 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 79 80 // This enum exists to make it more convenient to 81 // implement test for both D-Bus and REST APIs. 82 enum TargetApi { 83 // API socket 84 HttpApi(String), 85 // well known service name, object path 86 DBusApi(String, String), 87 } 88 89 impl TargetApi { 90 fn new_http_api(tmp_dir: &TempDir) -> Self { 91 Self::HttpApi(temp_api_path(tmp_dir)) 92 } 93 94 fn new_dbus_api(tmp_dir: &TempDir) -> Self { 95 // `tmp_dir` is in the form of "/tmp/chXXXXXX" 96 // and we take the `chXXXXXX` part as a unique identifier for the guest 97 let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); 98 99 Self::DBusApi( 100 format!("org.cloudhypervisor.{id}"), 101 format!("/org/cloudhypervisor/{id}"), 102 ) 103 } 104 105 fn guest_args(&self) -> Vec<String> { 106 match self { 107 TargetApi::HttpApi(api_socket) => { 108 vec![format!("--api-socket={}", api_socket.as_str())] 109 } 110 TargetApi::DBusApi(service_name, object_path) => { 111 vec![ 112 format!("--dbus-service-name={}", service_name.as_str()), 113 format!("--dbus-object-path={}", object_path.as_str()), 114 ] 115 } 116 } 117 } 118 119 fn remote_args(&self) -> Vec<String> { 120 // `guest_args` and `remote_args` are consistent with each other 121 self.guest_args() 122 } 123 124 fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { 125 let mut cmd = Command::new(clh_command("ch-remote")); 126 cmd.args(self.remote_args()); 127 cmd.arg(command); 128 129 if let Some(arg) = arg { 130 cmd.arg(arg); 131 } 132 133 let output = cmd.output().unwrap(); 134 if output.status.success() { 135 true 136 } else { 137 eprintln!("Error running ch-remote command: {:?}", &cmd); 138 let stderr = String::from_utf8_lossy(&output.stderr); 139 eprintln!("stderr: {stderr}"); 140 false 141 } 142 } 143 } 144 145 // Start cloud-hypervisor with no VM parameters, only the API server running. 146 // From the API: Create a VM, boot it and check that it looks as expected. 147 fn _test_api_create_boot(target_api: TargetApi, guest: Guest) { 148 let mut child = GuestCommand::new(&guest) 149 .args(target_api.guest_args()) 150 .capture_output() 151 .spawn() 152 .unwrap(); 153 154 thread::sleep(std::time::Duration::new(1, 0)); 155 156 // Verify API server is running 157 assert!(target_api.remote_command("ping", None)); 158 159 // Create the VM first 160 let cpu_count: u8 = 4; 161 let request_body = guest.api_create_body( 162 cpu_count, 163 direct_kernel_boot_path().to_str().unwrap(), 164 DIRECT_KERNEL_BOOT_CMDLINE, 165 ); 166 167 let temp_config_path = guest.tmp_dir.as_path().join("config"); 168 std::fs::write(&temp_config_path, request_body).unwrap(); 169 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 170 171 assert!(target_api.remote_command("create", Some(create_config),)); 172 173 // Then boot it 174 assert!(target_api.remote_command("boot", None)); 175 thread::sleep(std::time::Duration::new(20, 0)); 176 177 let r = std::panic::catch_unwind(|| { 178 // Check that the VM booted as expected 179 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 180 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 181 }); 182 183 let _ = child.kill(); 184 let output = child.wait_with_output().unwrap(); 185 186 handle_child_output(r, &output); 187 } 188 189 // Start cloud-hypervisor with no VM parameters, only the API server running. 190 // From the API: Create a VM, boot it and check it can be shutdown and then 191 // booted again 192 fn _test_api_shutdown(target_api: TargetApi, guest: Guest) { 193 let mut child = GuestCommand::new(&guest) 194 .args(target_api.guest_args()) 195 .capture_output() 196 .spawn() 197 .unwrap(); 198 199 thread::sleep(std::time::Duration::new(1, 0)); 200 201 // Verify API server is running 202 assert!(target_api.remote_command("ping", None)); 203 204 // Create the VM first 205 let cpu_count: u8 = 4; 206 let request_body = guest.api_create_body( 207 cpu_count, 208 direct_kernel_boot_path().to_str().unwrap(), 209 DIRECT_KERNEL_BOOT_CMDLINE, 210 ); 211 212 let temp_config_path = guest.tmp_dir.as_path().join("config"); 213 std::fs::write(&temp_config_path, request_body).unwrap(); 214 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 215 216 let r = std::panic::catch_unwind(|| { 217 assert!(target_api.remote_command("create", Some(create_config))); 218 219 // Then boot it 220 assert!(target_api.remote_command("boot", None)); 221 222 guest.wait_vm_boot(None).unwrap(); 223 224 // Check that the VM booted as expected 225 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 226 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 227 228 // Sync and shutdown without powering off to prevent filesystem 229 // corruption. 230 guest.ssh_command("sync").unwrap(); 231 guest.ssh_command("sudo shutdown -H now").unwrap(); 232 233 // Wait for the guest to be fully shutdown 234 thread::sleep(std::time::Duration::new(20, 0)); 235 236 // Then shut it down 237 assert!(target_api.remote_command("shutdown", None)); 238 239 // Then boot it again 240 assert!(target_api.remote_command("boot", None)); 241 242 guest.wait_vm_boot(None).unwrap(); 243 244 // Check that the VM booted as expected 245 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 246 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 247 }); 248 249 let _ = child.kill(); 250 let output = child.wait_with_output().unwrap(); 251 252 handle_child_output(r, &output); 253 } 254 255 // Start cloud-hypervisor with no VM parameters, only the API server running. 256 // From the API: Create a VM, boot it and check it can be deleted and then recreated 257 // booted again. 258 fn _test_api_delete(target_api: TargetApi, guest: Guest) { 259 let mut child = GuestCommand::new(&guest) 260 .args(target_api.guest_args()) 261 .capture_output() 262 .spawn() 263 .unwrap(); 264 265 thread::sleep(std::time::Duration::new(1, 0)); 266 267 // Verify API server is running 268 assert!(target_api.remote_command("ping", None)); 269 270 // Create the VM first 271 let cpu_count: u8 = 4; 272 let request_body = guest.api_create_body( 273 cpu_count, 274 direct_kernel_boot_path().to_str().unwrap(), 275 DIRECT_KERNEL_BOOT_CMDLINE, 276 ); 277 let temp_config_path = guest.tmp_dir.as_path().join("config"); 278 std::fs::write(&temp_config_path, request_body).unwrap(); 279 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 280 281 let r = std::panic::catch_unwind(|| { 282 assert!(target_api.remote_command("create", Some(create_config))); 283 284 // Then boot it 285 assert!(target_api.remote_command("boot", None)); 286 287 guest.wait_vm_boot(None).unwrap(); 288 289 // Check that the VM booted as expected 290 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 291 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 292 293 // Sync and shutdown without powering off to prevent filesystem 294 // corruption. 295 guest.ssh_command("sync").unwrap(); 296 guest.ssh_command("sudo shutdown -H now").unwrap(); 297 298 // Wait for the guest to be fully shutdown 299 thread::sleep(std::time::Duration::new(20, 0)); 300 301 // Then delete it 302 assert!(target_api.remote_command("delete", None)); 303 304 assert!(target_api.remote_command("create", Some(create_config))); 305 306 // Then boot it again 307 assert!(target_api.remote_command("boot", None)); 308 309 guest.wait_vm_boot(None).unwrap(); 310 311 // Check that the VM booted as expected 312 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 313 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 314 }); 315 316 let _ = child.kill(); 317 let output = child.wait_with_output().unwrap(); 318 319 handle_child_output(r, &output); 320 } 321 322 // Start cloud-hypervisor with no VM parameters, only the API server running. 323 // From the API: Create a VM, boot it and check that it looks as expected. 324 // Then we pause the VM, check that it's no longer available. 325 // Finally we resume the VM and check that it's available. 326 fn _test_api_pause_resume(target_api: TargetApi, guest: Guest) { 327 let mut child = GuestCommand::new(&guest) 328 .args(target_api.guest_args()) 329 .capture_output() 330 .spawn() 331 .unwrap(); 332 333 thread::sleep(std::time::Duration::new(1, 0)); 334 335 // Verify API server is running 336 assert!(target_api.remote_command("ping", None)); 337 338 // Create the VM first 339 let cpu_count: u8 = 4; 340 let request_body = guest.api_create_body( 341 cpu_count, 342 direct_kernel_boot_path().to_str().unwrap(), 343 DIRECT_KERNEL_BOOT_CMDLINE, 344 ); 345 346 let temp_config_path = guest.tmp_dir.as_path().join("config"); 347 std::fs::write(&temp_config_path, request_body).unwrap(); 348 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 349 350 assert!(target_api.remote_command("create", Some(create_config))); 351 352 // Then boot it 353 assert!(target_api.remote_command("boot", None)); 354 thread::sleep(std::time::Duration::new(20, 0)); 355 356 let r = std::panic::catch_unwind(|| { 357 // Check that the VM booted as expected 358 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 359 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 360 361 // We now pause the VM 362 assert!(target_api.remote_command("pause", None)); 363 364 // Check pausing again fails 365 assert!(!target_api.remote_command("pause", None)); 366 367 thread::sleep(std::time::Duration::new(2, 0)); 368 369 // SSH into the VM should fail 370 assert!(ssh_command_ip( 371 "grep -c processor /proc/cpuinfo", 372 &guest.network.guest_ip, 373 2, 374 5 375 ) 376 .is_err()); 377 378 // Resume the VM 379 assert!(target_api.remote_command("resume", None)); 380 381 // Check resuming again fails 382 assert!(!target_api.remote_command("resume", None)); 383 384 thread::sleep(std::time::Duration::new(2, 0)); 385 386 // Now we should be able to SSH back in and get the right number of CPUs 387 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 388 }); 389 390 let _ = child.kill(); 391 let output = child.wait_with_output().unwrap(); 392 393 handle_child_output(r, &output); 394 } 395 396 fn _test_pty_interaction(pty_path: PathBuf) { 397 let mut cf = std::fs::OpenOptions::new() 398 .write(true) 399 .read(true) 400 .open(pty_path) 401 .unwrap(); 402 403 // Some dumb sleeps but we don't want to write 404 // before the console is up and we don't want 405 // to try and write the next line before the 406 // login process is ready. 407 thread::sleep(std::time::Duration::new(5, 0)); 408 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 409 thread::sleep(std::time::Duration::new(2, 0)); 410 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 411 thread::sleep(std::time::Duration::new(2, 0)); 412 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 413 thread::sleep(std::time::Duration::new(2, 0)); 414 415 // read pty and ensure they have a login shell 416 // some fairly hacky workarounds to avoid looping 417 // forever in case the channel is blocked getting output 418 let ptyc = pty_read(cf); 419 let mut empty = 0; 420 let mut prev = String::new(); 421 loop { 422 thread::sleep(std::time::Duration::new(2, 0)); 423 match ptyc.try_recv() { 424 Ok(line) => { 425 empty = 0; 426 prev = prev + &line; 427 if prev.contains("test_pty_console") { 428 break; 429 } 430 } 431 Err(mpsc::TryRecvError::Empty) => { 432 empty += 1; 433 assert!(empty <= 5, "No login on pty"); 434 } 435 _ => { 436 panic!("No login on pty") 437 } 438 } 439 } 440 } 441 442 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 443 let mut workload_path = dirs::home_dir().unwrap(); 444 workload_path.push("workloads"); 445 446 let mut virtiofsd_path = workload_path; 447 virtiofsd_path.push("virtiofsd"); 448 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 449 450 let virtiofsd_socket_path = 451 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 452 453 // Start the daemon 454 let child = Command::new(virtiofsd_path.as_str()) 455 .args(["--shared-dir", shared_dir]) 456 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 457 .args(["--cache", "never"]) 458 .spawn() 459 .unwrap(); 460 461 thread::sleep(std::time::Duration::new(10, 0)); 462 463 (child, virtiofsd_socket_path) 464 } 465 466 fn prepare_vubd( 467 tmp_dir: &TempDir, 468 blk_img: &str, 469 num_queues: usize, 470 rdonly: bool, 471 direct: bool, 472 ) -> (std::process::Child, String) { 473 let mut workload_path = dirs::home_dir().unwrap(); 474 workload_path.push("workloads"); 475 476 let mut blk_file_path = workload_path; 477 blk_file_path.push(blk_img); 478 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 479 480 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 481 482 // Start the daemon 483 let child = Command::new(clh_command("vhost_user_block")) 484 .args([ 485 "--block-backend", 486 format!( 487 "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" 488 ) 489 .as_str(), 490 ]) 491 .spawn() 492 .unwrap(); 493 494 thread::sleep(std::time::Duration::new(10, 0)); 495 496 (child, vubd_socket_path) 497 } 498 499 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 500 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 501 } 502 503 fn temp_api_path(tmp_dir: &TempDir) -> String { 504 String::from( 505 tmp_dir 506 .as_path() 507 .join("cloud-hypervisor.sock") 508 .to_str() 509 .unwrap(), 510 ) 511 } 512 513 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 514 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 515 } 516 517 // Creates the directory and returns the path. 518 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 519 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 520 std::fs::create_dir(&snapshot_dir).unwrap(); 521 snapshot_dir 522 } 523 524 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 525 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 526 vmcore_file 527 } 528 529 // Creates the path for direct kernel boot and return the path. 530 // For x86_64, this function returns the vmlinux kernel path. 531 // For AArch64, this function returns the PE kernel path. 532 fn direct_kernel_boot_path() -> PathBuf { 533 let mut workload_path = dirs::home_dir().unwrap(); 534 workload_path.push("workloads"); 535 536 let mut kernel_path = workload_path; 537 #[cfg(target_arch = "x86_64")] 538 kernel_path.push("vmlinux"); 539 #[cfg(target_arch = "aarch64")] 540 kernel_path.push("Image"); 541 542 kernel_path 543 } 544 545 fn edk2_path() -> PathBuf { 546 let mut workload_path = dirs::home_dir().unwrap(); 547 workload_path.push("workloads"); 548 let mut edk2_path = workload_path; 549 edk2_path.push(OVMF_NAME); 550 551 edk2_path 552 } 553 554 fn cloud_hypervisor_release_path() -> String { 555 let mut workload_path = dirs::home_dir().unwrap(); 556 workload_path.push("workloads"); 557 558 let mut ch_release_path = workload_path; 559 #[cfg(target_arch = "x86_64")] 560 ch_release_path.push("cloud-hypervisor-static"); 561 #[cfg(target_arch = "aarch64")] 562 ch_release_path.push("cloud-hypervisor-static-aarch64"); 563 564 ch_release_path.into_os_string().into_string().unwrap() 565 } 566 567 fn prepare_vhost_user_net_daemon( 568 tmp_dir: &TempDir, 569 ip: &str, 570 tap: Option<&str>, 571 mtu: Option<u16>, 572 num_queues: usize, 573 client_mode: bool, 574 ) -> (std::process::Command, String) { 575 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 576 577 // Start the daemon 578 let mut net_params = format!( 579 "ip={ip},mask=255.255.255.0,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" 580 ); 581 582 if let Some(tap) = tap { 583 net_params.push_str(format!(",tap={tap}").as_str()); 584 } 585 586 if let Some(mtu) = mtu { 587 net_params.push_str(format!(",mtu={mtu}").as_str()); 588 } 589 590 let mut command = Command::new(clh_command("vhost_user_net")); 591 command.args(["--net-backend", net_params.as_str()]); 592 593 (command, vunet_socket_path) 594 } 595 596 fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { 597 let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); 598 let swtpm_socket_path = String::from( 599 tmp_dir 600 .as_path() 601 .join("swtpm") 602 .join("swtpm.sock") 603 .to_str() 604 .unwrap(), 605 ); 606 std::fs::create_dir(&swtpm_tpm_dir).unwrap(); 607 608 let mut swtpm_command = Command::new("swtpm"); 609 let swtpm_args = [ 610 "socket", 611 "--tpmstate", 612 &format!("dir={swtpm_tpm_dir}"), 613 "--ctrl", 614 &format!("type=unixio,path={swtpm_socket_path}"), 615 "--flags", 616 "startup-clear", 617 "--tpm2", 618 ]; 619 swtpm_command.args(swtpm_args); 620 621 (swtpm_command, swtpm_socket_path) 622 } 623 624 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 625 let mut cmd = Command::new(clh_command("ch-remote")); 626 cmd.args([&format!("--api-socket={api_socket}"), command]); 627 628 if let Some(arg) = arg { 629 cmd.arg(arg); 630 } 631 let output = cmd.output().unwrap(); 632 if output.status.success() { 633 true 634 } else { 635 eprintln!("Error running ch-remote command: {:?}", &cmd); 636 let stderr = String::from_utf8_lossy(&output.stderr); 637 eprintln!("stderr: {stderr}"); 638 false 639 } 640 } 641 642 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 643 let mut cmd = Command::new(clh_command("ch-remote")); 644 cmd.args([&format!("--api-socket={api_socket}"), command]); 645 646 if let Some(arg) = arg { 647 cmd.arg(arg); 648 } 649 650 let output = cmd.output().expect("Failed to launch ch-remote"); 651 652 (output.status.success(), output.stdout) 653 } 654 655 fn resize_command( 656 api_socket: &str, 657 desired_vcpus: Option<u8>, 658 desired_ram: Option<usize>, 659 desired_balloon: Option<usize>, 660 event_file: Option<&str>, 661 ) -> bool { 662 let mut cmd = Command::new(clh_command("ch-remote")); 663 cmd.args([&format!("--api-socket={api_socket}"), "resize"]); 664 665 if let Some(desired_vcpus) = desired_vcpus { 666 cmd.arg(format!("--cpus={desired_vcpus}")); 667 } 668 669 if let Some(desired_ram) = desired_ram { 670 cmd.arg(format!("--memory={desired_ram}")); 671 } 672 673 if let Some(desired_balloon) = desired_balloon { 674 cmd.arg(format!("--balloon={desired_balloon}")); 675 } 676 677 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 678 679 if let Some(event_path) = event_file { 680 let latest_events = [ 681 &MetaEvent { 682 event: "resizing".to_string(), 683 device_id: None, 684 }, 685 &MetaEvent { 686 event: "resized".to_string(), 687 device_id: None, 688 }, 689 ]; 690 assert!(check_latest_events_exact(&latest_events, event_path)); 691 } 692 693 ret 694 } 695 696 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 697 let mut cmd = Command::new(clh_command("ch-remote")); 698 cmd.args([ 699 &format!("--api-socket={api_socket}"), 700 "resize-zone", 701 &format!("--id={id}"), 702 &format!("--size={desired_size}"), 703 ]); 704 705 cmd.status().expect("Failed to launch ch-remote").success() 706 } 707 708 // setup OVS-DPDK bridge and ports 709 fn setup_ovs_dpdk() { 710 // setup OVS-DPDK 711 assert!(exec_host_command_status("service openvswitch-switch start").success()); 712 assert!(exec_host_command_status("ovs-vsctl init").success()); 713 assert!( 714 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 715 .success() 716 ); 717 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 718 719 // Create OVS-DPDK bridge and ports 720 assert!(exec_host_command_status( 721 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 722 ) 723 .success()); 724 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 725 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 726 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 727 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 728 } 729 fn cleanup_ovs_dpdk() { 730 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 731 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 732 } 733 // Setup two guests and ensure they are connected through ovs-dpdk 734 fn setup_ovs_dpdk_guests( 735 guest1: &Guest, 736 guest2: &Guest, 737 api_socket: &str, 738 release_binary: bool, 739 ) -> (Child, Child) { 740 setup_ovs_dpdk(); 741 742 let clh_path = if !release_binary { 743 clh_command("cloud-hypervisor") 744 } else { 745 cloud_hypervisor_release_path() 746 }; 747 748 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 749 .args(["--cpus", "boot=2"]) 750 .args(["--memory", "size=0,shared=on"]) 751 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 752 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 753 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 754 .default_disks() 755 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 756 .capture_output() 757 .spawn() 758 .unwrap(); 759 760 #[cfg(target_arch = "x86_64")] 761 let guest_net_iface = "ens5"; 762 #[cfg(target_arch = "aarch64")] 763 let guest_net_iface = "enp0s5"; 764 765 let r = std::panic::catch_unwind(|| { 766 guest1.wait_vm_boot(None).unwrap(); 767 768 guest1 769 .ssh_command(&format!( 770 "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" 771 )) 772 .unwrap(); 773 guest1 774 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 775 .unwrap(); 776 777 let guest_ip = guest1.network.guest_ip.clone(); 778 thread::spawn(move || { 779 ssh_command_ip( 780 "nc -l 12345", 781 &guest_ip, 782 DEFAULT_SSH_RETRIES, 783 DEFAULT_SSH_TIMEOUT, 784 ) 785 .unwrap(); 786 }); 787 }); 788 if r.is_err() { 789 cleanup_ovs_dpdk(); 790 791 let _ = child1.kill(); 792 let output = child1.wait_with_output().unwrap(); 793 handle_child_output(r, &output); 794 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 795 } 796 797 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 798 .args(["--api-socket", api_socket]) 799 .args(["--cpus", "boot=2"]) 800 .args(["--memory", "size=0,shared=on"]) 801 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 802 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 803 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 804 .default_disks() 805 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 806 .capture_output() 807 .spawn() 808 .unwrap(); 809 810 let r = std::panic::catch_unwind(|| { 811 guest2.wait_vm_boot(None).unwrap(); 812 813 guest2 814 .ssh_command(&format!( 815 "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" 816 )) 817 .unwrap(); 818 guest2 819 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 820 .unwrap(); 821 822 // Check the connection works properly between the two VMs 823 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 824 }); 825 if r.is_err() { 826 cleanup_ovs_dpdk(); 827 828 let _ = child1.kill(); 829 let _ = child2.kill(); 830 let output = child2.wait_with_output().unwrap(); 831 handle_child_output(r, &output); 832 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 833 } 834 835 (child1, child2) 836 } 837 838 enum FwType { 839 Ovmf, 840 RustHypervisorFirmware, 841 } 842 843 fn fw_path(_fw_type: FwType) -> String { 844 let mut workload_path = dirs::home_dir().unwrap(); 845 workload_path.push("workloads"); 846 847 let mut fw_path = workload_path; 848 #[cfg(target_arch = "aarch64")] 849 fw_path.push("CLOUDHV_EFI.fd"); 850 #[cfg(target_arch = "x86_64")] 851 { 852 match _fw_type { 853 FwType::Ovmf => fw_path.push(OVMF_NAME), 854 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 855 } 856 } 857 858 fw_path.to_str().unwrap().to_string() 859 } 860 861 #[derive(Debug)] 862 struct MetaEvent { 863 event: String, 864 device_id: Option<String>, 865 } 866 867 impl MetaEvent { 868 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 869 let mut matched = false; 870 if v["event"].as_str().unwrap() == self.event { 871 if let Some(device_id) = &self.device_id { 872 if v["properties"]["id"].as_str().unwrap() == device_id { 873 matched = true 874 } 875 } else { 876 matched = true; 877 } 878 } 879 matched 880 } 881 } 882 883 // Parse the event_monitor file based on the format that each event 884 // is followed by a double newline 885 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 886 let content = fs::read(event_file).unwrap(); 887 let mut ret = Vec::new(); 888 for entry in String::from_utf8_lossy(&content) 889 .trim() 890 .split("\n\n") 891 .collect::<Vec<&str>>() 892 { 893 ret.push(serde_json::from_str(entry).unwrap()); 894 } 895 896 ret 897 } 898 899 // Return true if all events from the input 'expected_events' are matched sequentially 900 // with events from the 'event_file' 901 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 902 let json_events = parse_event_file(event_file); 903 let len = expected_events.len(); 904 let mut idx = 0; 905 for e in &json_events { 906 if idx == len { 907 break; 908 } 909 if expected_events[idx].match_with_json_event(e) { 910 idx += 1; 911 } 912 } 913 914 let ret = idx == len; 915 916 if !ret { 917 eprintln!( 918 "\n\n==== Start 'check_sequential_events' failed ==== \ 919 \n\nexpected_events={:?}\nactual_events={:?} \ 920 \n\n==== End 'check_sequential_events' failed ====", 921 expected_events, json_events, 922 ); 923 } 924 925 ret 926 } 927 928 // Return true if all events from the input 'expected_events' are matched exactly 929 // with events from the 'event_file' 930 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 931 let json_events = parse_event_file(event_file); 932 assert!(expected_events.len() <= json_events.len()); 933 let json_events = &json_events[..expected_events.len()]; 934 935 for (idx, e) in json_events.iter().enumerate() { 936 if !expected_events[idx].match_with_json_event(e) { 937 eprintln!( 938 "\n\n==== Start 'check_sequential_events_exact' failed ==== \ 939 \n\nexpected_events={:?}\nactual_events={:?} \ 940 \n\n==== End 'check_sequential_events_exact' failed ====", 941 expected_events, json_events, 942 ); 943 944 return false; 945 } 946 } 947 948 true 949 } 950 951 // Return true if events from the input 'latest_events' are matched exactly 952 // with the most recent events from the 'event_file' 953 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 954 let json_events = parse_event_file(event_file); 955 assert!(latest_events.len() <= json_events.len()); 956 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 957 958 for (idx, e) in json_events.iter().enumerate() { 959 if !latest_events[idx].match_with_json_event(e) { 960 eprintln!( 961 "\n\n==== Start 'check_latest_events_exact' failed ==== \ 962 \n\nexpected_events={:?}\nactual_events={:?} \ 963 \n\n==== End 'check_latest_events_exact' failed ====", 964 latest_events, json_events, 965 ); 966 967 return false; 968 } 969 } 970 971 true 972 } 973 974 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 975 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 976 let guest = Guest::new(Box::new(focal)); 977 let total_vcpus = threads_per_core * cores_per_package * packages; 978 let direct_kernel_boot_path = direct_kernel_boot_path(); 979 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 980 let fw_path = fw_path(FwType::RustHypervisorFirmware); 981 if use_fw { 982 kernel_path = fw_path.as_str(); 983 } 984 985 let mut child = GuestCommand::new(&guest) 986 .args([ 987 "--cpus", 988 &format!( 989 "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" 990 ), 991 ]) 992 .args(["--memory", "size=512M"]) 993 .args(["--kernel", kernel_path]) 994 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 995 .default_disks() 996 .default_net() 997 .capture_output() 998 .spawn() 999 .unwrap(); 1000 1001 let r = std::panic::catch_unwind(|| { 1002 guest.wait_vm_boot(None).unwrap(); 1003 assert_eq!( 1004 guest.get_cpu_count().unwrap_or_default(), 1005 u32::from(total_vcpus) 1006 ); 1007 assert_eq!( 1008 guest 1009 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1010 .unwrap() 1011 .trim() 1012 .parse::<u8>() 1013 .unwrap_or(0), 1014 threads_per_core 1015 ); 1016 1017 assert_eq!( 1018 guest 1019 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1020 .unwrap() 1021 .trim() 1022 .parse::<u8>() 1023 .unwrap_or(0), 1024 cores_per_package 1025 ); 1026 1027 assert_eq!( 1028 guest 1029 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1030 .unwrap() 1031 .trim() 1032 .parse::<u8>() 1033 .unwrap_or(0), 1034 packages 1035 ); 1036 }); 1037 1038 let _ = child.kill(); 1039 let output = child.wait_with_output().unwrap(); 1040 1041 handle_child_output(r, &output); 1042 } 1043 1044 #[allow(unused_variables)] 1045 fn _test_guest_numa_nodes(acpi: bool) { 1046 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1047 let guest = Guest::new(Box::new(focal)); 1048 let api_socket = temp_api_path(&guest.tmp_dir); 1049 #[cfg(target_arch = "x86_64")] 1050 let kernel_path = direct_kernel_boot_path(); 1051 #[cfg(target_arch = "aarch64")] 1052 let kernel_path = if acpi { 1053 edk2_path() 1054 } else { 1055 direct_kernel_boot_path() 1056 }; 1057 1058 let mut child = GuestCommand::new(&guest) 1059 .args(["--cpus", "boot=6,max=12"]) 1060 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 1061 .args([ 1062 "--memory-zone", 1063 "id=mem0,size=1G,hotplug_size=3G", 1064 "id=mem1,size=2G,hotplug_size=3G", 1065 "id=mem2,size=3G,hotplug_size=3G", 1066 ]) 1067 .args([ 1068 "--numa", 1069 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 1070 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 1071 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 1072 ]) 1073 .args(["--kernel", kernel_path.to_str().unwrap()]) 1074 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1075 .args(["--api-socket", &api_socket]) 1076 .capture_output() 1077 .default_disks() 1078 .default_net() 1079 .spawn() 1080 .unwrap(); 1081 1082 let r = std::panic::catch_unwind(|| { 1083 guest.wait_vm_boot(None).unwrap(); 1084 1085 guest.check_numa_common( 1086 Some(&[960_000, 1_920_000, 2_880_000]), 1087 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 1088 Some(&["10 15 20", "20 10 25", "25 30 10"]), 1089 ); 1090 1091 // AArch64 currently does not support hotplug, and therefore we only 1092 // test hotplug-related function on x86_64 here. 1093 #[cfg(target_arch = "x86_64")] 1094 { 1095 guest.enable_memory_hotplug(); 1096 1097 // Resize every memory zone and check each associated NUMA node 1098 // has been assigned the right amount of memory. 1099 resize_zone_command(&api_socket, "mem0", "4G"); 1100 resize_zone_command(&api_socket, "mem1", "4G"); 1101 resize_zone_command(&api_socket, "mem2", "4G"); 1102 // Resize to the maximum amount of CPUs and check each NUMA 1103 // node has been assigned the right CPUs set. 1104 resize_command(&api_socket, Some(12), None, None, None); 1105 thread::sleep(std::time::Duration::new(5, 0)); 1106 1107 guest.check_numa_common( 1108 Some(&[3_840_000, 3_840_000, 3_840_000]), 1109 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 1110 None, 1111 ); 1112 } 1113 }); 1114 1115 let _ = child.kill(); 1116 let output = child.wait_with_output().unwrap(); 1117 1118 handle_child_output(r, &output); 1119 } 1120 1121 #[allow(unused_variables)] 1122 fn _test_power_button(acpi: bool) { 1123 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1124 let guest = Guest::new(Box::new(focal)); 1125 let mut cmd = GuestCommand::new(&guest); 1126 let api_socket = temp_api_path(&guest.tmp_dir); 1127 1128 #[cfg(target_arch = "x86_64")] 1129 let kernel_path = direct_kernel_boot_path(); 1130 #[cfg(target_arch = "aarch64")] 1131 let kernel_path = if acpi { 1132 edk2_path() 1133 } else { 1134 direct_kernel_boot_path() 1135 }; 1136 1137 cmd.args(["--cpus", "boot=1"]) 1138 .args(["--memory", "size=512M"]) 1139 .args(["--kernel", kernel_path.to_str().unwrap()]) 1140 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1141 .capture_output() 1142 .default_disks() 1143 .default_net() 1144 .args(["--api-socket", &api_socket]); 1145 1146 let child = cmd.spawn().unwrap(); 1147 1148 let r = std::panic::catch_unwind(|| { 1149 guest.wait_vm_boot(None).unwrap(); 1150 assert!(remote_command(&api_socket, "power-button", None)); 1151 }); 1152 1153 let output = child.wait_with_output().unwrap(); 1154 assert!(output.status.success()); 1155 handle_child_output(r, &output); 1156 } 1157 1158 type PrepareNetDaemon = dyn Fn( 1159 &TempDir, 1160 &str, 1161 Option<&str>, 1162 Option<u16>, 1163 usize, 1164 bool, 1165 ) -> (std::process::Command, String); 1166 1167 fn test_vhost_user_net( 1168 tap: Option<&str>, 1169 num_queues: usize, 1170 prepare_daemon: &PrepareNetDaemon, 1171 generate_host_mac: bool, 1172 client_mode_daemon: bool, 1173 ) { 1174 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1175 let guest = Guest::new(Box::new(focal)); 1176 let api_socket = temp_api_path(&guest.tmp_dir); 1177 1178 let kernel_path = direct_kernel_boot_path(); 1179 1180 let host_mac = if generate_host_mac { 1181 Some(MacAddr::local_random()) 1182 } else { 1183 None 1184 }; 1185 1186 let mtu = Some(3000); 1187 1188 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 1189 &guest.tmp_dir, 1190 &guest.network.host_ip, 1191 tap, 1192 mtu, 1193 num_queues, 1194 client_mode_daemon, 1195 ); 1196 1197 let net_params = format!( 1198 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 1199 guest.network.guest_mac, 1200 vunet_socket_path, 1201 num_queues, 1202 if let Some(host_mac) = host_mac { 1203 format!(",host_mac={host_mac}") 1204 } else { 1205 "".to_owned() 1206 }, 1207 if client_mode_daemon { 1208 "server" 1209 } else { 1210 "client" 1211 }, 1212 ); 1213 1214 let mut ch_command = GuestCommand::new(&guest); 1215 ch_command 1216 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 1217 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1218 .args(["--kernel", kernel_path.to_str().unwrap()]) 1219 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1220 .default_disks() 1221 .args(["--net", net_params.as_str()]) 1222 .args(["--api-socket", &api_socket]) 1223 .capture_output(); 1224 1225 let mut daemon_child: std::process::Child; 1226 let mut child: std::process::Child; 1227 1228 if client_mode_daemon { 1229 child = ch_command.spawn().unwrap(); 1230 // Make sure the VMM is waiting for the backend to connect 1231 thread::sleep(std::time::Duration::new(10, 0)); 1232 daemon_child = daemon_command.spawn().unwrap(); 1233 } else { 1234 daemon_child = daemon_command.spawn().unwrap(); 1235 // Make sure the backend is waiting for the VMM to connect 1236 thread::sleep(std::time::Duration::new(10, 0)); 1237 child = ch_command.spawn().unwrap(); 1238 } 1239 1240 let r = std::panic::catch_unwind(|| { 1241 guest.wait_vm_boot(None).unwrap(); 1242 1243 if let Some(tap_name) = tap { 1244 let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); 1245 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1246 } 1247 1248 if let Some(host_mac) = tap { 1249 let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); 1250 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1251 } 1252 1253 #[cfg(target_arch = "aarch64")] 1254 let iface = "enp0s4"; 1255 #[cfg(target_arch = "x86_64")] 1256 let iface = "ens4"; 1257 1258 assert_eq!( 1259 guest 1260 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 1261 .unwrap() 1262 .trim(), 1263 "3000" 1264 ); 1265 1266 // 1 network interface + default localhost ==> 2 interfaces 1267 // It's important to note that this test is fully exercising the 1268 // vhost-user-net implementation and the associated backend since 1269 // it does not define any --net network interface. That means all 1270 // the ssh communication in that test happens through the network 1271 // interface backed by vhost-user-net. 1272 assert_eq!( 1273 guest 1274 .ssh_command("ip -o link | wc -l") 1275 .unwrap() 1276 .trim() 1277 .parse::<u32>() 1278 .unwrap_or_default(), 1279 2 1280 ); 1281 1282 // The following pci devices will appear on guest with PCI-MSI 1283 // interrupt vectors assigned. 1284 // 1 virtio-console with 3 vectors: config, Rx, Tx 1285 // 1 virtio-blk with 2 vectors: config, Request 1286 // 1 virtio-blk with 2 vectors: config, Request 1287 // 1 virtio-rng with 2 vectors: config, Request 1288 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1289 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1290 // Based on the above, the total vectors should 14. 1291 #[cfg(target_arch = "x86_64")] 1292 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1293 #[cfg(target_arch = "aarch64")] 1294 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1295 assert_eq!( 1296 guest 1297 .ssh_command(grep_cmd) 1298 .unwrap() 1299 .trim() 1300 .parse::<u32>() 1301 .unwrap_or_default(), 1302 10 + (num_queues as u32) 1303 ); 1304 1305 // ACPI feature is needed. 1306 #[cfg(target_arch = "x86_64")] 1307 { 1308 guest.enable_memory_hotplug(); 1309 1310 // Add RAM to the VM 1311 let desired_ram = 1024 << 20; 1312 resize_command(&api_socket, None, Some(desired_ram), None, None); 1313 1314 thread::sleep(std::time::Duration::new(10, 0)); 1315 1316 // Here by simply checking the size (through ssh), we validate 1317 // the connection is still working, which means vhost-user-net 1318 // keeps working after the resize. 1319 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1320 } 1321 }); 1322 1323 let _ = child.kill(); 1324 let output = child.wait_with_output().unwrap(); 1325 1326 thread::sleep(std::time::Duration::new(5, 0)); 1327 let _ = daemon_child.kill(); 1328 let _ = daemon_child.wait(); 1329 1330 handle_child_output(r, &output); 1331 } 1332 1333 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1334 1335 fn test_vhost_user_blk( 1336 num_queues: usize, 1337 readonly: bool, 1338 direct: bool, 1339 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1340 ) { 1341 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1342 let guest = Guest::new(Box::new(focal)); 1343 let api_socket = temp_api_path(&guest.tmp_dir); 1344 1345 let kernel_path = direct_kernel_boot_path(); 1346 1347 let (blk_params, daemon_child) = { 1348 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1349 // Start the daemon 1350 let (daemon_child, vubd_socket_path) = 1351 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1352 1353 ( 1354 format!( 1355 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1356 ), 1357 Some(daemon_child), 1358 ) 1359 }; 1360 1361 let mut child = GuestCommand::new(&guest) 1362 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1363 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1364 .args(["--kernel", kernel_path.to_str().unwrap()]) 1365 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1366 .args([ 1367 "--disk", 1368 format!( 1369 "path={}", 1370 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1371 ) 1372 .as_str(), 1373 format!( 1374 "path={}", 1375 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1376 ) 1377 .as_str(), 1378 blk_params.as_str(), 1379 ]) 1380 .default_net() 1381 .args(["--api-socket", &api_socket]) 1382 .capture_output() 1383 .spawn() 1384 .unwrap(); 1385 1386 let r = std::panic::catch_unwind(|| { 1387 guest.wait_vm_boot(None).unwrap(); 1388 1389 // Check both if /dev/vdc exists and if the block size is 16M. 1390 assert_eq!( 1391 guest 1392 .ssh_command("lsblk | grep vdc | grep -c 16M") 1393 .unwrap() 1394 .trim() 1395 .parse::<u32>() 1396 .unwrap_or_default(), 1397 1 1398 ); 1399 1400 // Check if this block is RO or RW. 1401 assert_eq!( 1402 guest 1403 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1404 .unwrap() 1405 .trim() 1406 .parse::<u32>() 1407 .unwrap_or_default(), 1408 readonly as u32 1409 ); 1410 1411 // Check if the number of queues in /sys/block/vdc/mq matches the 1412 // expected num_queues. 1413 assert_eq!( 1414 guest 1415 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1416 .unwrap() 1417 .trim() 1418 .parse::<u32>() 1419 .unwrap_or_default(), 1420 num_queues as u32 1421 ); 1422 1423 // Mount the device 1424 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1425 guest.ssh_command("mkdir mount_image").unwrap(); 1426 guest 1427 .ssh_command( 1428 format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), 1429 ) 1430 .unwrap(); 1431 1432 // Check the content of the block device. The file "foo" should 1433 // contain "bar". 1434 assert_eq!( 1435 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1436 "bar" 1437 ); 1438 1439 // ACPI feature is needed. 1440 #[cfg(target_arch = "x86_64")] 1441 { 1442 guest.enable_memory_hotplug(); 1443 1444 // Add RAM to the VM 1445 let desired_ram = 1024 << 20; 1446 resize_command(&api_socket, None, Some(desired_ram), None, None); 1447 1448 thread::sleep(std::time::Duration::new(10, 0)); 1449 1450 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1451 1452 // Check again the content of the block device after the resize 1453 // has been performed. 1454 assert_eq!( 1455 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1456 "bar" 1457 ); 1458 } 1459 1460 // Unmount the device 1461 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1462 guest.ssh_command("rm -r mount_image").unwrap(); 1463 }); 1464 1465 let _ = child.kill(); 1466 let output = child.wait_with_output().unwrap(); 1467 1468 if let Some(mut daemon_child) = daemon_child { 1469 thread::sleep(std::time::Duration::new(5, 0)); 1470 let _ = daemon_child.kill(); 1471 let _ = daemon_child.wait(); 1472 } 1473 1474 handle_child_output(r, &output); 1475 } 1476 1477 fn test_boot_from_vhost_user_blk( 1478 num_queues: usize, 1479 readonly: bool, 1480 direct: bool, 1481 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1482 ) { 1483 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1484 let guest = Guest::new(Box::new(focal)); 1485 1486 let kernel_path = direct_kernel_boot_path(); 1487 1488 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1489 1490 let (blk_boot_params, daemon_child) = { 1491 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1492 // Start the daemon 1493 let (daemon_child, vubd_socket_path) = prepare_daemon( 1494 &guest.tmp_dir, 1495 disk_path.as_str(), 1496 num_queues, 1497 readonly, 1498 direct, 1499 ); 1500 1501 ( 1502 format!( 1503 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1504 ), 1505 Some(daemon_child), 1506 ) 1507 }; 1508 1509 let mut child = GuestCommand::new(&guest) 1510 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1511 .args(["--memory", "size=512M,shared=on"]) 1512 .args(["--kernel", kernel_path.to_str().unwrap()]) 1513 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1514 .args([ 1515 "--disk", 1516 blk_boot_params.as_str(), 1517 format!( 1518 "path={}", 1519 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1520 ) 1521 .as_str(), 1522 ]) 1523 .default_net() 1524 .capture_output() 1525 .spawn() 1526 .unwrap(); 1527 1528 let r = std::panic::catch_unwind(|| { 1529 guest.wait_vm_boot(None).unwrap(); 1530 1531 // Just check the VM booted correctly. 1532 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1533 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1534 }); 1535 let _ = child.kill(); 1536 let output = child.wait_with_output().unwrap(); 1537 1538 if let Some(mut daemon_child) = daemon_child { 1539 thread::sleep(std::time::Duration::new(5, 0)); 1540 let _ = daemon_child.kill(); 1541 let _ = daemon_child.wait(); 1542 } 1543 1544 handle_child_output(r, &output); 1545 } 1546 1547 fn _test_virtio_fs( 1548 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1549 hotplug: bool, 1550 pci_segment: Option<u16>, 1551 ) { 1552 #[cfg(target_arch = "aarch64")] 1553 let focal_image = if hotplug { 1554 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1555 } else { 1556 FOCAL_IMAGE_NAME.to_string() 1557 }; 1558 #[cfg(target_arch = "x86_64")] 1559 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1560 let focal = UbuntuDiskConfig::new(focal_image); 1561 let guest = Guest::new(Box::new(focal)); 1562 let api_socket = temp_api_path(&guest.tmp_dir); 1563 1564 let mut workload_path = dirs::home_dir().unwrap(); 1565 workload_path.push("workloads"); 1566 1567 let mut shared_dir = workload_path; 1568 shared_dir.push("shared_dir"); 1569 1570 #[cfg(target_arch = "x86_64")] 1571 let kernel_path = direct_kernel_boot_path(); 1572 #[cfg(target_arch = "aarch64")] 1573 let kernel_path = if hotplug { 1574 edk2_path() 1575 } else { 1576 direct_kernel_boot_path() 1577 }; 1578 1579 let (mut daemon_child, virtiofsd_socket_path) = 1580 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1581 1582 let mut guest_command = GuestCommand::new(&guest); 1583 guest_command 1584 .args(["--cpus", "boot=1"]) 1585 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1586 .args(["--kernel", kernel_path.to_str().unwrap()]) 1587 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1588 .default_disks() 1589 .default_net() 1590 .args(["--api-socket", &api_socket]); 1591 if pci_segment.is_some() { 1592 guest_command.args([ 1593 "--platform", 1594 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 1595 ]); 1596 } 1597 1598 let fs_params = format!( 1599 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1600 virtiofsd_socket_path, 1601 if let Some(pci_segment) = pci_segment { 1602 format!(",pci_segment={pci_segment}") 1603 } else { 1604 "".to_owned() 1605 } 1606 ); 1607 1608 if !hotplug { 1609 guest_command.args(["--fs", fs_params.as_str()]); 1610 } 1611 1612 let mut child = guest_command.capture_output().spawn().unwrap(); 1613 1614 let r = std::panic::catch_unwind(|| { 1615 guest.wait_vm_boot(None).unwrap(); 1616 1617 if hotplug { 1618 // Add fs to the VM 1619 let (cmd_success, cmd_output) = 1620 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1621 assert!(cmd_success); 1622 1623 if let Some(pci_segment) = pci_segment { 1624 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1625 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1626 ))); 1627 } else { 1628 assert!(String::from_utf8_lossy(&cmd_output) 1629 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1630 } 1631 1632 thread::sleep(std::time::Duration::new(10, 0)); 1633 } 1634 1635 // Mount shared directory through virtio_fs filesystem 1636 guest 1637 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1638 .unwrap(); 1639 1640 // Check file1 exists and its content is "foo" 1641 assert_eq!( 1642 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1643 "foo" 1644 ); 1645 // Check file2 does not exist 1646 guest 1647 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1648 .unwrap(); 1649 1650 // Check file3 exists and its content is "bar" 1651 assert_eq!( 1652 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1653 "bar" 1654 ); 1655 1656 // ACPI feature is needed. 1657 #[cfg(target_arch = "x86_64")] 1658 { 1659 guest.enable_memory_hotplug(); 1660 1661 // Add RAM to the VM 1662 let desired_ram = 1024 << 20; 1663 resize_command(&api_socket, None, Some(desired_ram), None, None); 1664 1665 thread::sleep(std::time::Duration::new(30, 0)); 1666 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1667 1668 // After the resize, check again that file1 exists and its 1669 // content is "foo". 1670 assert_eq!( 1671 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1672 "foo" 1673 ); 1674 } 1675 1676 if hotplug { 1677 // Remove from VM 1678 guest.ssh_command("sudo umount mount_dir").unwrap(); 1679 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1680 } 1681 }); 1682 1683 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1684 thread::sleep(std::time::Duration::new(10, 0)); 1685 let (daemon_child, virtiofsd_socket_path) = 1686 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1687 1688 let r = std::panic::catch_unwind(|| { 1689 thread::sleep(std::time::Duration::new(10, 0)); 1690 let fs_params = format!( 1691 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1692 virtiofsd_socket_path, 1693 if let Some(pci_segment) = pci_segment { 1694 format!(",pci_segment={pci_segment}") 1695 } else { 1696 "".to_owned() 1697 } 1698 ); 1699 1700 // Add back and check it works 1701 let (cmd_success, cmd_output) = 1702 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1703 assert!(cmd_success); 1704 if let Some(pci_segment) = pci_segment { 1705 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1706 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1707 ))); 1708 } else { 1709 assert!(String::from_utf8_lossy(&cmd_output) 1710 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1711 } 1712 1713 thread::sleep(std::time::Duration::new(10, 0)); 1714 // Mount shared directory through virtio_fs filesystem 1715 guest 1716 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1717 .unwrap(); 1718 1719 // Check file1 exists and its content is "foo" 1720 assert_eq!( 1721 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1722 "foo" 1723 ); 1724 }); 1725 1726 (r, Some(daemon_child)) 1727 } else { 1728 (r, None) 1729 }; 1730 1731 let _ = child.kill(); 1732 let output = child.wait_with_output().unwrap(); 1733 1734 let _ = daemon_child.kill(); 1735 let _ = daemon_child.wait(); 1736 1737 if let Some(mut daemon_child) = hotplug_daemon_child { 1738 let _ = daemon_child.kill(); 1739 let _ = daemon_child.wait(); 1740 } 1741 1742 handle_child_output(r, &output); 1743 } 1744 1745 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1746 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1747 let guest = Guest::new(Box::new(focal)); 1748 1749 let kernel_path = direct_kernel_boot_path(); 1750 1751 let pmem_temp_file = TempFile::new().unwrap(); 1752 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1753 1754 std::process::Command::new("mkfs.ext4") 1755 .arg(pmem_temp_file.as_path()) 1756 .output() 1757 .expect("Expect creating disk image to succeed"); 1758 1759 let mut child = GuestCommand::new(&guest) 1760 .args(["--cpus", "boot=1"]) 1761 .args(["--memory", "size=512M"]) 1762 .args(["--kernel", kernel_path.to_str().unwrap()]) 1763 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1764 .default_disks() 1765 .default_net() 1766 .args([ 1767 "--pmem", 1768 format!( 1769 "file={}{}{}", 1770 pmem_temp_file.as_path().to_str().unwrap(), 1771 if specify_size { ",size=128M" } else { "" }, 1772 if discard_writes { 1773 ",discard_writes=on" 1774 } else { 1775 "" 1776 } 1777 ) 1778 .as_str(), 1779 ]) 1780 .capture_output() 1781 .spawn() 1782 .unwrap(); 1783 1784 let r = std::panic::catch_unwind(|| { 1785 guest.wait_vm_boot(None).unwrap(); 1786 1787 // Check for the presence of /dev/pmem0 1788 assert_eq!( 1789 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1790 "/dev/pmem0" 1791 ); 1792 1793 // Check changes persist after reboot 1794 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1795 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1796 guest 1797 .ssh_command("echo test123 | sudo tee /mnt/test") 1798 .unwrap(); 1799 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1800 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1801 1802 guest.reboot_linux(0, None); 1803 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1804 assert_eq!( 1805 guest 1806 .ssh_command("sudo cat /mnt/test || true") 1807 .unwrap() 1808 .trim(), 1809 if discard_writes { "" } else { "test123" } 1810 ); 1811 }); 1812 1813 let _ = child.kill(); 1814 let output = child.wait_with_output().unwrap(); 1815 1816 handle_child_output(r, &output); 1817 } 1818 1819 fn get_fd_count(pid: u32) -> usize { 1820 fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() 1821 } 1822 1823 fn _test_virtio_vsock(hotplug: bool) { 1824 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1825 let guest = Guest::new(Box::new(focal)); 1826 1827 #[cfg(target_arch = "x86_64")] 1828 let kernel_path = direct_kernel_boot_path(); 1829 #[cfg(target_arch = "aarch64")] 1830 let kernel_path = if hotplug { 1831 edk2_path() 1832 } else { 1833 direct_kernel_boot_path() 1834 }; 1835 1836 let socket = temp_vsock_path(&guest.tmp_dir); 1837 let api_socket = temp_api_path(&guest.tmp_dir); 1838 1839 let mut cmd = GuestCommand::new(&guest); 1840 cmd.args(["--api-socket", &api_socket]); 1841 cmd.args(["--cpus", "boot=1"]); 1842 cmd.args(["--memory", "size=512M"]); 1843 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1844 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1845 cmd.default_disks(); 1846 cmd.default_net(); 1847 1848 if !hotplug { 1849 cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); 1850 } 1851 1852 let mut child = cmd.capture_output().spawn().unwrap(); 1853 1854 let r = std::panic::catch_unwind(|| { 1855 guest.wait_vm_boot(None).unwrap(); 1856 1857 if hotplug { 1858 let (cmd_success, cmd_output) = remote_command_w_output( 1859 &api_socket, 1860 "add-vsock", 1861 Some(format!("cid=3,socket={socket},id=test0").as_str()), 1862 ); 1863 assert!(cmd_success); 1864 assert!(String::from_utf8_lossy(&cmd_output) 1865 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1866 thread::sleep(std::time::Duration::new(10, 0)); 1867 // Check adding a second one fails 1868 assert!(!remote_command( 1869 &api_socket, 1870 "add-vsock", 1871 Some("cid=1234,socket=/tmp/fail") 1872 )); 1873 } 1874 1875 // Validate vsock works as expected. 1876 guest.check_vsock(socket.as_str()); 1877 guest.reboot_linux(0, None); 1878 // Validate vsock still works after a reboot. 1879 guest.check_vsock(socket.as_str()); 1880 1881 if hotplug { 1882 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1883 } 1884 }); 1885 1886 let _ = child.kill(); 1887 let output = child.wait_with_output().unwrap(); 1888 1889 handle_child_output(r, &output); 1890 } 1891 1892 fn get_ksm_pages_shared() -> u32 { 1893 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1894 .unwrap() 1895 .trim() 1896 .parse::<u32>() 1897 .unwrap() 1898 } 1899 1900 fn test_memory_mergeable(mergeable: bool) { 1901 let memory_param = if mergeable { 1902 "mergeable=on" 1903 } else { 1904 "mergeable=off" 1905 }; 1906 1907 // We are assuming the rest of the system in our CI is not using mergeable memory 1908 let ksm_ps_init = get_ksm_pages_shared(); 1909 assert!(ksm_ps_init == 0); 1910 1911 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1912 let guest1 = Guest::new(Box::new(focal1)); 1913 let mut child1 = GuestCommand::new(&guest1) 1914 .args(["--cpus", "boot=1"]) 1915 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1916 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1917 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1918 .default_disks() 1919 .args(["--net", guest1.default_net_string().as_str()]) 1920 .args(["--serial", "tty", "--console", "off"]) 1921 .capture_output() 1922 .spawn() 1923 .unwrap(); 1924 1925 let r = std::panic::catch_unwind(|| { 1926 guest1.wait_vm_boot(None).unwrap(); 1927 }); 1928 if r.is_err() { 1929 let _ = child1.kill(); 1930 let output = child1.wait_with_output().unwrap(); 1931 handle_child_output(r, &output); 1932 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1933 } 1934 1935 let ksm_ps_guest1 = get_ksm_pages_shared(); 1936 1937 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1938 let guest2 = Guest::new(Box::new(focal2)); 1939 let mut child2 = GuestCommand::new(&guest2) 1940 .args(["--cpus", "boot=1"]) 1941 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1942 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1943 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1944 .default_disks() 1945 .args(["--net", guest2.default_net_string().as_str()]) 1946 .args(["--serial", "tty", "--console", "off"]) 1947 .capture_output() 1948 .spawn() 1949 .unwrap(); 1950 1951 let r = std::panic::catch_unwind(|| { 1952 guest2.wait_vm_boot(None).unwrap(); 1953 let ksm_ps_guest2 = get_ksm_pages_shared(); 1954 1955 if mergeable { 1956 println!( 1957 "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" 1958 ); 1959 // We are expecting the number of shared pages to increase as the number of VM increases 1960 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1961 } else { 1962 assert!(ksm_ps_guest1 == 0); 1963 assert!(ksm_ps_guest2 == 0); 1964 } 1965 }); 1966 1967 let _ = child1.kill(); 1968 let _ = child2.kill(); 1969 1970 let output = child1.wait_with_output().unwrap(); 1971 child2.wait().unwrap(); 1972 1973 handle_child_output(r, &output); 1974 } 1975 1976 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 1977 let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); 1978 let reader = io::BufReader::new(smaps); 1979 1980 let mut skip_map: bool = false; 1981 let mut region_name: String = "".to_string(); 1982 let mut region_maps = HashMap::new(); 1983 for line in reader.lines() { 1984 let l = line.unwrap(); 1985 1986 if l.contains('-') { 1987 let values: Vec<&str> = l.split_whitespace().collect(); 1988 region_name = values.last().unwrap().trim().to_string(); 1989 if region_name == "0" { 1990 region_name = "anonymous".to_string() 1991 } 1992 } 1993 1994 // Each section begins with something that looks like: 1995 // Size: 2184 kB 1996 if l.starts_with("Size:") { 1997 let values: Vec<&str> = l.split_whitespace().collect(); 1998 let map_size = values[1].parse::<u32>().unwrap(); 1999 // We skip the assigned guest RAM map, its RSS is only 2000 // dependent on the guest actual memory usage. 2001 // Everything else can be added to the VMM overhead. 2002 skip_map = map_size >= guest_memory_size; 2003 continue; 2004 } 2005 2006 // If this is a map we're taking into account, then we only 2007 // count the RSS. The sum of all counted RSS is the VMM overhead. 2008 if !skip_map && l.starts_with("Rss:") { 2009 let values: Vec<&str> = l.split_whitespace().collect(); 2010 let value = values[1].trim().parse::<u32>().unwrap(); 2011 *region_maps.entry(region_name.clone()).or_insert(0) += value; 2012 } 2013 } 2014 2015 region_maps 2016 } 2017 2018 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 2019 let mut total = 0; 2020 2021 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 2022 eprintln!("{region_name}: {value}"); 2023 total += value; 2024 } 2025 2026 total 2027 } 2028 2029 fn process_rss_kib(pid: u32) -> usize { 2030 let command = format!("ps -q {pid} -o rss="); 2031 let rss = exec_host_command_output(&command); 2032 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 2033 } 2034 2035 // 10MB is our maximum accepted overhead. 2036 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 2037 2038 #[derive(PartialEq, Eq, PartialOrd)] 2039 struct Counters { 2040 rx_bytes: u64, 2041 rx_frames: u64, 2042 tx_bytes: u64, 2043 tx_frames: u64, 2044 read_bytes: u64, 2045 write_bytes: u64, 2046 read_ops: u64, 2047 write_ops: u64, 2048 } 2049 2050 fn get_counters(api_socket: &str) -> Counters { 2051 // Get counters 2052 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 2053 assert!(cmd_success); 2054 2055 let counters: HashMap<&str, HashMap<&str, u64>> = 2056 serde_json::from_slice(&cmd_output).unwrap_or_default(); 2057 2058 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 2059 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 2060 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 2061 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 2062 2063 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 2064 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 2065 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 2066 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 2067 2068 Counters { 2069 rx_bytes, 2070 rx_frames, 2071 tx_bytes, 2072 tx_frames, 2073 read_bytes, 2074 write_bytes, 2075 read_ops, 2076 write_ops, 2077 } 2078 } 2079 2080 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 2081 let (tx, rx) = mpsc::channel::<String>(); 2082 thread::spawn(move || loop { 2083 thread::sleep(std::time::Duration::new(1, 0)); 2084 let mut buf = [0; 512]; 2085 match pty.read(&mut buf) { 2086 Ok(_) => { 2087 let output = std::str::from_utf8(&buf).unwrap().to_string(); 2088 match tx.send(output) { 2089 Ok(_) => (), 2090 Err(_) => break, 2091 } 2092 } 2093 Err(_) => break, 2094 } 2095 }); 2096 rx 2097 } 2098 2099 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 2100 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2101 assert!(cmd_success); 2102 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2103 assert_eq!("Pty", info["config"][pty_type]["mode"]); 2104 PathBuf::from( 2105 info["config"][pty_type]["file"] 2106 .as_str() 2107 .expect("Missing pty path"), 2108 ) 2109 } 2110 2111 // VFIO test network setup. 2112 // We reserve a different IP class for it: 172.18.0.0/24. 2113 #[cfg(target_arch = "x86_64")] 2114 fn setup_vfio_network_interfaces() { 2115 // 'vfio-br0' 2116 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 2117 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 2118 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 2119 // 'vfio-tap0' 2120 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 2121 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 2122 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 2123 // 'vfio-tap1' 2124 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 2125 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 2126 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 2127 // 'vfio-tap2' 2128 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 2129 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 2130 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 2131 // 'vfio-tap3' 2132 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 2133 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 2134 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 2135 } 2136 2137 // Tear VFIO test network down 2138 #[cfg(target_arch = "x86_64")] 2139 fn cleanup_vfio_network_interfaces() { 2140 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 2141 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 2142 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 2143 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 2144 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 2145 } 2146 2147 fn balloon_size(api_socket: &str) -> u64 { 2148 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2149 assert!(cmd_success); 2150 2151 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2152 let total_mem = &info["config"]["memory"]["size"] 2153 .to_string() 2154 .parse::<u64>() 2155 .unwrap(); 2156 let actual_mem = &info["memory_actual_size"] 2157 .to_string() 2158 .parse::<u64>() 2159 .unwrap(); 2160 total_mem - actual_mem 2161 } 2162 2163 fn vm_state(api_socket: &str) -> String { 2164 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2165 assert!(cmd_success); 2166 2167 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2168 let state = &info["state"].as_str().unwrap(); 2169 2170 state.to_string() 2171 } 2172 2173 // This test validates that it can find the virtio-iommu device at first. 2174 // It also verifies that both disks and the network card are attached to 2175 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 2176 // The last interesting part of this test is that it exercises the network 2177 // interface attached to the virtual IOMMU since this is the one used to 2178 // send all commands through SSH. 2179 fn _test_virtio_iommu(acpi: bool) { 2180 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 2181 // Focal image is still old. 2182 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 2183 // the kernel binary has been updated. 2184 #[cfg(target_arch = "aarch64")] 2185 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 2186 #[cfg(target_arch = "x86_64")] 2187 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2188 let focal = UbuntuDiskConfig::new(focal_image); 2189 let guest = Guest::new(Box::new(focal)); 2190 2191 #[cfg(target_arch = "x86_64")] 2192 let kernel_path = direct_kernel_boot_path(); 2193 #[cfg(target_arch = "aarch64")] 2194 let kernel_path = if acpi { 2195 edk2_path() 2196 } else { 2197 direct_kernel_boot_path() 2198 }; 2199 2200 let mut child = GuestCommand::new(&guest) 2201 .args(["--cpus", "boot=1"]) 2202 .args(["--memory", "size=512M"]) 2203 .args(["--kernel", kernel_path.to_str().unwrap()]) 2204 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2205 .args([ 2206 "--disk", 2207 format!( 2208 "path={},iommu=on", 2209 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2210 ) 2211 .as_str(), 2212 format!( 2213 "path={},iommu=on", 2214 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2215 ) 2216 .as_str(), 2217 ]) 2218 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 2219 .capture_output() 2220 .spawn() 2221 .unwrap(); 2222 2223 let r = std::panic::catch_unwind(|| { 2224 guest.wait_vm_boot(None).unwrap(); 2225 2226 // Verify the virtio-iommu device is present. 2227 assert!(guest 2228 .does_device_vendor_pair_match("0x1057", "0x1af4") 2229 .unwrap_or_default()); 2230 2231 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 2232 // different with ACPI. 2233 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 2234 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 2235 // and appear under folder '/sys/kernel/iommu_groups/'. 2236 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 2237 // which is the console. The first disk "0000:00:02.0" is in group '1'. 2238 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 2239 // contains "0000:00:02.0" which is the first disk. 2240 // 2241 // Verify the iommu group of the first disk. 2242 let iommu_group = !acpi as i32; 2243 assert_eq!( 2244 guest 2245 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2246 .unwrap() 2247 .trim(), 2248 "0000:00:02.0" 2249 ); 2250 2251 // Verify the iommu group of the second disk. 2252 let iommu_group = if acpi { 1 } else { 2 }; 2253 assert_eq!( 2254 guest 2255 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2256 .unwrap() 2257 .trim(), 2258 "0000:00:03.0" 2259 ); 2260 2261 // Verify the iommu group of the network card. 2262 let iommu_group = if acpi { 2 } else { 3 }; 2263 assert_eq!( 2264 guest 2265 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2266 .unwrap() 2267 .trim(), 2268 "0000:00:04.0" 2269 ); 2270 }); 2271 2272 let _ = child.kill(); 2273 let output = child.wait_with_output().unwrap(); 2274 2275 handle_child_output(r, &output); 2276 } 2277 2278 fn get_reboot_count(guest: &Guest) -> u32 { 2279 guest 2280 .ssh_command("sudo last | grep -c reboot") 2281 .unwrap() 2282 .trim() 2283 .parse::<u32>() 2284 .unwrap_or_default() 2285 } 2286 2287 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 2288 // Check for PCI device 2289 assert!(guest 2290 .does_device_vendor_pair_match("0x1063", "0x1af4") 2291 .unwrap_or_default()); 2292 2293 // Enable systemd watchdog 2294 guest 2295 .ssh_command(&format!( 2296 "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" 2297 )) 2298 .unwrap(); 2299 } 2300 2301 fn make_guest_panic(guest: &Guest) { 2302 // Check for pvpanic device 2303 assert!(guest 2304 .does_device_vendor_pair_match("0x0011", "0x1b36") 2305 .unwrap_or_default()); 2306 2307 // Trigger guest a panic 2308 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 2309 } 2310 2311 mod common_parallel { 2312 use std::{fs::OpenOptions, io::SeekFrom}; 2313 2314 use crate::*; 2315 2316 #[test] 2317 #[cfg(target_arch = "x86_64")] 2318 fn test_focal_hypervisor_fw() { 2319 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 2320 } 2321 2322 #[test] 2323 #[cfg(target_arch = "x86_64")] 2324 fn test_focal_ovmf() { 2325 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 2326 } 2327 2328 #[cfg(target_arch = "x86_64")] 2329 fn test_simple_launch(fw_path: String, disk_path: &str) { 2330 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 2331 let guest = Guest::new(disk_config); 2332 let event_path = temp_event_monitor_path(&guest.tmp_dir); 2333 2334 let mut child = GuestCommand::new(&guest) 2335 .args(["--cpus", "boot=1"]) 2336 .args(["--memory", "size=512M"]) 2337 .args(["--kernel", fw_path.as_str()]) 2338 .default_disks() 2339 .default_net() 2340 .args(["--serial", "tty", "--console", "off"]) 2341 .args(["--event-monitor", format!("path={event_path}").as_str()]) 2342 .capture_output() 2343 .spawn() 2344 .unwrap(); 2345 2346 let r = std::panic::catch_unwind(|| { 2347 guest.wait_vm_boot(Some(120)).unwrap(); 2348 2349 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2350 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2351 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2352 2353 let expected_sequential_events = [ 2354 &MetaEvent { 2355 event: "starting".to_string(), 2356 device_id: None, 2357 }, 2358 &MetaEvent { 2359 event: "booting".to_string(), 2360 device_id: None, 2361 }, 2362 &MetaEvent { 2363 event: "booted".to_string(), 2364 device_id: None, 2365 }, 2366 &MetaEvent { 2367 event: "activated".to_string(), 2368 device_id: Some("_disk0".to_string()), 2369 }, 2370 &MetaEvent { 2371 event: "reset".to_string(), 2372 device_id: Some("_disk0".to_string()), 2373 }, 2374 ]; 2375 assert!(check_sequential_events( 2376 &expected_sequential_events, 2377 &event_path 2378 )); 2379 2380 // It's been observed on the Bionic image that udev and snapd 2381 // services can cause some delay in the VM's shutdown. Disabling 2382 // them improves the reliability of this test. 2383 let _ = guest.ssh_command("sudo systemctl disable udev"); 2384 let _ = guest.ssh_command("sudo systemctl stop udev"); 2385 let _ = guest.ssh_command("sudo systemctl disable snapd"); 2386 let _ = guest.ssh_command("sudo systemctl stop snapd"); 2387 2388 guest.ssh_command("sudo poweroff").unwrap(); 2389 thread::sleep(std::time::Duration::new(20, 0)); 2390 let latest_events = [ 2391 &MetaEvent { 2392 event: "shutdown".to_string(), 2393 device_id: None, 2394 }, 2395 &MetaEvent { 2396 event: "deleted".to_string(), 2397 device_id: None, 2398 }, 2399 &MetaEvent { 2400 event: "shutdown".to_string(), 2401 device_id: None, 2402 }, 2403 ]; 2404 assert!(check_latest_events_exact(&latest_events, &event_path)); 2405 }); 2406 2407 let _ = child.kill(); 2408 let output = child.wait_with_output().unwrap(); 2409 2410 handle_child_output(r, &output); 2411 } 2412 2413 #[test] 2414 fn test_multi_cpu() { 2415 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2416 let jammy = UbuntuDiskConfig::new(jammy_image); 2417 let guest = Guest::new(Box::new(jammy)); 2418 2419 let mut cmd = GuestCommand::new(&guest); 2420 cmd.args(["--cpus", "boot=2,max=4"]) 2421 .args(["--memory", "size=512M"]) 2422 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2423 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2424 .capture_output() 2425 .default_disks() 2426 .default_net(); 2427 2428 let mut child = cmd.spawn().unwrap(); 2429 2430 let r = std::panic::catch_unwind(|| { 2431 guest.wait_vm_boot(Some(120)).unwrap(); 2432 2433 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2434 2435 assert_eq!( 2436 guest 2437 .ssh_command( 2438 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2439 ) 2440 .unwrap() 2441 .trim(), 2442 "smp: Brought up 1 node, 2 CPUs" 2443 ); 2444 }); 2445 2446 let _ = child.kill(); 2447 let output = child.wait_with_output().unwrap(); 2448 2449 handle_child_output(r, &output); 2450 } 2451 2452 #[test] 2453 fn test_cpu_topology_421() { 2454 test_cpu_topology(4, 2, 1, false); 2455 } 2456 2457 #[test] 2458 fn test_cpu_topology_142() { 2459 test_cpu_topology(1, 4, 2, false); 2460 } 2461 2462 #[test] 2463 fn test_cpu_topology_262() { 2464 test_cpu_topology(2, 6, 2, false); 2465 } 2466 2467 #[test] 2468 #[cfg(target_arch = "x86_64")] 2469 #[cfg(not(feature = "mshv"))] 2470 fn test_cpu_physical_bits() { 2471 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2472 let guest = Guest::new(Box::new(focal)); 2473 let max_phys_bits: u8 = 36; 2474 let mut child = GuestCommand::new(&guest) 2475 .args(["--cpus", &format!("max_phys_bits={max_phys_bits}")]) 2476 .args(["--memory", "size=512M"]) 2477 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2478 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2479 .default_disks() 2480 .default_net() 2481 .capture_output() 2482 .spawn() 2483 .unwrap(); 2484 2485 let r = std::panic::catch_unwind(|| { 2486 guest.wait_vm_boot(None).unwrap(); 2487 2488 assert!( 2489 guest 2490 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2491 .unwrap() 2492 .trim() 2493 .parse::<u8>() 2494 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2495 ); 2496 }); 2497 2498 let _ = child.kill(); 2499 let output = child.wait_with_output().unwrap(); 2500 2501 handle_child_output(r, &output); 2502 } 2503 2504 #[test] 2505 fn test_cpu_affinity() { 2506 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2507 let guest = Guest::new(Box::new(focal)); 2508 2509 // We need the host to have at least 4 CPUs if we want to be able 2510 // to run this test. 2511 let host_cpus_count = exec_host_command_output("nproc"); 2512 assert!( 2513 String::from_utf8_lossy(&host_cpus_count.stdout) 2514 .trim() 2515 .parse::<u16>() 2516 .unwrap_or(0) 2517 >= 4 2518 ); 2519 2520 let mut child = GuestCommand::new(&guest) 2521 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2522 .args(["--memory", "size=512M"]) 2523 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2524 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2525 .default_disks() 2526 .default_net() 2527 .capture_output() 2528 .spawn() 2529 .unwrap(); 2530 2531 let r = std::panic::catch_unwind(|| { 2532 guest.wait_vm_boot(None).unwrap(); 2533 let pid = child.id(); 2534 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2535 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2536 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2537 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2538 }); 2539 2540 let _ = child.kill(); 2541 let output = child.wait_with_output().unwrap(); 2542 2543 handle_child_output(r, &output); 2544 } 2545 2546 #[test] 2547 #[cfg(not(feature = "mshv"))] 2548 fn test_large_vm() { 2549 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2550 let guest = Guest::new(Box::new(focal)); 2551 let mut cmd = GuestCommand::new(&guest); 2552 cmd.args(["--cpus", "boot=48"]) 2553 .args(["--memory", "size=5120M"]) 2554 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2555 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2556 .args(["--serial", "tty"]) 2557 .args(["--console", "off"]) 2558 .capture_output() 2559 .default_disks() 2560 .default_net(); 2561 2562 let mut child = cmd.spawn().unwrap(); 2563 2564 guest.wait_vm_boot(None).unwrap(); 2565 2566 let r = std::panic::catch_unwind(|| { 2567 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2568 assert_eq!( 2569 guest 2570 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2571 .unwrap() 2572 .trim(), 2573 "0-47" 2574 ); 2575 2576 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2577 }); 2578 2579 let _ = child.kill(); 2580 let output = child.wait_with_output().unwrap(); 2581 2582 handle_child_output(r, &output); 2583 } 2584 2585 #[test] 2586 #[cfg(not(feature = "mshv"))] 2587 fn test_huge_memory() { 2588 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2589 let guest = Guest::new(Box::new(focal)); 2590 let mut cmd = GuestCommand::new(&guest); 2591 cmd.args(["--cpus", "boot=1"]) 2592 .args(["--memory", "size=128G"]) 2593 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2594 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2595 .capture_output() 2596 .default_disks() 2597 .default_net(); 2598 2599 let mut child = cmd.spawn().unwrap(); 2600 2601 guest.wait_vm_boot(Some(120)).unwrap(); 2602 2603 let r = std::panic::catch_unwind(|| { 2604 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2605 }); 2606 2607 let _ = child.kill(); 2608 let output = child.wait_with_output().unwrap(); 2609 2610 handle_child_output(r, &output); 2611 } 2612 2613 #[test] 2614 fn test_power_button() { 2615 _test_power_button(false); 2616 } 2617 2618 #[test] 2619 #[cfg(not(feature = "mshv"))] 2620 fn test_user_defined_memory_regions() { 2621 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2622 let guest = Guest::new(Box::new(focal)); 2623 let api_socket = temp_api_path(&guest.tmp_dir); 2624 2625 let kernel_path = direct_kernel_boot_path(); 2626 2627 let mut child = GuestCommand::new(&guest) 2628 .args(["--cpus", "boot=1"]) 2629 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2630 .args([ 2631 "--memory-zone", 2632 "id=mem0,size=1G,hotplug_size=2G", 2633 "id=mem1,size=1G,shared=on", 2634 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2635 ]) 2636 .args(["--kernel", kernel_path.to_str().unwrap()]) 2637 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2638 .args(["--api-socket", &api_socket]) 2639 .capture_output() 2640 .default_disks() 2641 .default_net() 2642 .spawn() 2643 .unwrap(); 2644 2645 let r = std::panic::catch_unwind(|| { 2646 guest.wait_vm_boot(None).unwrap(); 2647 2648 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2649 2650 guest.enable_memory_hotplug(); 2651 2652 resize_zone_command(&api_socket, "mem0", "3G"); 2653 thread::sleep(std::time::Duration::new(5, 0)); 2654 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2655 resize_zone_command(&api_socket, "mem2", "3G"); 2656 thread::sleep(std::time::Duration::new(5, 0)); 2657 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2658 resize_zone_command(&api_socket, "mem0", "2G"); 2659 thread::sleep(std::time::Duration::new(5, 0)); 2660 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2661 resize_zone_command(&api_socket, "mem2", "2G"); 2662 thread::sleep(std::time::Duration::new(5, 0)); 2663 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2664 2665 guest.reboot_linux(0, None); 2666 2667 // Check the amount of RAM after reboot 2668 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2669 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2670 2671 // Check if we can still resize down to the initial 'boot'size 2672 resize_zone_command(&api_socket, "mem0", "1G"); 2673 thread::sleep(std::time::Duration::new(5, 0)); 2674 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2675 resize_zone_command(&api_socket, "mem2", "1G"); 2676 thread::sleep(std::time::Duration::new(5, 0)); 2677 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2678 }); 2679 2680 let _ = child.kill(); 2681 let output = child.wait_with_output().unwrap(); 2682 2683 handle_child_output(r, &output); 2684 } 2685 2686 #[test] 2687 #[cfg(not(feature = "mshv"))] 2688 fn test_guest_numa_nodes() { 2689 _test_guest_numa_nodes(false); 2690 } 2691 2692 #[test] 2693 #[cfg(target_arch = "x86_64")] 2694 fn test_iommu_segments() { 2695 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2696 let focal = UbuntuDiskConfig::new(focal_image); 2697 let guest = Guest::new(Box::new(focal)); 2698 2699 // Prepare another disk file for the virtio-disk device 2700 let test_disk_path = String::from( 2701 guest 2702 .tmp_dir 2703 .as_path() 2704 .join("test-disk.raw") 2705 .to_str() 2706 .unwrap(), 2707 ); 2708 assert!( 2709 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2710 ); 2711 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2712 2713 let api_socket = temp_api_path(&guest.tmp_dir); 2714 let mut cmd = GuestCommand::new(&guest); 2715 2716 cmd.args(["--cpus", "boot=1"]) 2717 .args(["--api-socket", &api_socket]) 2718 .args(["--memory", "size=512M"]) 2719 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2720 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2721 .args([ 2722 "--platform", 2723 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS},iommu_segments=[1]"), 2724 ]) 2725 .default_disks() 2726 .capture_output() 2727 .default_net(); 2728 2729 let mut child = cmd.spawn().unwrap(); 2730 2731 guest.wait_vm_boot(None).unwrap(); 2732 2733 let r = std::panic::catch_unwind(|| { 2734 let (cmd_success, cmd_output) = remote_command_w_output( 2735 &api_socket, 2736 "add-disk", 2737 Some( 2738 format!( 2739 "path={},id=test0,pci_segment=1,iommu=on", 2740 test_disk_path.as_str() 2741 ) 2742 .as_str(), 2743 ), 2744 ); 2745 assert!(cmd_success); 2746 assert!(String::from_utf8_lossy(&cmd_output) 2747 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2748 2749 // Check IOMMU setup 2750 assert!(guest 2751 .does_device_vendor_pair_match("0x1057", "0x1af4") 2752 .unwrap_or_default()); 2753 assert_eq!( 2754 guest 2755 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2756 .unwrap() 2757 .trim(), 2758 "0001:00:01.0" 2759 ); 2760 }); 2761 2762 let _ = child.kill(); 2763 let output = child.wait_with_output().unwrap(); 2764 2765 handle_child_output(r, &output); 2766 } 2767 2768 #[test] 2769 fn test_pci_msi() { 2770 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2771 let guest = Guest::new(Box::new(focal)); 2772 let mut cmd = GuestCommand::new(&guest); 2773 cmd.args(["--cpus", "boot=1"]) 2774 .args(["--memory", "size=512M"]) 2775 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2776 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2777 .capture_output() 2778 .default_disks() 2779 .default_net(); 2780 2781 let mut child = cmd.spawn().unwrap(); 2782 2783 guest.wait_vm_boot(None).unwrap(); 2784 2785 #[cfg(target_arch = "x86_64")] 2786 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2787 #[cfg(target_arch = "aarch64")] 2788 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2789 2790 let r = std::panic::catch_unwind(|| { 2791 assert_eq!( 2792 guest 2793 .ssh_command(grep_cmd) 2794 .unwrap() 2795 .trim() 2796 .parse::<u32>() 2797 .unwrap_or_default(), 2798 12 2799 ); 2800 }); 2801 2802 let _ = child.kill(); 2803 let output = child.wait_with_output().unwrap(); 2804 2805 handle_child_output(r, &output); 2806 } 2807 2808 #[test] 2809 fn test_virtio_net_ctrl_queue() { 2810 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2811 let guest = Guest::new(Box::new(focal)); 2812 let mut cmd = GuestCommand::new(&guest); 2813 cmd.args(["--cpus", "boot=1"]) 2814 .args(["--memory", "size=512M"]) 2815 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2816 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2817 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2818 .capture_output() 2819 .default_disks(); 2820 2821 let mut child = cmd.spawn().unwrap(); 2822 2823 guest.wait_vm_boot(None).unwrap(); 2824 2825 #[cfg(target_arch = "aarch64")] 2826 let iface = "enp0s4"; 2827 #[cfg(target_arch = "x86_64")] 2828 let iface = "ens4"; 2829 2830 let r = std::panic::catch_unwind(|| { 2831 assert_eq!( 2832 guest 2833 .ssh_command( 2834 format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() 2835 ) 2836 .unwrap() 2837 .trim(), 2838 "success" 2839 ); 2840 assert_eq!( 2841 guest 2842 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 2843 .unwrap() 2844 .trim(), 2845 "3000" 2846 ); 2847 }); 2848 2849 let _ = child.kill(); 2850 let output = child.wait_with_output().unwrap(); 2851 2852 handle_child_output(r, &output); 2853 } 2854 2855 #[test] 2856 fn test_pci_multiple_segments() { 2857 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2858 let guest = Guest::new(Box::new(focal)); 2859 2860 // Prepare another disk file for the virtio-disk device 2861 let test_disk_path = String::from( 2862 guest 2863 .tmp_dir 2864 .as_path() 2865 .join("test-disk.raw") 2866 .to_str() 2867 .unwrap(), 2868 ); 2869 assert!( 2870 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2871 ); 2872 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2873 2874 let mut cmd = GuestCommand::new(&guest); 2875 cmd.args(["--cpus", "boot=1"]) 2876 .args(["--memory", "size=512M"]) 2877 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2878 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2879 .args([ 2880 "--platform", 2881 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 2882 ]) 2883 .args([ 2884 "--disk", 2885 format!( 2886 "path={}", 2887 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2888 ) 2889 .as_str(), 2890 format!( 2891 "path={}", 2892 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2893 ) 2894 .as_str(), 2895 format!("path={test_disk_path},pci_segment=15").as_str(), 2896 ]) 2897 .capture_output() 2898 .default_net(); 2899 2900 let mut child = cmd.spawn().unwrap(); 2901 2902 guest.wait_vm_boot(None).unwrap(); 2903 2904 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2905 2906 let r = std::panic::catch_unwind(|| { 2907 // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. 2908 assert_eq!( 2909 guest 2910 .ssh_command(grep_cmd) 2911 .unwrap() 2912 .trim() 2913 .parse::<u16>() 2914 .unwrap_or_default(), 2915 MAX_NUM_PCI_SEGMENTS 2916 ); 2917 2918 // Check both if /dev/vdc exists and if the block size is 4M. 2919 assert_eq!( 2920 guest 2921 .ssh_command("lsblk | grep vdc | grep -c 4M") 2922 .unwrap() 2923 .trim() 2924 .parse::<u32>() 2925 .unwrap_or_default(), 2926 1 2927 ); 2928 2929 // Mount the device. 2930 guest.ssh_command("mkdir mount_image").unwrap(); 2931 guest 2932 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 2933 .unwrap(); 2934 // Grant all users with write permission. 2935 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 2936 2937 // Write something to the device. 2938 guest 2939 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 2940 .unwrap(); 2941 2942 // Check the content of the block device. The file "foo" should 2943 // contain "bar". 2944 assert_eq!( 2945 guest 2946 .ssh_command("sudo cat mount_image/foo") 2947 .unwrap() 2948 .trim(), 2949 "bar" 2950 ); 2951 }); 2952 2953 let _ = child.kill(); 2954 let output = child.wait_with_output().unwrap(); 2955 2956 handle_child_output(r, &output); 2957 } 2958 2959 #[test] 2960 fn test_pci_multiple_segments_numa_node() { 2961 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2962 let guest = Guest::new(Box::new(focal)); 2963 let api_socket = temp_api_path(&guest.tmp_dir); 2964 #[cfg(target_arch = "x86_64")] 2965 let kernel_path = direct_kernel_boot_path(); 2966 #[cfg(target_arch = "aarch64")] 2967 let kernel_path = edk2_path(); 2968 2969 // Prepare another disk file for the virtio-disk device 2970 let test_disk_path = String::from( 2971 guest 2972 .tmp_dir 2973 .as_path() 2974 .join("test-disk.raw") 2975 .to_str() 2976 .unwrap(), 2977 ); 2978 assert!( 2979 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2980 ); 2981 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2982 const TEST_DISK_NODE: u16 = 1; 2983 2984 let mut child = GuestCommand::new(&guest) 2985 .args(["--platform", "num_pci_segments=2"]) 2986 .args(["--cpus", "boot=2"]) 2987 .args(["--memory", "size=0"]) 2988 .args(["--memory-zone", "id=mem0,size=256M", "id=mem1,size=256M"]) 2989 .args([ 2990 "--numa", 2991 "guest_numa_id=0,cpus=[0],distances=[1@20],memory_zones=mem0,pci_segments=[0]", 2992 "guest_numa_id=1,cpus=[1],distances=[0@20],memory_zones=mem1,pci_segments=[1]", 2993 ]) 2994 .args(["--kernel", kernel_path.to_str().unwrap()]) 2995 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2996 .args(["--api-socket", &api_socket]) 2997 .capture_output() 2998 .args([ 2999 "--disk", 3000 format!( 3001 "path={}", 3002 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3003 ) 3004 .as_str(), 3005 format!( 3006 "path={}", 3007 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3008 ) 3009 .as_str(), 3010 format!("path={test_disk_path},pci_segment={TEST_DISK_NODE}").as_str(), 3011 ]) 3012 .default_net() 3013 .spawn() 3014 .unwrap(); 3015 3016 let cmd = "cat /sys/block/vdc/device/../numa_node"; 3017 3018 let r = std::panic::catch_unwind(|| { 3019 guest.wait_vm_boot(None).unwrap(); 3020 3021 assert_eq!( 3022 guest 3023 .ssh_command(cmd) 3024 .unwrap() 3025 .trim() 3026 .parse::<u16>() 3027 .unwrap_or_default(), 3028 TEST_DISK_NODE 3029 ); 3030 }); 3031 3032 let _ = child.kill(); 3033 let output = child.wait_with_output().unwrap(); 3034 3035 handle_child_output(r, &output); 3036 } 3037 3038 #[test] 3039 fn test_direct_kernel_boot() { 3040 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3041 let guest = Guest::new(Box::new(focal)); 3042 3043 let kernel_path = direct_kernel_boot_path(); 3044 3045 let mut child = GuestCommand::new(&guest) 3046 .args(["--cpus", "boot=1"]) 3047 .args(["--memory", "size=512M"]) 3048 .args(["--kernel", kernel_path.to_str().unwrap()]) 3049 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3050 .default_disks() 3051 .default_net() 3052 .capture_output() 3053 .spawn() 3054 .unwrap(); 3055 3056 let r = std::panic::catch_unwind(|| { 3057 guest.wait_vm_boot(None).unwrap(); 3058 3059 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3060 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3061 3062 let grep_cmd = if cfg!(target_arch = "x86_64") { 3063 "grep -c PCI-MSI /proc/interrupts" 3064 } else { 3065 "grep -c ITS-MSI /proc/interrupts" 3066 }; 3067 assert_eq!( 3068 guest 3069 .ssh_command(grep_cmd) 3070 .unwrap() 3071 .trim() 3072 .parse::<u32>() 3073 .unwrap_or_default(), 3074 12 3075 ); 3076 }); 3077 3078 let _ = child.kill(); 3079 let output = child.wait_with_output().unwrap(); 3080 3081 handle_child_output(r, &output); 3082 } 3083 3084 fn _test_virtio_block(image_name: &str, disable_io_uring: bool, disable_aio: bool) { 3085 let focal = UbuntuDiskConfig::new(image_name.to_string()); 3086 let guest = Guest::new(Box::new(focal)); 3087 3088 let mut workload_path = dirs::home_dir().unwrap(); 3089 workload_path.push("workloads"); 3090 3091 let mut blk_file_path = workload_path; 3092 blk_file_path.push("blk.img"); 3093 3094 let kernel_path = direct_kernel_boot_path(); 3095 3096 let mut cloud_child = GuestCommand::new(&guest) 3097 .args(["--cpus", "boot=4"]) 3098 .args(["--memory", "size=512M,shared=on"]) 3099 .args(["--kernel", kernel_path.to_str().unwrap()]) 3100 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3101 .args([ 3102 "--disk", 3103 format!( 3104 "path={}", 3105 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3106 ) 3107 .as_str(), 3108 format!( 3109 "path={}", 3110 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3111 ) 3112 .as_str(), 3113 format!( 3114 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", 3115 blk_file_path.to_str().unwrap(), 3116 disable_io_uring, 3117 disable_aio, 3118 ) 3119 .as_str(), 3120 ]) 3121 .default_net() 3122 .capture_output() 3123 .spawn() 3124 .unwrap(); 3125 3126 let r = std::panic::catch_unwind(|| { 3127 guest.wait_vm_boot(None).unwrap(); 3128 3129 // Check both if /dev/vdc exists and if the block size is 16M. 3130 assert_eq!( 3131 guest 3132 .ssh_command("lsblk | grep vdc | grep -c 16M") 3133 .unwrap() 3134 .trim() 3135 .parse::<u32>() 3136 .unwrap_or_default(), 3137 1 3138 ); 3139 3140 // Check both if /dev/vdc exists and if this block is RO. 3141 assert_eq!( 3142 guest 3143 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 3144 .unwrap() 3145 .trim() 3146 .parse::<u32>() 3147 .unwrap_or_default(), 3148 1 3149 ); 3150 3151 // Check if the number of queues is 4. 3152 assert_eq!( 3153 guest 3154 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 3155 .unwrap() 3156 .trim() 3157 .parse::<u32>() 3158 .unwrap_or_default(), 3159 4 3160 ); 3161 }); 3162 3163 let _ = cloud_child.kill(); 3164 let output = cloud_child.wait_with_output().unwrap(); 3165 3166 handle_child_output(r, &output); 3167 } 3168 3169 #[test] 3170 fn test_virtio_block_io_uring() { 3171 _test_virtio_block(FOCAL_IMAGE_NAME, false, true) 3172 } 3173 3174 #[test] 3175 fn test_virtio_block_aio() { 3176 _test_virtio_block(FOCAL_IMAGE_NAME, true, false) 3177 } 3178 3179 #[test] 3180 fn test_virtio_block_sync() { 3181 _test_virtio_block(FOCAL_IMAGE_NAME, true, true) 3182 } 3183 3184 #[test] 3185 fn test_virtio_block_qcow2() { 3186 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false, false) 3187 } 3188 3189 #[test] 3190 fn test_virtio_block_qcow2_backing_file() { 3191 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE, false, false) 3192 } 3193 3194 #[test] 3195 fn test_virtio_block_vhd() { 3196 let mut workload_path = dirs::home_dir().unwrap(); 3197 workload_path.push("workloads"); 3198 3199 let mut raw_file_path = workload_path.clone(); 3200 let mut vhd_file_path = workload_path; 3201 raw_file_path.push(FOCAL_IMAGE_NAME); 3202 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 3203 3204 // Generate VHD file from RAW file 3205 std::process::Command::new("qemu-img") 3206 .arg("convert") 3207 .arg("-p") 3208 .args(["-f", "raw"]) 3209 .args(["-O", "vpc"]) 3210 .args(["-o", "subformat=fixed"]) 3211 .arg(raw_file_path.to_str().unwrap()) 3212 .arg(vhd_file_path.to_str().unwrap()) 3213 .output() 3214 .expect("Expect generating VHD image from RAW image"); 3215 3216 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false, false) 3217 } 3218 3219 #[test] 3220 fn test_virtio_block_vhdx() { 3221 let mut workload_path = dirs::home_dir().unwrap(); 3222 workload_path.push("workloads"); 3223 3224 let mut raw_file_path = workload_path.clone(); 3225 let mut vhdx_file_path = workload_path; 3226 raw_file_path.push(FOCAL_IMAGE_NAME); 3227 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 3228 3229 // Generate dynamic VHDX file from RAW file 3230 std::process::Command::new("qemu-img") 3231 .arg("convert") 3232 .arg("-p") 3233 .args(["-f", "raw"]) 3234 .args(["-O", "vhdx"]) 3235 .arg(raw_file_path.to_str().unwrap()) 3236 .arg(vhdx_file_path.to_str().unwrap()) 3237 .output() 3238 .expect("Expect generating dynamic VHDx image from RAW image"); 3239 3240 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false, false) 3241 } 3242 3243 #[test] 3244 fn test_virtio_block_dynamic_vhdx_expand() { 3245 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 3246 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 3247 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 3248 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 3249 3250 let mut workload_path = dirs::home_dir().unwrap(); 3251 workload_path.push("workloads"); 3252 3253 let mut vhdx_file_path = workload_path; 3254 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 3255 let vhdx_path = vhdx_file_path.to_str().unwrap(); 3256 3257 // Generate a 100 MiB dynamic VHDX file 3258 std::process::Command::new("qemu-img") 3259 .arg("create") 3260 .args(["-f", "vhdx"]) 3261 .arg(vhdx_path) 3262 .arg(VIRTUAL_DISK_SIZE.to_string()) 3263 .output() 3264 .expect("Expect generating dynamic VHDx image from RAW image"); 3265 3266 // Check if the size matches with empty VHDx file size 3267 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 3268 3269 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3270 let guest = Guest::new(Box::new(focal)); 3271 let kernel_path = direct_kernel_boot_path(); 3272 3273 let mut cloud_child = GuestCommand::new(&guest) 3274 .args(["--cpus", "boot=1"]) 3275 .args(["--memory", "size=512M"]) 3276 .args(["--kernel", kernel_path.to_str().unwrap()]) 3277 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3278 .args([ 3279 "--disk", 3280 format!( 3281 "path={}", 3282 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3283 ) 3284 .as_str(), 3285 format!( 3286 "path={}", 3287 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3288 ) 3289 .as_str(), 3290 format!("path={vhdx_path}").as_str(), 3291 ]) 3292 .default_net() 3293 .capture_output() 3294 .spawn() 3295 .unwrap(); 3296 3297 let r = std::panic::catch_unwind(|| { 3298 guest.wait_vm_boot(None).unwrap(); 3299 3300 // Check both if /dev/vdc exists and if the block size is 100 MiB. 3301 assert_eq!( 3302 guest 3303 .ssh_command("lsblk | grep vdc | grep -c 100M") 3304 .unwrap() 3305 .trim() 3306 .parse::<u32>() 3307 .unwrap_or_default(), 3308 1 3309 ); 3310 3311 // Write 100 MB of data to the VHDx disk 3312 guest 3313 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 3314 .unwrap(); 3315 }); 3316 3317 // Check if the size matches with expected expanded VHDx file size 3318 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 3319 3320 let _ = cloud_child.kill(); 3321 let output = cloud_child.wait_with_output().unwrap(); 3322 3323 handle_child_output(r, &output); 3324 } 3325 3326 fn vhdx_image_size(disk_name: &str) -> u64 { 3327 std::fs::File::open(disk_name) 3328 .unwrap() 3329 .seek(SeekFrom::End(0)) 3330 .unwrap() 3331 } 3332 3333 #[test] 3334 fn test_virtio_block_direct_and_firmware() { 3335 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3336 let guest = Guest::new(Box::new(focal)); 3337 3338 // The OS disk must be copied to a location that is not backed by 3339 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 3340 // with EINVAL because tmpfs doesn't support this flag. 3341 let mut workloads_path = dirs::home_dir().unwrap(); 3342 workloads_path.push("workloads"); 3343 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 3344 let mut os_path = os_dir.as_path().to_path_buf(); 3345 os_path.push("osdisk.img"); 3346 rate_limited_copy( 3347 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3348 os_path.as_path(), 3349 ) 3350 .expect("copying of OS disk failed"); 3351 3352 let mut child = GuestCommand::new(&guest) 3353 .args(["--cpus", "boot=1"]) 3354 .args(["--memory", "size=512M"]) 3355 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 3356 .args([ 3357 "--disk", 3358 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 3359 format!( 3360 "path={}", 3361 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3362 ) 3363 .as_str(), 3364 ]) 3365 .default_net() 3366 .capture_output() 3367 .spawn() 3368 .unwrap(); 3369 3370 let r = std::panic::catch_unwind(|| { 3371 guest.wait_vm_boot(Some(120)).unwrap(); 3372 }); 3373 3374 let _ = child.kill(); 3375 let output = child.wait_with_output().unwrap(); 3376 3377 handle_child_output(r, &output); 3378 } 3379 3380 #[test] 3381 fn test_vhost_user_net_default() { 3382 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 3383 } 3384 3385 #[test] 3386 fn test_vhost_user_net_named_tap() { 3387 test_vhost_user_net( 3388 Some("mytap0"), 3389 2, 3390 &prepare_vhost_user_net_daemon, 3391 false, 3392 false, 3393 ) 3394 } 3395 3396 #[test] 3397 fn test_vhost_user_net_existing_tap() { 3398 test_vhost_user_net( 3399 Some("vunet-tap0"), 3400 2, 3401 &prepare_vhost_user_net_daemon, 3402 false, 3403 false, 3404 ) 3405 } 3406 3407 #[test] 3408 fn test_vhost_user_net_multiple_queues() { 3409 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 3410 } 3411 3412 #[test] 3413 fn test_vhost_user_net_tap_multiple_queues() { 3414 test_vhost_user_net( 3415 Some("vunet-tap1"), 3416 4, 3417 &prepare_vhost_user_net_daemon, 3418 false, 3419 false, 3420 ) 3421 } 3422 3423 #[test] 3424 fn test_vhost_user_net_host_mac() { 3425 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 3426 } 3427 3428 #[test] 3429 fn test_vhost_user_net_client_mode() { 3430 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 3431 } 3432 3433 #[test] 3434 #[cfg(not(target_arch = "aarch64"))] 3435 fn test_vhost_user_blk_default() { 3436 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 3437 } 3438 3439 #[test] 3440 #[cfg(not(target_arch = "aarch64"))] 3441 fn test_vhost_user_blk_readonly() { 3442 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3443 } 3444 3445 #[test] 3446 #[cfg(not(target_arch = "aarch64"))] 3447 fn test_vhost_user_blk_direct() { 3448 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3449 } 3450 3451 #[test] 3452 fn test_boot_from_vhost_user_blk_default() { 3453 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3454 } 3455 3456 #[test] 3457 #[cfg(target_arch = "x86_64")] 3458 fn test_split_irqchip() { 3459 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3460 let guest = Guest::new(Box::new(focal)); 3461 3462 let mut child = GuestCommand::new(&guest) 3463 .args(["--cpus", "boot=1"]) 3464 .args(["--memory", "size=512M"]) 3465 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3466 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3467 .default_disks() 3468 .default_net() 3469 .capture_output() 3470 .spawn() 3471 .unwrap(); 3472 3473 let r = std::panic::catch_unwind(|| { 3474 guest.wait_vm_boot(None).unwrap(); 3475 3476 assert_eq!( 3477 guest 3478 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3479 .unwrap() 3480 .trim() 3481 .parse::<u32>() 3482 .unwrap_or(1), 3483 0 3484 ); 3485 assert_eq!( 3486 guest 3487 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3488 .unwrap() 3489 .trim() 3490 .parse::<u32>() 3491 .unwrap_or(1), 3492 0 3493 ); 3494 }); 3495 3496 let _ = child.kill(); 3497 let output = child.wait_with_output().unwrap(); 3498 3499 handle_child_output(r, &output); 3500 } 3501 3502 #[test] 3503 #[cfg(target_arch = "x86_64")] 3504 fn test_dmi_serial_number() { 3505 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3506 let guest = Guest::new(Box::new(focal)); 3507 3508 let mut child = GuestCommand::new(&guest) 3509 .args(["--cpus", "boot=1"]) 3510 .args(["--memory", "size=512M"]) 3511 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3512 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3513 .args(["--platform", "serial_number=a=b;c=d"]) 3514 .default_disks() 3515 .default_net() 3516 .capture_output() 3517 .spawn() 3518 .unwrap(); 3519 3520 let r = std::panic::catch_unwind(|| { 3521 guest.wait_vm_boot(None).unwrap(); 3522 3523 assert_eq!( 3524 guest 3525 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3526 .unwrap() 3527 .trim(), 3528 "a=b;c=d" 3529 ); 3530 }); 3531 3532 let _ = child.kill(); 3533 let output = child.wait_with_output().unwrap(); 3534 3535 handle_child_output(r, &output); 3536 } 3537 3538 #[test] 3539 #[cfg(target_arch = "x86_64")] 3540 fn test_dmi_uuid() { 3541 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3542 let guest = Guest::new(Box::new(focal)); 3543 3544 let mut child = GuestCommand::new(&guest) 3545 .args(["--cpus", "boot=1"]) 3546 .args(["--memory", "size=512M"]) 3547 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3548 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3549 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3550 .default_disks() 3551 .default_net() 3552 .capture_output() 3553 .spawn() 3554 .unwrap(); 3555 3556 let r = std::panic::catch_unwind(|| { 3557 guest.wait_vm_boot(None).unwrap(); 3558 3559 assert_eq!( 3560 guest 3561 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3562 .unwrap() 3563 .trim(), 3564 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3565 ); 3566 }); 3567 3568 let _ = child.kill(); 3569 let output = child.wait_with_output().unwrap(); 3570 3571 handle_child_output(r, &output); 3572 } 3573 3574 #[test] 3575 #[cfg(target_arch = "x86_64")] 3576 fn test_dmi_oem_strings() { 3577 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3578 let guest = Guest::new(Box::new(focal)); 3579 3580 let s1 = "io.systemd.credential:xx=yy"; 3581 let s2 = "This is a test string"; 3582 3583 let oem_strings = format!("oem_strings=[{s1},{s2}]"); 3584 3585 let mut child = GuestCommand::new(&guest) 3586 .args(["--cpus", "boot=1"]) 3587 .args(["--memory", "size=512M"]) 3588 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3589 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3590 .args(["--platform", &oem_strings]) 3591 .default_disks() 3592 .default_net() 3593 .capture_output() 3594 .spawn() 3595 .unwrap(); 3596 3597 let r = std::panic::catch_unwind(|| { 3598 guest.wait_vm_boot(None).unwrap(); 3599 3600 assert_eq!( 3601 guest 3602 .ssh_command("sudo dmidecode --oem-string count") 3603 .unwrap() 3604 .trim(), 3605 "2" 3606 ); 3607 3608 assert_eq!( 3609 guest 3610 .ssh_command("sudo dmidecode --oem-string 1") 3611 .unwrap() 3612 .trim(), 3613 s1 3614 ); 3615 3616 assert_eq!( 3617 guest 3618 .ssh_command("sudo dmidecode --oem-string 2") 3619 .unwrap() 3620 .trim(), 3621 s2 3622 ); 3623 }); 3624 3625 let _ = child.kill(); 3626 let output = child.wait_with_output().unwrap(); 3627 3628 handle_child_output(r, &output); 3629 } 3630 3631 #[test] 3632 fn test_virtio_fs() { 3633 _test_virtio_fs(&prepare_virtiofsd, false, None) 3634 } 3635 3636 #[test] 3637 fn test_virtio_fs_hotplug() { 3638 _test_virtio_fs(&prepare_virtiofsd, true, None) 3639 } 3640 3641 #[test] 3642 #[cfg(not(feature = "mshv"))] 3643 fn test_virtio_fs_multi_segment_hotplug() { 3644 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3645 } 3646 3647 #[test] 3648 #[cfg(not(feature = "mshv"))] 3649 fn test_virtio_fs_multi_segment() { 3650 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3651 } 3652 3653 #[test] 3654 fn test_virtio_pmem_persist_writes() { 3655 test_virtio_pmem(false, false) 3656 } 3657 3658 #[test] 3659 fn test_virtio_pmem_discard_writes() { 3660 test_virtio_pmem(true, false) 3661 } 3662 3663 #[test] 3664 fn test_virtio_pmem_with_size() { 3665 test_virtio_pmem(true, true) 3666 } 3667 3668 #[test] 3669 fn test_boot_from_virtio_pmem() { 3670 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3671 let guest = Guest::new(Box::new(focal)); 3672 3673 let kernel_path = direct_kernel_boot_path(); 3674 3675 let mut child = GuestCommand::new(&guest) 3676 .args(["--cpus", "boot=1"]) 3677 .args(["--memory", "size=512M"]) 3678 .args(["--kernel", kernel_path.to_str().unwrap()]) 3679 .args([ 3680 "--disk", 3681 format!( 3682 "path={}", 3683 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3684 ) 3685 .as_str(), 3686 ]) 3687 .default_net() 3688 .args([ 3689 "--pmem", 3690 format!( 3691 "file={},size={}", 3692 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3693 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3694 .unwrap() 3695 .len() 3696 ) 3697 .as_str(), 3698 ]) 3699 .args([ 3700 "--cmdline", 3701 DIRECT_KERNEL_BOOT_CMDLINE 3702 .replace("vda1", "pmem0p1") 3703 .as_str(), 3704 ]) 3705 .capture_output() 3706 .spawn() 3707 .unwrap(); 3708 3709 let r = std::panic::catch_unwind(|| { 3710 guest.wait_vm_boot(None).unwrap(); 3711 3712 // Simple checks to validate the VM booted properly 3713 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3714 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3715 }); 3716 3717 let _ = child.kill(); 3718 let output = child.wait_with_output().unwrap(); 3719 3720 handle_child_output(r, &output); 3721 } 3722 3723 #[test] 3724 fn test_multiple_network_interfaces() { 3725 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3726 let guest = Guest::new(Box::new(focal)); 3727 3728 let kernel_path = direct_kernel_boot_path(); 3729 3730 let mut child = GuestCommand::new(&guest) 3731 .args(["--cpus", "boot=1"]) 3732 .args(["--memory", "size=512M"]) 3733 .args(["--kernel", kernel_path.to_str().unwrap()]) 3734 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3735 .default_disks() 3736 .args([ 3737 "--net", 3738 guest.default_net_string().as_str(), 3739 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3740 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3741 ]) 3742 .capture_output() 3743 .spawn() 3744 .unwrap(); 3745 3746 let r = std::panic::catch_unwind(|| { 3747 guest.wait_vm_boot(None).unwrap(); 3748 3749 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3750 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3751 3752 // 3 network interfaces + default localhost ==> 4 interfaces 3753 assert_eq!( 3754 guest 3755 .ssh_command("ip -o link | wc -l") 3756 .unwrap() 3757 .trim() 3758 .parse::<u32>() 3759 .unwrap_or_default(), 3760 4 3761 ); 3762 }); 3763 3764 let _ = child.kill(); 3765 let output = child.wait_with_output().unwrap(); 3766 3767 handle_child_output(r, &output); 3768 } 3769 3770 #[test] 3771 #[cfg(target_arch = "aarch64")] 3772 fn test_pmu_on() { 3773 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3774 let guest = Guest::new(Box::new(focal)); 3775 let mut child = GuestCommand::new(&guest) 3776 .args(["--cpus", "boot=1"]) 3777 .args(["--memory", "size=512M"]) 3778 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3779 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3780 .default_disks() 3781 .default_net() 3782 .capture_output() 3783 .spawn() 3784 .unwrap(); 3785 3786 let r = std::panic::catch_unwind(|| { 3787 guest.wait_vm_boot(None).unwrap(); 3788 3789 // Test that PMU exists. 3790 assert_eq!( 3791 guest 3792 .ssh_command(GREP_PMU_IRQ_CMD) 3793 .unwrap() 3794 .trim() 3795 .parse::<u32>() 3796 .unwrap_or_default(), 3797 1 3798 ); 3799 }); 3800 3801 let _ = child.kill(); 3802 let output = child.wait_with_output().unwrap(); 3803 3804 handle_child_output(r, &output); 3805 } 3806 3807 #[test] 3808 fn test_serial_off() { 3809 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3810 let guest = Guest::new(Box::new(focal)); 3811 let mut child = GuestCommand::new(&guest) 3812 .args(["--cpus", "boot=1"]) 3813 .args(["--memory", "size=512M"]) 3814 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3815 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3816 .default_disks() 3817 .default_net() 3818 .args(["--serial", "off"]) 3819 .capture_output() 3820 .spawn() 3821 .unwrap(); 3822 3823 let r = std::panic::catch_unwind(|| { 3824 guest.wait_vm_boot(None).unwrap(); 3825 3826 // Test that there is no ttyS0 3827 assert_eq!( 3828 guest 3829 .ssh_command(GREP_SERIAL_IRQ_CMD) 3830 .unwrap() 3831 .trim() 3832 .parse::<u32>() 3833 .unwrap_or(1), 3834 0 3835 ); 3836 }); 3837 3838 let _ = child.kill(); 3839 let output = child.wait_with_output().unwrap(); 3840 3841 handle_child_output(r, &output); 3842 } 3843 3844 #[test] 3845 fn test_serial_null() { 3846 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3847 let guest = Guest::new(Box::new(focal)); 3848 let mut cmd = GuestCommand::new(&guest); 3849 #[cfg(target_arch = "x86_64")] 3850 let console_str: &str = "console=ttyS0"; 3851 #[cfg(target_arch = "aarch64")] 3852 let console_str: &str = "console=ttyAMA0"; 3853 3854 cmd.args(["--cpus", "boot=1"]) 3855 .args(["--memory", "size=512M"]) 3856 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3857 .args([ 3858 "--cmdline", 3859 DIRECT_KERNEL_BOOT_CMDLINE 3860 .replace("console=hvc0 ", console_str) 3861 .as_str(), 3862 ]) 3863 .default_disks() 3864 .default_net() 3865 .args(["--serial", "null"]) 3866 .args(["--console", "off"]) 3867 .capture_output(); 3868 3869 let mut child = cmd.spawn().unwrap(); 3870 3871 let r = std::panic::catch_unwind(|| { 3872 guest.wait_vm_boot(None).unwrap(); 3873 3874 // Test that there is a ttyS0 3875 assert_eq!( 3876 guest 3877 .ssh_command(GREP_SERIAL_IRQ_CMD) 3878 .unwrap() 3879 .trim() 3880 .parse::<u32>() 3881 .unwrap_or_default(), 3882 1 3883 ); 3884 }); 3885 3886 let _ = child.kill(); 3887 let output = child.wait_with_output().unwrap(); 3888 handle_child_output(r, &output); 3889 3890 let r = std::panic::catch_unwind(|| { 3891 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3892 }); 3893 3894 handle_child_output(r, &output); 3895 } 3896 3897 #[test] 3898 fn test_serial_tty() { 3899 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3900 let guest = Guest::new(Box::new(focal)); 3901 3902 let kernel_path = direct_kernel_boot_path(); 3903 3904 #[cfg(target_arch = "x86_64")] 3905 let console_str: &str = "console=ttyS0"; 3906 #[cfg(target_arch = "aarch64")] 3907 let console_str: &str = "console=ttyAMA0"; 3908 3909 let mut child = GuestCommand::new(&guest) 3910 .args(["--cpus", "boot=1"]) 3911 .args(["--memory", "size=512M"]) 3912 .args(["--kernel", kernel_path.to_str().unwrap()]) 3913 .args([ 3914 "--cmdline", 3915 DIRECT_KERNEL_BOOT_CMDLINE 3916 .replace("console=hvc0 ", console_str) 3917 .as_str(), 3918 ]) 3919 .default_disks() 3920 .default_net() 3921 .args(["--serial", "tty"]) 3922 .args(["--console", "off"]) 3923 .capture_output() 3924 .spawn() 3925 .unwrap(); 3926 3927 let r = std::panic::catch_unwind(|| { 3928 guest.wait_vm_boot(None).unwrap(); 3929 3930 // Test that there is a ttyS0 3931 assert_eq!( 3932 guest 3933 .ssh_command(GREP_SERIAL_IRQ_CMD) 3934 .unwrap() 3935 .trim() 3936 .parse::<u32>() 3937 .unwrap_or_default(), 3938 1 3939 ); 3940 }); 3941 3942 // This sleep is needed to wait for the login prompt 3943 thread::sleep(std::time::Duration::new(2, 0)); 3944 3945 let _ = child.kill(); 3946 let output = child.wait_with_output().unwrap(); 3947 handle_child_output(r, &output); 3948 3949 let r = std::panic::catch_unwind(|| { 3950 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3951 }); 3952 3953 handle_child_output(r, &output); 3954 } 3955 3956 #[test] 3957 fn test_serial_file() { 3958 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3959 let guest = Guest::new(Box::new(focal)); 3960 3961 let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output"); 3962 #[cfg(target_arch = "x86_64")] 3963 let console_str: &str = "console=ttyS0"; 3964 #[cfg(target_arch = "aarch64")] 3965 let console_str: &str = "console=ttyAMA0"; 3966 3967 let mut child = GuestCommand::new(&guest) 3968 .args(["--cpus", "boot=1"]) 3969 .args(["--memory", "size=512M"]) 3970 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3971 .args([ 3972 "--cmdline", 3973 DIRECT_KERNEL_BOOT_CMDLINE 3974 .replace("console=hvc0 ", console_str) 3975 .as_str(), 3976 ]) 3977 .default_disks() 3978 .default_net() 3979 .args([ 3980 "--serial", 3981 format!("file={}", serial_path.to_str().unwrap()).as_str(), 3982 ]) 3983 .capture_output() 3984 .spawn() 3985 .unwrap(); 3986 3987 let r = std::panic::catch_unwind(|| { 3988 guest.wait_vm_boot(None).unwrap(); 3989 3990 // Test that there is a ttyS0 3991 assert_eq!( 3992 guest 3993 .ssh_command(GREP_SERIAL_IRQ_CMD) 3994 .unwrap() 3995 .trim() 3996 .parse::<u32>() 3997 .unwrap_or_default(), 3998 1 3999 ); 4000 4001 guest.ssh_command("sudo shutdown -h now").unwrap(); 4002 }); 4003 4004 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4005 let _ = child.kill(); 4006 let output = child.wait_with_output().unwrap(); 4007 handle_child_output(r, &output); 4008 4009 let r = std::panic::catch_unwind(|| { 4010 // Check that the cloud-hypervisor binary actually terminated 4011 assert!(output.status.success()); 4012 4013 // Do this check after shutdown of the VM as an easy way to ensure 4014 // all writes are flushed to disk 4015 let mut f = std::fs::File::open(serial_path).unwrap(); 4016 let mut buf = String::new(); 4017 f.read_to_string(&mut buf).unwrap(); 4018 assert!(buf.contains(CONSOLE_TEST_STRING)); 4019 }); 4020 4021 handle_child_output(r, &output); 4022 } 4023 4024 #[test] 4025 fn test_pty_interaction() { 4026 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4027 let guest = Guest::new(Box::new(focal)); 4028 let api_socket = temp_api_path(&guest.tmp_dir); 4029 let serial_option = if cfg!(target_arch = "x86_64") { 4030 " console=ttyS0" 4031 } else { 4032 " console=ttyAMA0" 4033 }; 4034 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4035 4036 let mut child = GuestCommand::new(&guest) 4037 .args(["--cpus", "boot=1"]) 4038 .args(["--memory", "size=512M"]) 4039 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4040 .args(["--cmdline", &cmdline]) 4041 .default_disks() 4042 .default_net() 4043 .args(["--serial", "null"]) 4044 .args(["--console", "pty"]) 4045 .args(["--api-socket", &api_socket]) 4046 .spawn() 4047 .unwrap(); 4048 4049 let r = std::panic::catch_unwind(|| { 4050 guest.wait_vm_boot(None).unwrap(); 4051 // Get pty fd for console 4052 let console_path = get_pty_path(&api_socket, "console"); 4053 _test_pty_interaction(console_path); 4054 4055 guest.ssh_command("sudo shutdown -h now").unwrap(); 4056 }); 4057 4058 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4059 let _ = child.kill(); 4060 let output = child.wait_with_output().unwrap(); 4061 handle_child_output(r, &output); 4062 4063 let r = std::panic::catch_unwind(|| { 4064 // Check that the cloud-hypervisor binary actually terminated 4065 assert!(output.status.success()) 4066 }); 4067 handle_child_output(r, &output); 4068 } 4069 4070 #[test] 4071 fn test_serial_socket_interaction() { 4072 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4073 let guest = Guest::new(Box::new(focal)); 4074 let serial_socket = guest.tmp_dir.as_path().join("/tmp/serial.socket"); 4075 let serial_socket_pty = guest.tmp_dir.as_path().join("/tmp/serial.pty"); 4076 let serial_option = if cfg!(target_arch = "x86_64") { 4077 " console=ttyS0" 4078 } else { 4079 " console=ttyAMA0" 4080 }; 4081 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4082 4083 let mut child = GuestCommand::new(&guest) 4084 .args(["--cpus", "boot=1"]) 4085 .args(["--memory", "size=512M"]) 4086 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4087 .args(["--cmdline", &cmdline]) 4088 .default_disks() 4089 .default_net() 4090 .args(["--console", "null"]) 4091 .args([ 4092 "--serial", 4093 format!("socket={}", serial_socket.to_str().unwrap()).as_str(), 4094 ]) 4095 .spawn() 4096 .unwrap(); 4097 4098 let _ = std::panic::catch_unwind(|| { 4099 guest.wait_vm_boot(None).unwrap(); 4100 }); 4101 4102 let mut socat_command = Command::new("socat"); 4103 let socat_args = [ 4104 &format!("pty,link={},raw", serial_socket_pty.display()), 4105 &format!("UNIX-CONNECT:{}", serial_socket.display()), 4106 ]; 4107 socat_command.args(socat_args); 4108 4109 let mut socat_child = socat_command.spawn().unwrap(); 4110 thread::sleep(std::time::Duration::new(1, 0)); 4111 4112 let _ = std::panic::catch_unwind(|| { 4113 _test_pty_interaction(serial_socket_pty); 4114 }); 4115 4116 let _ = socat_child.kill(); 4117 4118 let r = std::panic::catch_unwind(|| { 4119 guest.ssh_command("sudo shutdown -h now").unwrap(); 4120 }); 4121 4122 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4123 let _ = child.kill(); 4124 let output = child.wait_with_output().unwrap(); 4125 handle_child_output(r, &output); 4126 4127 let r = std::panic::catch_unwind(|| { 4128 // Check that the cloud-hypervisor binary actually terminated 4129 assert!(output.status.success()) 4130 }); 4131 handle_child_output(r, &output); 4132 } 4133 4134 #[test] 4135 fn test_virtio_console() { 4136 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4137 let guest = Guest::new(Box::new(focal)); 4138 4139 let kernel_path = direct_kernel_boot_path(); 4140 4141 let mut child = GuestCommand::new(&guest) 4142 .args(["--cpus", "boot=1"]) 4143 .args(["--memory", "size=512M"]) 4144 .args(["--kernel", kernel_path.to_str().unwrap()]) 4145 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4146 .default_disks() 4147 .default_net() 4148 .args(["--console", "tty"]) 4149 .args(["--serial", "null"]) 4150 .capture_output() 4151 .spawn() 4152 .unwrap(); 4153 4154 let text = String::from("On a branch floating down river a cricket, singing."); 4155 let cmd = format!("echo {text} | sudo tee /dev/hvc0"); 4156 4157 let r = std::panic::catch_unwind(|| { 4158 guest.wait_vm_boot(None).unwrap(); 4159 4160 assert!(guest 4161 .does_device_vendor_pair_match("0x1043", "0x1af4") 4162 .unwrap_or_default()); 4163 4164 guest.ssh_command(&cmd).unwrap(); 4165 }); 4166 4167 let _ = child.kill(); 4168 let output = child.wait_with_output().unwrap(); 4169 handle_child_output(r, &output); 4170 4171 let r = std::panic::catch_unwind(|| { 4172 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 4173 }); 4174 4175 handle_child_output(r, &output); 4176 } 4177 4178 #[test] 4179 fn test_console_file() { 4180 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4181 let guest = Guest::new(Box::new(focal)); 4182 4183 let console_path = guest.tmp_dir.as_path().join("/tmp/console-output"); 4184 let mut child = GuestCommand::new(&guest) 4185 .args(["--cpus", "boot=1"]) 4186 .args(["--memory", "size=512M"]) 4187 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4188 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4189 .default_disks() 4190 .default_net() 4191 .args([ 4192 "--console", 4193 format!("file={}", console_path.to_str().unwrap()).as_str(), 4194 ]) 4195 .capture_output() 4196 .spawn() 4197 .unwrap(); 4198 4199 guest.wait_vm_boot(None).unwrap(); 4200 4201 guest.ssh_command("sudo shutdown -h now").unwrap(); 4202 4203 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4204 let _ = child.kill(); 4205 let output = child.wait_with_output().unwrap(); 4206 4207 let r = std::panic::catch_unwind(|| { 4208 // Check that the cloud-hypervisor binary actually terminated 4209 assert!(output.status.success()); 4210 4211 // Do this check after shutdown of the VM as an easy way to ensure 4212 // all writes are flushed to disk 4213 let mut f = std::fs::File::open(console_path).unwrap(); 4214 let mut buf = String::new(); 4215 f.read_to_string(&mut buf).unwrap(); 4216 4217 if !buf.contains(CONSOLE_TEST_STRING) { 4218 eprintln!( 4219 "\n\n==== Console file output ====\n\n{buf}\n\n==== End console file output ====" 4220 ); 4221 } 4222 assert!(buf.contains(CONSOLE_TEST_STRING)); 4223 }); 4224 4225 handle_child_output(r, &output); 4226 } 4227 4228 #[test] 4229 #[cfg(target_arch = "x86_64")] 4230 #[cfg(not(feature = "mshv"))] 4231 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 4232 // backed networking interfaces, bound through a simple bridge on the host. 4233 // So if the nested cloud-hypervisor succeeds in getting a directly 4234 // assigned interface from its cloud-hypervisor host, we should be able to 4235 // ssh into it, and verify that it's running with the right kernel command 4236 // line (We tag the command line from cloud-hypervisor for that purpose). 4237 // The third device is added to validate that hotplug works correctly since 4238 // it is being added to the L2 VM through hotplugging mechanism. 4239 // Also, we pass-through a virtio-blk device to the L2 VM to test the 32-bit 4240 // vfio device support 4241 fn test_vfio() { 4242 setup_vfio_network_interfaces(); 4243 4244 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4245 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 4246 4247 let mut workload_path = dirs::home_dir().unwrap(); 4248 workload_path.push("workloads"); 4249 4250 let kernel_path = direct_kernel_boot_path(); 4251 4252 let mut vfio_path = workload_path.clone(); 4253 vfio_path.push("vfio"); 4254 4255 let mut cloud_init_vfio_base_path = vfio_path.clone(); 4256 cloud_init_vfio_base_path.push("cloudinit.img"); 4257 4258 // We copy our cloudinit into the vfio mount point, for the nested 4259 // cloud-hypervisor guest to use. 4260 rate_limited_copy( 4261 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 4262 &cloud_init_vfio_base_path, 4263 ) 4264 .expect("copying of cloud-init disk failed"); 4265 4266 let mut vfio_disk_path = workload_path.clone(); 4267 vfio_disk_path.push("vfio.img"); 4268 4269 // Create the vfio disk image 4270 let output = Command::new("mkfs.ext4") 4271 .arg("-d") 4272 .arg(vfio_path.to_str().unwrap()) 4273 .arg(vfio_disk_path.to_str().unwrap()) 4274 .arg("2g") 4275 .output() 4276 .unwrap(); 4277 if !output.status.success() { 4278 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 4279 panic!("mkfs.ext4 command generated an error"); 4280 } 4281 4282 let mut blk_file_path = workload_path; 4283 blk_file_path.push("blk.img"); 4284 4285 let vfio_tap0 = "vfio-tap0"; 4286 let vfio_tap1 = "vfio-tap1"; 4287 let vfio_tap2 = "vfio-tap2"; 4288 let vfio_tap3 = "vfio-tap3"; 4289 4290 let mut child = GuestCommand::new(&guest) 4291 .args(["--cpus", "boot=4"]) 4292 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 4293 .args(["--kernel", kernel_path.to_str().unwrap()]) 4294 .args([ 4295 "--disk", 4296 format!( 4297 "path={}", 4298 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4299 ) 4300 .as_str(), 4301 format!( 4302 "path={}", 4303 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4304 ) 4305 .as_str(), 4306 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 4307 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 4308 ]) 4309 .args([ 4310 "--cmdline", 4311 format!( 4312 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 4313 ) 4314 .as_str(), 4315 ]) 4316 .args([ 4317 "--net", 4318 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 4319 format!( 4320 "tap={},mac={},iommu=on", 4321 vfio_tap1, guest.network.l2_guest_mac1 4322 ) 4323 .as_str(), 4324 format!( 4325 "tap={},mac={},iommu=on", 4326 vfio_tap2, guest.network.l2_guest_mac2 4327 ) 4328 .as_str(), 4329 format!( 4330 "tap={},mac={},iommu=on", 4331 vfio_tap3, guest.network.l2_guest_mac3 4332 ) 4333 .as_str(), 4334 ]) 4335 .capture_output() 4336 .spawn() 4337 .unwrap(); 4338 4339 thread::sleep(std::time::Duration::new(30, 0)); 4340 4341 let r = std::panic::catch_unwind(|| { 4342 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 4343 thread::sleep(std::time::Duration::new(120, 0)); 4344 4345 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 4346 // added to its kernel command line. 4347 // Let's ssh into it and verify that it's there. If it is it means 4348 // we're in the right guest (The L2 one) because the QEMU L1 guest 4349 // does not have this command line tag. 4350 assert!(check_matched_lines_count( 4351 guest.ssh_command_l2_1("cat /proc/cmdline").unwrap().trim(), 4352 vec!["VFIOTAG"], 4353 1 4354 )); 4355 4356 // Let's also verify from the second virtio-net device passed to 4357 // the L2 VM. 4358 assert!(check_matched_lines_count( 4359 guest.ssh_command_l2_2("cat /proc/cmdline").unwrap().trim(), 4360 vec!["VFIOTAG"], 4361 1 4362 )); 4363 4364 // Check the amount of PCI devices appearing in L2 VM. 4365 assert!(check_lines_count( 4366 guest 4367 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4368 .unwrap() 4369 .trim(), 4370 8 4371 )); 4372 4373 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 4374 assert!(check_matched_lines_count( 4375 guest.ssh_command_l2_1("lsblk").unwrap().trim(), 4376 vec!["vdc", "16M"], 4377 1 4378 )); 4379 4380 // Hotplug an extra virtio-net device through L2 VM. 4381 guest 4382 .ssh_command_l1( 4383 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 4384 ) 4385 .unwrap(); 4386 guest 4387 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 4388 .unwrap(); 4389 let vfio_hotplug_output = guest 4390 .ssh_command_l1( 4391 "sudo /mnt/ch-remote \ 4392 --api-socket=/tmp/ch_api.sock \ 4393 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 4394 ) 4395 .unwrap(); 4396 assert!(check_matched_lines_count( 4397 vfio_hotplug_output.trim(), 4398 vec!["{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}"], 4399 1 4400 )); 4401 4402 thread::sleep(std::time::Duration::new(10, 0)); 4403 4404 // Let's also verify from the third virtio-net device passed to 4405 // the L2 VM. This third device has been hotplugged through the L2 4406 // VM, so this is our way to validate hotplug works for VFIO PCI. 4407 assert!(check_matched_lines_count( 4408 guest.ssh_command_l2_3("cat /proc/cmdline").unwrap().trim(), 4409 vec!["VFIOTAG"], 4410 1 4411 )); 4412 4413 // Check the amount of PCI devices appearing in L2 VM. 4414 // There should be one more device than before, raising the count 4415 // up to 9 PCI devices. 4416 assert!(check_lines_count( 4417 guest 4418 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4419 .unwrap() 4420 .trim(), 4421 9 4422 )); 4423 4424 // Let's now verify that we can correctly remove the virtio-net 4425 // device through the "remove-device" command responsible for 4426 // unplugging VFIO devices. 4427 guest 4428 .ssh_command_l1( 4429 "sudo /mnt/ch-remote \ 4430 --api-socket=/tmp/ch_api.sock \ 4431 remove-device vfio123", 4432 ) 4433 .unwrap(); 4434 thread::sleep(std::time::Duration::new(10, 0)); 4435 4436 // Check the amount of PCI devices appearing in L2 VM is back down 4437 // to 8 devices. 4438 assert!(check_lines_count( 4439 guest 4440 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4441 .unwrap() 4442 .trim(), 4443 8 4444 )); 4445 4446 // Perform memory hotplug in L2 and validate the memory is showing 4447 // up as expected. In order to check, we will use the virtio-net 4448 // device already passed through L2 as a VFIO device, this will 4449 // verify that VFIO devices are functional with memory hotplug. 4450 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 4451 guest 4452 .ssh_command_l2_1( 4453 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 4454 ) 4455 .unwrap(); 4456 guest 4457 .ssh_command_l1( 4458 "sudo /mnt/ch-remote \ 4459 --api-socket=/tmp/ch_api.sock \ 4460 resize --memory=1073741824", 4461 ) 4462 .unwrap(); 4463 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 4464 }); 4465 4466 let _ = child.kill(); 4467 let output = child.wait_with_output().unwrap(); 4468 4469 cleanup_vfio_network_interfaces(); 4470 4471 handle_child_output(r, &output); 4472 } 4473 4474 #[test] 4475 fn test_direct_kernel_boot_noacpi() { 4476 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4477 let guest = Guest::new(Box::new(focal)); 4478 4479 let kernel_path = direct_kernel_boot_path(); 4480 4481 let mut child = GuestCommand::new(&guest) 4482 .args(["--cpus", "boot=1"]) 4483 .args(["--memory", "size=512M"]) 4484 .args(["--kernel", kernel_path.to_str().unwrap()]) 4485 .args([ 4486 "--cmdline", 4487 format!("{DIRECT_KERNEL_BOOT_CMDLINE} acpi=off").as_str(), 4488 ]) 4489 .default_disks() 4490 .default_net() 4491 .capture_output() 4492 .spawn() 4493 .unwrap(); 4494 4495 let r = std::panic::catch_unwind(|| { 4496 guest.wait_vm_boot(None).unwrap(); 4497 4498 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4499 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4500 }); 4501 4502 let _ = child.kill(); 4503 let output = child.wait_with_output().unwrap(); 4504 4505 handle_child_output(r, &output); 4506 } 4507 4508 #[test] 4509 fn test_virtio_vsock() { 4510 _test_virtio_vsock(false) 4511 } 4512 4513 #[test] 4514 fn test_virtio_vsock_hotplug() { 4515 _test_virtio_vsock(true); 4516 } 4517 4518 #[test] 4519 fn test_api_http_shutdown() { 4520 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4521 let guest = Guest::new(Box::new(focal)); 4522 4523 _test_api_shutdown(TargetApi::new_http_api(&guest.tmp_dir), guest) 4524 } 4525 4526 #[test] 4527 fn test_api_http_delete() { 4528 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4529 let guest = Guest::new(Box::new(focal)); 4530 4531 _test_api_delete(TargetApi::new_http_api(&guest.tmp_dir), guest); 4532 } 4533 4534 #[test] 4535 fn test_api_http_pause_resume() { 4536 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4537 let guest = Guest::new(Box::new(focal)); 4538 4539 _test_api_pause_resume(TargetApi::new_http_api(&guest.tmp_dir), guest) 4540 } 4541 4542 #[test] 4543 fn test_api_http_create_boot() { 4544 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4545 let guest = Guest::new(Box::new(focal)); 4546 4547 _test_api_create_boot(TargetApi::new_http_api(&guest.tmp_dir), guest) 4548 } 4549 4550 #[test] 4551 fn test_virtio_iommu() { 4552 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4553 } 4554 4555 #[test] 4556 // We cannot force the software running in the guest to reprogram the BAR 4557 // with some different addresses, but we have a reliable way of testing it 4558 // with a standard Linux kernel. 4559 // By removing a device from the PCI tree, and then rescanning the tree, 4560 // Linux consistently chooses to reorganize the PCI device BARs to other 4561 // locations in the guest address space. 4562 // This test creates a dedicated PCI network device to be checked as being 4563 // properly probed first, then removing it, and adding it again by doing a 4564 // rescan. 4565 fn test_pci_bar_reprogramming() { 4566 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4567 let guest = Guest::new(Box::new(focal)); 4568 4569 #[cfg(target_arch = "x86_64")] 4570 let kernel_path = direct_kernel_boot_path(); 4571 #[cfg(target_arch = "aarch64")] 4572 let kernel_path = edk2_path(); 4573 4574 let mut child = GuestCommand::new(&guest) 4575 .args(["--cpus", "boot=1"]) 4576 .args(["--memory", "size=512M"]) 4577 .args(["--kernel", kernel_path.to_str().unwrap()]) 4578 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4579 .default_disks() 4580 .args([ 4581 "--net", 4582 guest.default_net_string().as_str(), 4583 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4584 ]) 4585 .capture_output() 4586 .spawn() 4587 .unwrap(); 4588 4589 let r = std::panic::catch_unwind(|| { 4590 guest.wait_vm_boot(None).unwrap(); 4591 4592 // 2 network interfaces + default localhost ==> 3 interfaces 4593 assert_eq!( 4594 guest 4595 .ssh_command("ip -o link | wc -l") 4596 .unwrap() 4597 .trim() 4598 .parse::<u32>() 4599 .unwrap_or_default(), 4600 3 4601 ); 4602 4603 let init_bar_addr = guest 4604 .ssh_command( 4605 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4606 ) 4607 .unwrap(); 4608 4609 // Remove the PCI device 4610 guest 4611 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4612 .unwrap(); 4613 4614 // Only 1 network interface left + default localhost ==> 2 interfaces 4615 assert_eq!( 4616 guest 4617 .ssh_command("ip -o link | wc -l") 4618 .unwrap() 4619 .trim() 4620 .parse::<u32>() 4621 .unwrap_or_default(), 4622 2 4623 ); 4624 4625 // Remove the PCI device 4626 guest 4627 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4628 .unwrap(); 4629 4630 // Back to 2 network interface + default localhost ==> 3 interfaces 4631 assert_eq!( 4632 guest 4633 .ssh_command("ip -o link | wc -l") 4634 .unwrap() 4635 .trim() 4636 .parse::<u32>() 4637 .unwrap_or_default(), 4638 3 4639 ); 4640 4641 let new_bar_addr = guest 4642 .ssh_command( 4643 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4644 ) 4645 .unwrap(); 4646 4647 // Let's compare the BAR addresses for our virtio-net device. 4648 // They should be different as we expect the BAR reprogramming 4649 // to have happened. 4650 assert_ne!(init_bar_addr, new_bar_addr); 4651 }); 4652 4653 let _ = child.kill(); 4654 let output = child.wait_with_output().unwrap(); 4655 4656 handle_child_output(r, &output); 4657 } 4658 4659 #[test] 4660 fn test_memory_mergeable_off() { 4661 test_memory_mergeable(false) 4662 } 4663 4664 #[test] 4665 #[cfg(target_arch = "x86_64")] 4666 fn test_cpu_hotplug() { 4667 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4668 let guest = Guest::new(Box::new(focal)); 4669 let api_socket = temp_api_path(&guest.tmp_dir); 4670 4671 let kernel_path = direct_kernel_boot_path(); 4672 4673 let mut child = GuestCommand::new(&guest) 4674 .args(["--cpus", "boot=2,max=4"]) 4675 .args(["--memory", "size=512M"]) 4676 .args(["--kernel", kernel_path.to_str().unwrap()]) 4677 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4678 .default_disks() 4679 .default_net() 4680 .args(["--api-socket", &api_socket]) 4681 .capture_output() 4682 .spawn() 4683 .unwrap(); 4684 4685 let r = std::panic::catch_unwind(|| { 4686 guest.wait_vm_boot(None).unwrap(); 4687 4688 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4689 4690 // Resize the VM 4691 let desired_vcpus = 4; 4692 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4693 4694 guest 4695 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4696 .unwrap(); 4697 guest 4698 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4699 .unwrap(); 4700 thread::sleep(std::time::Duration::new(10, 0)); 4701 assert_eq!( 4702 guest.get_cpu_count().unwrap_or_default(), 4703 u32::from(desired_vcpus) 4704 ); 4705 4706 guest.reboot_linux(0, None); 4707 4708 assert_eq!( 4709 guest.get_cpu_count().unwrap_or_default(), 4710 u32::from(desired_vcpus) 4711 ); 4712 4713 // Resize the VM 4714 let desired_vcpus = 2; 4715 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4716 4717 thread::sleep(std::time::Duration::new(10, 0)); 4718 assert_eq!( 4719 guest.get_cpu_count().unwrap_or_default(), 4720 u32::from(desired_vcpus) 4721 ); 4722 4723 // Resize the VM back up to 4 4724 let desired_vcpus = 4; 4725 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4726 4727 guest 4728 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4729 .unwrap(); 4730 guest 4731 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4732 .unwrap(); 4733 thread::sleep(std::time::Duration::new(10, 0)); 4734 assert_eq!( 4735 guest.get_cpu_count().unwrap_or_default(), 4736 u32::from(desired_vcpus) 4737 ); 4738 }); 4739 4740 let _ = child.kill(); 4741 let output = child.wait_with_output().unwrap(); 4742 4743 handle_child_output(r, &output); 4744 } 4745 4746 #[test] 4747 fn test_memory_hotplug() { 4748 #[cfg(target_arch = "aarch64")] 4749 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4750 #[cfg(target_arch = "x86_64")] 4751 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4752 let focal = UbuntuDiskConfig::new(focal_image); 4753 let guest = Guest::new(Box::new(focal)); 4754 let api_socket = temp_api_path(&guest.tmp_dir); 4755 4756 #[cfg(target_arch = "aarch64")] 4757 let kernel_path = edk2_path(); 4758 #[cfg(target_arch = "x86_64")] 4759 let kernel_path = direct_kernel_boot_path(); 4760 4761 let mut child = GuestCommand::new(&guest) 4762 .args(["--cpus", "boot=2,max=4"]) 4763 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4764 .args(["--kernel", kernel_path.to_str().unwrap()]) 4765 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4766 .default_disks() 4767 .default_net() 4768 .args(["--balloon", "size=0"]) 4769 .args(["--api-socket", &api_socket]) 4770 .capture_output() 4771 .spawn() 4772 .unwrap(); 4773 4774 let r = std::panic::catch_unwind(|| { 4775 guest.wait_vm_boot(None).unwrap(); 4776 4777 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4778 4779 guest.enable_memory_hotplug(); 4780 4781 // Add RAM to the VM 4782 let desired_ram = 1024 << 20; 4783 resize_command(&api_socket, None, Some(desired_ram), None, None); 4784 4785 thread::sleep(std::time::Duration::new(10, 0)); 4786 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4787 4788 // Use balloon to remove RAM from the VM 4789 let desired_balloon = 512 << 20; 4790 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4791 4792 thread::sleep(std::time::Duration::new(10, 0)); 4793 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4794 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4795 4796 guest.reboot_linux(0, None); 4797 4798 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4799 4800 // Use balloon add RAM to the VM 4801 let desired_balloon = 0; 4802 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4803 4804 thread::sleep(std::time::Duration::new(10, 0)); 4805 4806 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4807 4808 guest.enable_memory_hotplug(); 4809 4810 // Add RAM to the VM 4811 let desired_ram = 2048 << 20; 4812 resize_command(&api_socket, None, Some(desired_ram), None, None); 4813 4814 thread::sleep(std::time::Duration::new(10, 0)); 4815 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4816 4817 // Remove RAM to the VM (only applies after reboot) 4818 let desired_ram = 1024 << 20; 4819 resize_command(&api_socket, None, Some(desired_ram), None, None); 4820 4821 guest.reboot_linux(1, None); 4822 4823 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4824 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4825 }); 4826 4827 let _ = child.kill(); 4828 let output = child.wait_with_output().unwrap(); 4829 4830 handle_child_output(r, &output); 4831 } 4832 4833 #[test] 4834 #[cfg(not(feature = "mshv"))] 4835 fn test_virtio_mem() { 4836 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4837 let guest = Guest::new(Box::new(focal)); 4838 let api_socket = temp_api_path(&guest.tmp_dir); 4839 4840 let kernel_path = direct_kernel_boot_path(); 4841 4842 let mut child = GuestCommand::new(&guest) 4843 .args(["--cpus", "boot=2,max=4"]) 4844 .args([ 4845 "--memory", 4846 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4847 ]) 4848 .args(["--kernel", kernel_path.to_str().unwrap()]) 4849 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4850 .default_disks() 4851 .default_net() 4852 .args(["--api-socket", &api_socket]) 4853 .capture_output() 4854 .spawn() 4855 .unwrap(); 4856 4857 let r = std::panic::catch_unwind(|| { 4858 guest.wait_vm_boot(None).unwrap(); 4859 4860 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4861 4862 guest.enable_memory_hotplug(); 4863 4864 // Add RAM to the VM 4865 let desired_ram = 1024 << 20; 4866 resize_command(&api_socket, None, Some(desired_ram), None, None); 4867 4868 thread::sleep(std::time::Duration::new(10, 0)); 4869 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4870 4871 // Add RAM to the VM 4872 let desired_ram = 2048 << 20; 4873 resize_command(&api_socket, None, Some(desired_ram), None, None); 4874 4875 thread::sleep(std::time::Duration::new(10, 0)); 4876 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4877 4878 // Remove RAM from the VM 4879 let desired_ram = 1024 << 20; 4880 resize_command(&api_socket, None, Some(desired_ram), None, None); 4881 4882 thread::sleep(std::time::Duration::new(10, 0)); 4883 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4884 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4885 4886 guest.reboot_linux(0, None); 4887 4888 // Check the amount of memory after reboot is 1GiB 4889 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4890 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4891 4892 // Check we can still resize to 512MiB 4893 let desired_ram = 512 << 20; 4894 resize_command(&api_socket, None, Some(desired_ram), None, None); 4895 thread::sleep(std::time::Duration::new(10, 0)); 4896 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4897 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4898 }); 4899 4900 let _ = child.kill(); 4901 let output = child.wait_with_output().unwrap(); 4902 4903 handle_child_output(r, &output); 4904 } 4905 4906 #[test] 4907 #[cfg(target_arch = "x86_64")] 4908 #[cfg(not(feature = "mshv"))] 4909 // Test both vCPU and memory resizing together 4910 fn test_resize() { 4911 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4912 let guest = Guest::new(Box::new(focal)); 4913 let api_socket = temp_api_path(&guest.tmp_dir); 4914 4915 let kernel_path = direct_kernel_boot_path(); 4916 4917 let mut child = GuestCommand::new(&guest) 4918 .args(["--cpus", "boot=2,max=4"]) 4919 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4920 .args(["--kernel", kernel_path.to_str().unwrap()]) 4921 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4922 .default_disks() 4923 .default_net() 4924 .args(["--api-socket", &api_socket]) 4925 .capture_output() 4926 .spawn() 4927 .unwrap(); 4928 4929 let r = std::panic::catch_unwind(|| { 4930 guest.wait_vm_boot(None).unwrap(); 4931 4932 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4933 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4934 4935 guest.enable_memory_hotplug(); 4936 4937 // Resize the VM 4938 let desired_vcpus = 4; 4939 let desired_ram = 1024 << 20; 4940 resize_command( 4941 &api_socket, 4942 Some(desired_vcpus), 4943 Some(desired_ram), 4944 None, 4945 None, 4946 ); 4947 4948 guest 4949 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4950 .unwrap(); 4951 guest 4952 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4953 .unwrap(); 4954 thread::sleep(std::time::Duration::new(10, 0)); 4955 assert_eq!( 4956 guest.get_cpu_count().unwrap_or_default(), 4957 u32::from(desired_vcpus) 4958 ); 4959 4960 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4961 }); 4962 4963 let _ = child.kill(); 4964 let output = child.wait_with_output().unwrap(); 4965 4966 handle_child_output(r, &output); 4967 } 4968 4969 #[test] 4970 fn test_memory_overhead() { 4971 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4972 let guest = Guest::new(Box::new(focal)); 4973 4974 let kernel_path = direct_kernel_boot_path(); 4975 4976 let guest_memory_size_kb = 512 * 1024; 4977 4978 let mut child = GuestCommand::new(&guest) 4979 .args(["--cpus", "boot=1"]) 4980 .args(["--memory", format!("size={guest_memory_size_kb}K").as_str()]) 4981 .args(["--kernel", kernel_path.to_str().unwrap()]) 4982 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4983 .default_net() 4984 .default_disks() 4985 .capture_output() 4986 .spawn() 4987 .unwrap(); 4988 4989 guest.wait_vm_boot(None).unwrap(); 4990 4991 let r = std::panic::catch_unwind(|| { 4992 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 4993 eprintln!("Guest memory overhead: {overhead} vs {MAXIMUM_VMM_OVERHEAD_KB}"); 4994 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 4995 }); 4996 4997 let _ = child.kill(); 4998 let output = child.wait_with_output().unwrap(); 4999 5000 handle_child_output(r, &output); 5001 } 5002 5003 #[test] 5004 fn test_disk_hotplug() { 5005 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5006 let guest = Guest::new(Box::new(focal)); 5007 5008 #[cfg(target_arch = "x86_64")] 5009 let kernel_path = direct_kernel_boot_path(); 5010 #[cfg(target_arch = "aarch64")] 5011 let kernel_path = edk2_path(); 5012 5013 let api_socket = temp_api_path(&guest.tmp_dir); 5014 5015 let mut child = GuestCommand::new(&guest) 5016 .args(["--api-socket", &api_socket]) 5017 .args(["--cpus", "boot=1"]) 5018 .args(["--memory", "size=512M"]) 5019 .args(["--kernel", kernel_path.to_str().unwrap()]) 5020 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5021 .default_disks() 5022 .default_net() 5023 .capture_output() 5024 .spawn() 5025 .unwrap(); 5026 5027 let r = std::panic::catch_unwind(|| { 5028 guest.wait_vm_boot(None).unwrap(); 5029 5030 // Check /dev/vdc is not there 5031 assert_eq!( 5032 guest 5033 .ssh_command("lsblk | grep -c vdc.*16M || true") 5034 .unwrap() 5035 .trim() 5036 .parse::<u32>() 5037 .unwrap_or(1), 5038 0 5039 ); 5040 5041 // Now let's add the extra disk. 5042 let mut blk_file_path = dirs::home_dir().unwrap(); 5043 blk_file_path.push("workloads"); 5044 blk_file_path.push("blk.img"); 5045 let (cmd_success, cmd_output) = remote_command_w_output( 5046 &api_socket, 5047 "add-disk", 5048 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5049 ); 5050 assert!(cmd_success); 5051 assert!(String::from_utf8_lossy(&cmd_output) 5052 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5053 5054 thread::sleep(std::time::Duration::new(10, 0)); 5055 5056 // Check that /dev/vdc exists and the block size is 16M. 5057 assert_eq!( 5058 guest 5059 .ssh_command("lsblk | grep vdc | grep -c 16M") 5060 .unwrap() 5061 .trim() 5062 .parse::<u32>() 5063 .unwrap_or_default(), 5064 1 5065 ); 5066 // And check the block device can be read. 5067 guest 5068 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5069 .unwrap(); 5070 5071 // Let's remove it the extra disk. 5072 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5073 thread::sleep(std::time::Duration::new(5, 0)); 5074 // And check /dev/vdc is not there 5075 assert_eq!( 5076 guest 5077 .ssh_command("lsblk | grep -c vdc.*16M || true") 5078 .unwrap() 5079 .trim() 5080 .parse::<u32>() 5081 .unwrap_or(1), 5082 0 5083 ); 5084 5085 // And add it back to validate unplug did work correctly. 5086 let (cmd_success, cmd_output) = remote_command_w_output( 5087 &api_socket, 5088 "add-disk", 5089 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5090 ); 5091 assert!(cmd_success); 5092 assert!(String::from_utf8_lossy(&cmd_output) 5093 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5094 5095 thread::sleep(std::time::Duration::new(10, 0)); 5096 5097 // Check that /dev/vdc exists and the block size is 16M. 5098 assert_eq!( 5099 guest 5100 .ssh_command("lsblk | grep vdc | grep -c 16M") 5101 .unwrap() 5102 .trim() 5103 .parse::<u32>() 5104 .unwrap_or_default(), 5105 1 5106 ); 5107 // And check the block device can be read. 5108 guest 5109 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5110 .unwrap(); 5111 5112 // Reboot the VM. 5113 guest.reboot_linux(0, None); 5114 5115 // Check still there after reboot 5116 assert_eq!( 5117 guest 5118 .ssh_command("lsblk | grep vdc | grep -c 16M") 5119 .unwrap() 5120 .trim() 5121 .parse::<u32>() 5122 .unwrap_or_default(), 5123 1 5124 ); 5125 5126 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5127 5128 thread::sleep(std::time::Duration::new(20, 0)); 5129 5130 // Check device has gone away 5131 assert_eq!( 5132 guest 5133 .ssh_command("lsblk | grep -c vdc.*16M || true") 5134 .unwrap() 5135 .trim() 5136 .parse::<u32>() 5137 .unwrap_or(1), 5138 0 5139 ); 5140 5141 guest.reboot_linux(1, None); 5142 5143 // Check device still absent 5144 assert_eq!( 5145 guest 5146 .ssh_command("lsblk | grep -c vdc.*16M || true") 5147 .unwrap() 5148 .trim() 5149 .parse::<u32>() 5150 .unwrap_or(1), 5151 0 5152 ); 5153 }); 5154 5155 let _ = child.kill(); 5156 let output = child.wait_with_output().unwrap(); 5157 5158 handle_child_output(r, &output); 5159 } 5160 5161 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 5162 const LOOP_CONFIGURE: u64 = 0x4c0a; 5163 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 5164 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 5165 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 5166 5167 #[repr(C)] 5168 struct LoopInfo64 { 5169 lo_device: u64, 5170 lo_inode: u64, 5171 lo_rdevice: u64, 5172 lo_offset: u64, 5173 lo_sizelimit: u64, 5174 lo_number: u32, 5175 lo_encrypt_type: u32, 5176 lo_encrypt_key_size: u32, 5177 lo_flags: u32, 5178 lo_file_name: [u8; 64], 5179 lo_crypt_name: [u8; 64], 5180 lo_encrypt_key: [u8; 32], 5181 lo_init: [u64; 2], 5182 } 5183 5184 impl Default for LoopInfo64 { 5185 fn default() -> Self { 5186 LoopInfo64 { 5187 lo_device: 0, 5188 lo_inode: 0, 5189 lo_rdevice: 0, 5190 lo_offset: 0, 5191 lo_sizelimit: 0, 5192 lo_number: 0, 5193 lo_encrypt_type: 0, 5194 lo_encrypt_key_size: 0, 5195 lo_flags: 0, 5196 lo_file_name: [0; 64], 5197 lo_crypt_name: [0; 64], 5198 lo_encrypt_key: [0; 32], 5199 lo_init: [0; 2], 5200 } 5201 } 5202 } 5203 5204 #[derive(Default)] 5205 #[repr(C)] 5206 struct LoopConfig { 5207 fd: u32, 5208 block_size: u32, 5209 info: LoopInfo64, 5210 _reserved: [u64; 8], 5211 } 5212 5213 // Open loop-control device 5214 let loop_ctl_file = OpenOptions::new() 5215 .read(true) 5216 .write(true) 5217 .open(LOOP_CTL_PATH) 5218 .unwrap(); 5219 5220 // Request a free loop device 5221 let loop_device_number = 5222 unsafe { libc::ioctl(loop_ctl_file.as_raw_fd(), LOOP_CTL_GET_FREE as _) }; 5223 5224 if loop_device_number < 0 { 5225 panic!("Couldn't find a free loop device"); 5226 } 5227 5228 // Create loop device path 5229 let loop_device_path = format!("{LOOP_DEVICE_PREFIX}{loop_device_number}"); 5230 5231 // Open loop device 5232 let loop_device_file = OpenOptions::new() 5233 .read(true) 5234 .write(true) 5235 .open(&loop_device_path) 5236 .unwrap(); 5237 5238 // Open backing file 5239 let backing_file = OpenOptions::new() 5240 .read(true) 5241 .write(true) 5242 .open(backing_file_path) 5243 .unwrap(); 5244 5245 let loop_config = LoopConfig { 5246 fd: backing_file.as_raw_fd() as u32, 5247 block_size, 5248 ..Default::default() 5249 }; 5250 5251 for i in 0..num_retries { 5252 let ret = unsafe { 5253 libc::ioctl( 5254 loop_device_file.as_raw_fd(), 5255 LOOP_CONFIGURE as _, 5256 &loop_config, 5257 ) 5258 }; 5259 if ret != 0 { 5260 if i < num_retries - 1 { 5261 println!( 5262 "Iteration {}: Failed to configure the loop device {}: {}", 5263 i, 5264 loop_device_path, 5265 std::io::Error::last_os_error() 5266 ); 5267 } else { 5268 panic!( 5269 "Failed {} times trying to configure the loop device {}: {}", 5270 num_retries, 5271 loop_device_path, 5272 std::io::Error::last_os_error() 5273 ); 5274 } 5275 } else { 5276 break; 5277 } 5278 5279 // Wait for a bit before retrying 5280 thread::sleep(std::time::Duration::new(5, 0)); 5281 } 5282 5283 loop_device_path 5284 } 5285 5286 #[test] 5287 fn test_virtio_block_topology() { 5288 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5289 let guest = Guest::new(Box::new(focal)); 5290 5291 let kernel_path = direct_kernel_boot_path(); 5292 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5293 5294 let output = exec_host_command_output( 5295 format!( 5296 "qemu-img create -f raw {} 16M", 5297 test_disk_path.to_str().unwrap() 5298 ) 5299 .as_str(), 5300 ); 5301 if !output.status.success() { 5302 let stdout = String::from_utf8_lossy(&output.stdout); 5303 let stderr = String::from_utf8_lossy(&output.stderr); 5304 panic!("qemu-img command failed\nstdout\n{stdout}\nstderr\n{stderr}"); 5305 } 5306 5307 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5308 5309 let mut child = GuestCommand::new(&guest) 5310 .args(["--cpus", "boot=1"]) 5311 .args(["--memory", "size=512M"]) 5312 .args(["--kernel", kernel_path.to_str().unwrap()]) 5313 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5314 .args([ 5315 "--disk", 5316 format!( 5317 "path={}", 5318 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5319 ) 5320 .as_str(), 5321 format!( 5322 "path={}", 5323 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5324 ) 5325 .as_str(), 5326 format!("path={}", &loop_dev).as_str(), 5327 ]) 5328 .default_net() 5329 .capture_output() 5330 .spawn() 5331 .unwrap(); 5332 5333 let r = std::panic::catch_unwind(|| { 5334 guest.wait_vm_boot(None).unwrap(); 5335 5336 // MIN-IO column 5337 assert_eq!( 5338 guest 5339 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5340 .unwrap() 5341 .trim() 5342 .parse::<u32>() 5343 .unwrap_or_default(), 5344 4096 5345 ); 5346 // PHY-SEC column 5347 assert_eq!( 5348 guest 5349 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5350 .unwrap() 5351 .trim() 5352 .parse::<u32>() 5353 .unwrap_or_default(), 5354 4096 5355 ); 5356 // LOG-SEC column 5357 assert_eq!( 5358 guest 5359 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5360 .unwrap() 5361 .trim() 5362 .parse::<u32>() 5363 .unwrap_or_default(), 5364 4096 5365 ); 5366 }); 5367 5368 let _ = child.kill(); 5369 let output = child.wait_with_output().unwrap(); 5370 5371 handle_child_output(r, &output); 5372 5373 Command::new("losetup") 5374 .args(["-d", &loop_dev]) 5375 .output() 5376 .expect("loop device not found"); 5377 } 5378 5379 #[test] 5380 fn test_virtio_balloon_deflate_on_oom() { 5381 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5382 let guest = Guest::new(Box::new(focal)); 5383 5384 let kernel_path = direct_kernel_boot_path(); 5385 5386 let api_socket = temp_api_path(&guest.tmp_dir); 5387 5388 //Let's start a 4G guest with balloon occupied 2G memory 5389 let mut child = GuestCommand::new(&guest) 5390 .args(["--api-socket", &api_socket]) 5391 .args(["--cpus", "boot=1"]) 5392 .args(["--memory", "size=4G"]) 5393 .args(["--kernel", kernel_path.to_str().unwrap()]) 5394 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5395 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5396 .default_disks() 5397 .default_net() 5398 .capture_output() 5399 .spawn() 5400 .unwrap(); 5401 5402 let r = std::panic::catch_unwind(|| { 5403 guest.wait_vm_boot(None).unwrap(); 5404 5405 // Wait for balloon memory's initialization and check its size. 5406 // The virtio-balloon driver might take a few seconds to report the 5407 // balloon effective size back to the VMM. 5408 thread::sleep(std::time::Duration::new(20, 0)); 5409 5410 let orig_balloon = balloon_size(&api_socket); 5411 println!("The original balloon memory size is {orig_balloon} bytes"); 5412 assert!(orig_balloon == 2147483648); 5413 5414 // Two steps to verify if the 'deflate_on_oom' parameter works. 5415 // 1st: run a command to trigger an OOM in the guest. 5416 guest 5417 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5418 .unwrap(); 5419 5420 // Give some time for the OOM to happen in the guest and be reported 5421 // back to the host. 5422 thread::sleep(std::time::Duration::new(20, 0)); 5423 5424 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5425 let deflated_balloon = balloon_size(&api_socket); 5426 println!("After deflating, balloon memory size is {deflated_balloon} bytes"); 5427 // Verify the balloon size deflated 5428 assert!(deflated_balloon < 2147483648); 5429 }); 5430 5431 let _ = child.kill(); 5432 let output = child.wait_with_output().unwrap(); 5433 5434 handle_child_output(r, &output); 5435 } 5436 5437 #[test] 5438 #[cfg(not(feature = "mshv"))] 5439 fn test_virtio_balloon_free_page_reporting() { 5440 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5441 let guest = Guest::new(Box::new(focal)); 5442 5443 //Let's start a 4G guest with balloon occupied 2G memory 5444 let mut child = GuestCommand::new(&guest) 5445 .args(["--cpus", "boot=1"]) 5446 .args(["--memory", "size=4G"]) 5447 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5448 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5449 .args(["--balloon", "size=0,free_page_reporting=on"]) 5450 .default_disks() 5451 .default_net() 5452 .capture_output() 5453 .spawn() 5454 .unwrap(); 5455 5456 let pid = child.id(); 5457 let r = std::panic::catch_unwind(|| { 5458 guest.wait_vm_boot(None).unwrap(); 5459 5460 // Check the initial RSS is less than 1GiB 5461 let rss = process_rss_kib(pid); 5462 println!("RSS {rss} < 1048576"); 5463 assert!(rss < 1048576); 5464 5465 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5466 // seconds 5467 let guest_ip = guest.network.guest_ip.clone(); 5468 thread::spawn(move || { 5469 ssh_command_ip( 5470 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5471 &guest_ip, 5472 DEFAULT_SSH_RETRIES, 5473 DEFAULT_SSH_TIMEOUT, 5474 ) 5475 .unwrap(); 5476 }); 5477 5478 // Wait for 50 seconds to make sure the stress command is consuming 5479 // the expected amount of memory. 5480 thread::sleep(std::time::Duration::new(50, 0)); 5481 let rss = process_rss_kib(pid); 5482 println!("RSS {rss} >= 2097152"); 5483 assert!(rss >= 2097152); 5484 5485 // Wait for an extra minute to make sure the stress command has 5486 // completed and that the guest reported the free pages to the VMM 5487 // through the virtio-balloon device. We expect the RSS to be under 5488 // 2GiB. 5489 thread::sleep(std::time::Duration::new(60, 0)); 5490 let rss = process_rss_kib(pid); 5491 println!("RSS {rss} < 2097152"); 5492 assert!(rss < 2097152); 5493 }); 5494 5495 let _ = child.kill(); 5496 let output = child.wait_with_output().unwrap(); 5497 5498 handle_child_output(r, &output); 5499 } 5500 5501 #[test] 5502 fn test_pmem_hotplug() { 5503 _test_pmem_hotplug(None) 5504 } 5505 5506 #[test] 5507 fn test_pmem_multi_segment_hotplug() { 5508 _test_pmem_hotplug(Some(15)) 5509 } 5510 5511 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5512 #[cfg(target_arch = "aarch64")] 5513 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5514 #[cfg(target_arch = "x86_64")] 5515 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5516 let focal = UbuntuDiskConfig::new(focal_image); 5517 let guest = Guest::new(Box::new(focal)); 5518 5519 #[cfg(target_arch = "x86_64")] 5520 let kernel_path = direct_kernel_boot_path(); 5521 #[cfg(target_arch = "aarch64")] 5522 let kernel_path = edk2_path(); 5523 5524 let api_socket = temp_api_path(&guest.tmp_dir); 5525 5526 let mut cmd = GuestCommand::new(&guest); 5527 5528 cmd.args(["--api-socket", &api_socket]) 5529 .args(["--cpus", "boot=1"]) 5530 .args(["--memory", "size=512M"]) 5531 .args(["--kernel", kernel_path.to_str().unwrap()]) 5532 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5533 .default_disks() 5534 .default_net() 5535 .capture_output(); 5536 5537 if pci_segment.is_some() { 5538 cmd.args([ 5539 "--platform", 5540 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5541 ]); 5542 } 5543 5544 let mut child = cmd.spawn().unwrap(); 5545 5546 let r = std::panic::catch_unwind(|| { 5547 guest.wait_vm_boot(None).unwrap(); 5548 5549 // Check /dev/pmem0 is not there 5550 assert_eq!( 5551 guest 5552 .ssh_command("lsblk | grep -c pmem0 || true") 5553 .unwrap() 5554 .trim() 5555 .parse::<u32>() 5556 .unwrap_or(1), 5557 0 5558 ); 5559 5560 let pmem_temp_file = TempFile::new().unwrap(); 5561 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5562 let (cmd_success, cmd_output) = remote_command_w_output( 5563 &api_socket, 5564 "add-pmem", 5565 Some(&format!( 5566 "file={},id=test0{}", 5567 pmem_temp_file.as_path().to_str().unwrap(), 5568 if let Some(pci_segment) = pci_segment { 5569 format!(",pci_segment={pci_segment}") 5570 } else { 5571 "".to_owned() 5572 } 5573 )), 5574 ); 5575 assert!(cmd_success); 5576 if let Some(pci_segment) = pci_segment { 5577 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5578 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5579 ))); 5580 } else { 5581 assert!(String::from_utf8_lossy(&cmd_output) 5582 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5583 } 5584 5585 // Check that /dev/pmem0 exists and the block size is 128M 5586 assert_eq!( 5587 guest 5588 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5589 .unwrap() 5590 .trim() 5591 .parse::<u32>() 5592 .unwrap_or_default(), 5593 1 5594 ); 5595 5596 guest.reboot_linux(0, None); 5597 5598 // Check still there after reboot 5599 assert_eq!( 5600 guest 5601 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5602 .unwrap() 5603 .trim() 5604 .parse::<u32>() 5605 .unwrap_or_default(), 5606 1 5607 ); 5608 5609 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5610 5611 thread::sleep(std::time::Duration::new(20, 0)); 5612 5613 // Check device has gone away 5614 assert_eq!( 5615 guest 5616 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5617 .unwrap() 5618 .trim() 5619 .parse::<u32>() 5620 .unwrap_or(1), 5621 0 5622 ); 5623 5624 guest.reboot_linux(1, None); 5625 5626 // Check still absent after reboot 5627 assert_eq!( 5628 guest 5629 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5630 .unwrap() 5631 .trim() 5632 .parse::<u32>() 5633 .unwrap_or(1), 5634 0 5635 ); 5636 }); 5637 5638 let _ = child.kill(); 5639 let output = child.wait_with_output().unwrap(); 5640 5641 handle_child_output(r, &output); 5642 } 5643 5644 #[test] 5645 fn test_net_hotplug() { 5646 _test_net_hotplug(None) 5647 } 5648 5649 #[test] 5650 fn test_net_multi_segment_hotplug() { 5651 _test_net_hotplug(Some(15)) 5652 } 5653 5654 fn _test_net_hotplug(pci_segment: Option<u16>) { 5655 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5656 let guest = Guest::new(Box::new(focal)); 5657 5658 #[cfg(target_arch = "x86_64")] 5659 let kernel_path = direct_kernel_boot_path(); 5660 #[cfg(target_arch = "aarch64")] 5661 let kernel_path = edk2_path(); 5662 5663 let api_socket = temp_api_path(&guest.tmp_dir); 5664 5665 // Boot without network 5666 let mut cmd = GuestCommand::new(&guest); 5667 5668 cmd.args(["--api-socket", &api_socket]) 5669 .args(["--cpus", "boot=1"]) 5670 .args(["--memory", "size=512M"]) 5671 .args(["--kernel", kernel_path.to_str().unwrap()]) 5672 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5673 .default_disks() 5674 .capture_output(); 5675 5676 if pci_segment.is_some() { 5677 cmd.args([ 5678 "--platform", 5679 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5680 ]); 5681 } 5682 5683 let mut child = cmd.spawn().unwrap(); 5684 5685 thread::sleep(std::time::Duration::new(20, 0)); 5686 5687 let r = std::panic::catch_unwind(|| { 5688 // Add network 5689 let (cmd_success, cmd_output) = remote_command_w_output( 5690 &api_socket, 5691 "add-net", 5692 Some( 5693 format!( 5694 "{}{},id=test0", 5695 guest.default_net_string(), 5696 if let Some(pci_segment) = pci_segment { 5697 format!(",pci_segment={pci_segment}") 5698 } else { 5699 "".to_owned() 5700 } 5701 ) 5702 .as_str(), 5703 ), 5704 ); 5705 assert!(cmd_success); 5706 5707 if let Some(pci_segment) = pci_segment { 5708 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5709 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5710 ))); 5711 } else { 5712 assert!(String::from_utf8_lossy(&cmd_output) 5713 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5714 } 5715 5716 thread::sleep(std::time::Duration::new(5, 0)); 5717 5718 // 1 network interfaces + default localhost ==> 2 interfaces 5719 assert_eq!( 5720 guest 5721 .ssh_command("ip -o link | wc -l") 5722 .unwrap() 5723 .trim() 5724 .parse::<u32>() 5725 .unwrap_or_default(), 5726 2 5727 ); 5728 5729 // Remove network 5730 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5731 thread::sleep(std::time::Duration::new(5, 0)); 5732 5733 let (cmd_success, cmd_output) = remote_command_w_output( 5734 &api_socket, 5735 "add-net", 5736 Some( 5737 format!( 5738 "{}{},id=test1", 5739 guest.default_net_string(), 5740 if let Some(pci_segment) = pci_segment { 5741 format!(",pci_segment={pci_segment}") 5742 } else { 5743 "".to_owned() 5744 } 5745 ) 5746 .as_str(), 5747 ), 5748 ); 5749 assert!(cmd_success); 5750 5751 if let Some(pci_segment) = pci_segment { 5752 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5753 "{{\"id\":\"test1\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5754 ))); 5755 } else { 5756 assert!(String::from_utf8_lossy(&cmd_output) 5757 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5758 } 5759 5760 thread::sleep(std::time::Duration::new(5, 0)); 5761 5762 // 1 network interfaces + default localhost ==> 2 interfaces 5763 assert_eq!( 5764 guest 5765 .ssh_command("ip -o link | wc -l") 5766 .unwrap() 5767 .trim() 5768 .parse::<u32>() 5769 .unwrap_or_default(), 5770 2 5771 ); 5772 5773 guest.reboot_linux(0, None); 5774 5775 // Check still there after reboot 5776 // 1 network interfaces + default localhost ==> 2 interfaces 5777 assert_eq!( 5778 guest 5779 .ssh_command("ip -o link | wc -l") 5780 .unwrap() 5781 .trim() 5782 .parse::<u32>() 5783 .unwrap_or_default(), 5784 2 5785 ); 5786 }); 5787 5788 let _ = child.kill(); 5789 let output = child.wait_with_output().unwrap(); 5790 5791 handle_child_output(r, &output); 5792 } 5793 5794 #[test] 5795 fn test_initramfs() { 5796 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5797 let guest = Guest::new(Box::new(focal)); 5798 let mut workload_path = dirs::home_dir().unwrap(); 5799 workload_path.push("workloads"); 5800 5801 #[cfg(target_arch = "x86_64")] 5802 let mut kernels = vec![direct_kernel_boot_path()]; 5803 #[cfg(target_arch = "aarch64")] 5804 let kernels = vec![direct_kernel_boot_path()]; 5805 5806 #[cfg(target_arch = "x86_64")] 5807 { 5808 let mut pvh_kernel_path = workload_path.clone(); 5809 pvh_kernel_path.push("vmlinux"); 5810 kernels.push(pvh_kernel_path); 5811 } 5812 5813 let mut initramfs_path = workload_path; 5814 initramfs_path.push("alpine_initramfs.img"); 5815 5816 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5817 let cmdline = format!("console=hvc0 quiet TEST_STRING={test_string}"); 5818 5819 kernels.iter().for_each(|k_path| { 5820 let mut child = GuestCommand::new(&guest) 5821 .args(["--kernel", k_path.to_str().unwrap()]) 5822 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 5823 .args(["--cmdline", &cmdline]) 5824 .capture_output() 5825 .spawn() 5826 .unwrap(); 5827 5828 thread::sleep(std::time::Duration::new(20, 0)); 5829 5830 let _ = child.kill(); 5831 let output = child.wait_with_output().unwrap(); 5832 5833 let r = std::panic::catch_unwind(|| { 5834 let s = String::from_utf8_lossy(&output.stdout); 5835 5836 assert_ne!(s.lines().position(|line| line == test_string), None); 5837 }); 5838 5839 handle_child_output(r, &output); 5840 }); 5841 } 5842 5843 // One thing to note about this test. The virtio-net device is heavily used 5844 // through each ssh command. There's no need to perform a dedicated test to 5845 // verify the migration went well for virtio-net. 5846 #[test] 5847 #[cfg(not(feature = "mshv"))] 5848 fn test_snapshot_restore_hotplug_virtiomem() { 5849 _test_snapshot_restore(true); 5850 } 5851 5852 #[test] 5853 fn test_snapshot_restore_basic() { 5854 _test_snapshot_restore(false); 5855 } 5856 5857 fn _test_snapshot_restore(use_hotplug: bool) { 5858 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5859 let guest = Guest::new(Box::new(focal)); 5860 let kernel_path = direct_kernel_boot_path(); 5861 5862 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 5863 5864 let net_id = "net123"; 5865 let net_params = format!( 5866 "id={},tap=,mac={},ip={},mask=255.255.255.0", 5867 net_id, guest.network.guest_mac, guest.network.host_ip 5868 ); 5869 let mut mem_params = "size=4G"; 5870 5871 if use_hotplug { 5872 mem_params = "size=4G,hotplug_method=virtio-mem,hotplug_size=32G" 5873 } 5874 5875 let cloudinit_params = format!( 5876 "path={},iommu=on", 5877 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5878 ); 5879 5880 let socket = temp_vsock_path(&guest.tmp_dir); 5881 let event_path = temp_event_monitor_path(&guest.tmp_dir); 5882 5883 let mut child = GuestCommand::new(&guest) 5884 .args(["--api-socket", &api_socket_source]) 5885 .args(["--event-monitor", format!("path={event_path}").as_str()]) 5886 .args(["--cpus", "boot=4"]) 5887 .args(["--memory", mem_params]) 5888 .args(["--balloon", "size=0"]) 5889 .args(["--kernel", kernel_path.to_str().unwrap()]) 5890 .args([ 5891 "--disk", 5892 format!( 5893 "path={}", 5894 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5895 ) 5896 .as_str(), 5897 cloudinit_params.as_str(), 5898 ]) 5899 .args(["--net", net_params.as_str()]) 5900 .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) 5901 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5902 .capture_output() 5903 .spawn() 5904 .unwrap(); 5905 5906 let console_text = String::from("On a branch floating down river a cricket, singing."); 5907 // Create the snapshot directory 5908 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 5909 5910 let r = std::panic::catch_unwind(|| { 5911 guest.wait_vm_boot(None).unwrap(); 5912 5913 // Check the number of vCPUs 5914 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5915 // Check the guest RAM 5916 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5917 if use_hotplug { 5918 // Increase guest RAM with virtio-mem 5919 resize_command( 5920 &api_socket_source, 5921 None, 5922 Some(6 << 30), 5923 None, 5924 Some(&event_path), 5925 ); 5926 thread::sleep(std::time::Duration::new(5, 0)); 5927 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5928 // Use balloon to remove RAM from the VM 5929 resize_command( 5930 &api_socket_source, 5931 None, 5932 None, 5933 Some(1 << 30), 5934 Some(&event_path), 5935 ); 5936 thread::sleep(std::time::Duration::new(5, 0)); 5937 let total_memory = guest.get_total_memory().unwrap_or_default(); 5938 assert!(total_memory > 4_800_000); 5939 assert!(total_memory < 5_760_000); 5940 } 5941 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 5942 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5943 5944 // x86_64: We check that removing and adding back the virtio-net device 5945 // does not break the snapshot/restore support for virtio-pci. 5946 // This is an important thing to test as the hotplug will 5947 // trigger a PCI BAR reprogramming, which is a good way of 5948 // checking if the stored resources are correctly restored. 5949 // Unplug the virtio-net device 5950 // AArch64: Device hotplug is currently not supported, skipping here. 5951 #[cfg(target_arch = "x86_64")] 5952 { 5953 assert!(remote_command( 5954 &api_socket_source, 5955 "remove-device", 5956 Some(net_id), 5957 )); 5958 thread::sleep(std::time::Duration::new(10, 0)); 5959 let latest_events = [&MetaEvent { 5960 event: "device-removed".to_string(), 5961 device_id: Some(net_id.to_string()), 5962 }]; 5963 assert!(check_latest_events_exact(&latest_events, &event_path)); 5964 5965 // Plug the virtio-net device again 5966 assert!(remote_command( 5967 &api_socket_source, 5968 "add-net", 5969 Some(net_params.as_str()), 5970 )); 5971 thread::sleep(std::time::Duration::new(10, 0)); 5972 } 5973 5974 // Pause the VM 5975 assert!(remote_command(&api_socket_source, "pause", None)); 5976 let latest_events = [ 5977 &MetaEvent { 5978 event: "pausing".to_string(), 5979 device_id: None, 5980 }, 5981 &MetaEvent { 5982 event: "paused".to_string(), 5983 device_id: None, 5984 }, 5985 ]; 5986 assert!(check_latest_events_exact(&latest_events, &event_path)); 5987 5988 // Take a snapshot from the VM 5989 assert!(remote_command( 5990 &api_socket_source, 5991 "snapshot", 5992 Some(format!("file://{snapshot_dir}").as_str()), 5993 )); 5994 5995 // Wait to make sure the snapshot is completed 5996 thread::sleep(std::time::Duration::new(10, 0)); 5997 5998 let latest_events = [ 5999 &MetaEvent { 6000 event: "snapshotting".to_string(), 6001 device_id: None, 6002 }, 6003 &MetaEvent { 6004 event: "snapshotted".to_string(), 6005 device_id: None, 6006 }, 6007 ]; 6008 assert!(check_latest_events_exact(&latest_events, &event_path)); 6009 }); 6010 6011 // Shutdown the source VM and check console output 6012 let _ = child.kill(); 6013 let output = child.wait_with_output().unwrap(); 6014 handle_child_output(r, &output); 6015 6016 let r = std::panic::catch_unwind(|| { 6017 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 6018 }); 6019 6020 handle_child_output(r, &output); 6021 6022 // Remove the vsock socket file. 6023 Command::new("rm") 6024 .arg("-f") 6025 .arg(socket.as_str()) 6026 .output() 6027 .unwrap(); 6028 6029 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 6030 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 6031 6032 // Restore the VM from the snapshot 6033 let mut child = GuestCommand::new(&guest) 6034 .args(["--api-socket", &api_socket_restored]) 6035 .args([ 6036 "--event-monitor", 6037 format!("path={event_path_restored}").as_str(), 6038 ]) 6039 .args([ 6040 "--restore", 6041 format!("source_url=file://{snapshot_dir}").as_str(), 6042 ]) 6043 .capture_output() 6044 .spawn() 6045 .unwrap(); 6046 6047 // Wait for the VM to be restored 6048 thread::sleep(std::time::Duration::new(10, 0)); 6049 let expected_events = [ 6050 &MetaEvent { 6051 event: "starting".to_string(), 6052 device_id: None, 6053 }, 6054 &MetaEvent { 6055 event: "activated".to_string(), 6056 device_id: Some("__console".to_string()), 6057 }, 6058 &MetaEvent { 6059 event: "activated".to_string(), 6060 device_id: Some("__rng".to_string()), 6061 }, 6062 &MetaEvent { 6063 event: "restoring".to_string(), 6064 device_id: None, 6065 }, 6066 ]; 6067 assert!(check_sequential_events( 6068 &expected_events, 6069 &event_path_restored 6070 )); 6071 let latest_events = [&MetaEvent { 6072 event: "restored".to_string(), 6073 device_id: None, 6074 }]; 6075 assert!(check_latest_events_exact( 6076 &latest_events, 6077 &event_path_restored 6078 )); 6079 6080 let r = std::panic::catch_unwind(|| { 6081 // Resume the VM 6082 assert!(remote_command(&api_socket_restored, "resume", None)); 6083 let latest_events = [ 6084 &MetaEvent { 6085 event: "resuming".to_string(), 6086 device_id: None, 6087 }, 6088 &MetaEvent { 6089 event: "resumed".to_string(), 6090 device_id: None, 6091 }, 6092 ]; 6093 assert!(check_latest_events_exact( 6094 &latest_events, 6095 &event_path_restored 6096 )); 6097 6098 // Perform same checks to validate VM has been properly restored 6099 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 6100 let total_memory = guest.get_total_memory().unwrap_or_default(); 6101 if !use_hotplug { 6102 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6103 } else { 6104 assert!(total_memory > 4_800_000); 6105 assert!(total_memory < 5_760_000); 6106 // Deflate balloon to restore entire RAM to the VM 6107 resize_command(&api_socket_restored, None, None, Some(0), None); 6108 thread::sleep(std::time::Duration::new(5, 0)); 6109 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6110 // Decrease guest RAM with virtio-mem 6111 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 6112 thread::sleep(std::time::Duration::new(5, 0)); 6113 let total_memory = guest.get_total_memory().unwrap_or_default(); 6114 assert!(total_memory > 4_800_000); 6115 assert!(total_memory < 5_760_000); 6116 } 6117 6118 guest.check_devices_common(Some(&socket), Some(&console_text), None); 6119 }); 6120 // Shutdown the target VM and check console output 6121 let _ = child.kill(); 6122 let output = child.wait_with_output().unwrap(); 6123 handle_child_output(r, &output); 6124 6125 let r = std::panic::catch_unwind(|| { 6126 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 6127 }); 6128 6129 handle_child_output(r, &output); 6130 } 6131 6132 #[test] 6133 fn test_counters() { 6134 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6135 let guest = Guest::new(Box::new(focal)); 6136 let api_socket = temp_api_path(&guest.tmp_dir); 6137 6138 let mut cmd = GuestCommand::new(&guest); 6139 cmd.args(["--cpus", "boot=1"]) 6140 .args(["--memory", "size=512M"]) 6141 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 6142 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6143 .default_disks() 6144 .args(["--net", guest.default_net_string().as_str()]) 6145 .args(["--api-socket", &api_socket]) 6146 .capture_output(); 6147 6148 let mut child = cmd.spawn().unwrap(); 6149 6150 let r = std::panic::catch_unwind(|| { 6151 guest.wait_vm_boot(None).unwrap(); 6152 6153 let orig_counters = get_counters(&api_socket); 6154 guest 6155 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 6156 .unwrap(); 6157 6158 let new_counters = get_counters(&api_socket); 6159 6160 // Check that all the counters have increased 6161 assert!(new_counters > orig_counters); 6162 }); 6163 6164 let _ = child.kill(); 6165 let output = child.wait_with_output().unwrap(); 6166 6167 handle_child_output(r, &output); 6168 } 6169 6170 #[test] 6171 #[cfg(feature = "guest_debug")] 6172 fn test_coredump() { 6173 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6174 let guest = Guest::new(Box::new(focal)); 6175 let api_socket = temp_api_path(&guest.tmp_dir); 6176 6177 let mut cmd = GuestCommand::new(&guest); 6178 cmd.args(["--cpus", "boot=4"]) 6179 .args(["--memory", "size=4G"]) 6180 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6181 .default_disks() 6182 .args(["--net", guest.default_net_string().as_str()]) 6183 .args(["--api-socket", &api_socket]) 6184 .capture_output(); 6185 6186 let mut child = cmd.spawn().unwrap(); 6187 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6188 6189 let r = std::panic::catch_unwind(|| { 6190 guest.wait_vm_boot(None).unwrap(); 6191 6192 assert!(remote_command(&api_socket, "pause", None)); 6193 6194 assert!(remote_command( 6195 &api_socket, 6196 "coredump", 6197 Some(format!("file://{vmcore_file}").as_str()), 6198 )); 6199 6200 // the num of CORE notes should equals to vcpu 6201 let readelf_core_num_cmd = 6202 format!("readelf --all {vmcore_file} |grep CORE |grep -v Type |wc -l"); 6203 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 6204 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 6205 6206 // the num of QEMU notes should equals to vcpu 6207 let readelf_vmm_num_cmd = format!("readelf --all {vmcore_file} |grep QEMU |wc -l"); 6208 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 6209 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 6210 }); 6211 6212 let _ = child.kill(); 6213 let output = child.wait_with_output().unwrap(); 6214 6215 handle_child_output(r, &output); 6216 } 6217 6218 #[test] 6219 #[cfg(feature = "guest_debug")] 6220 fn test_coredump_no_pause() { 6221 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6222 let guest = Guest::new(Box::new(focal)); 6223 let api_socket = temp_api_path(&guest.tmp_dir); 6224 6225 let mut cmd = GuestCommand::new(&guest); 6226 cmd.args(["--cpus", "boot=4"]) 6227 .args(["--memory", "size=4G"]) 6228 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6229 .default_disks() 6230 .args(["--net", guest.default_net_string().as_str()]) 6231 .args(["--api-socket", &api_socket]) 6232 .capture_output(); 6233 6234 let mut child = cmd.spawn().unwrap(); 6235 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6236 6237 let r = std::panic::catch_unwind(|| { 6238 guest.wait_vm_boot(None).unwrap(); 6239 6240 assert!(remote_command( 6241 &api_socket, 6242 "coredump", 6243 Some(format!("file://{vmcore_file}").as_str()), 6244 )); 6245 6246 assert_eq!(vm_state(&api_socket), "Running"); 6247 }); 6248 6249 let _ = child.kill(); 6250 let output = child.wait_with_output().unwrap(); 6251 6252 handle_child_output(r, &output); 6253 } 6254 6255 #[test] 6256 fn test_watchdog() { 6257 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6258 let guest = Guest::new(Box::new(focal)); 6259 let api_socket = temp_api_path(&guest.tmp_dir); 6260 6261 let kernel_path = direct_kernel_boot_path(); 6262 6263 let mut cmd = GuestCommand::new(&guest); 6264 cmd.args(["--cpus", "boot=1"]) 6265 .args(["--memory", "size=512M"]) 6266 .args(["--kernel", kernel_path.to_str().unwrap()]) 6267 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6268 .default_disks() 6269 .args(["--net", guest.default_net_string().as_str()]) 6270 .args(["--watchdog"]) 6271 .args(["--api-socket", &api_socket]) 6272 .capture_output(); 6273 6274 let mut child = cmd.spawn().unwrap(); 6275 6276 let r = std::panic::catch_unwind(|| { 6277 guest.wait_vm_boot(None).unwrap(); 6278 6279 let mut expected_reboot_count = 1; 6280 6281 // Enable the watchdog with a 15s timeout 6282 enable_guest_watchdog(&guest, 15); 6283 6284 // Reboot and check that systemd has activated the watchdog 6285 guest.ssh_command("sudo reboot").unwrap(); 6286 guest.wait_vm_boot(None).unwrap(); 6287 expected_reboot_count += 1; 6288 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6289 assert_eq!( 6290 guest 6291 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6292 .unwrap() 6293 .trim() 6294 .parse::<u32>() 6295 .unwrap_or_default(), 6296 2 6297 ); 6298 6299 // Allow some normal time to elapse to check we don't get spurious reboots 6300 thread::sleep(std::time::Duration::new(40, 0)); 6301 // Check no reboot 6302 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6303 6304 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6305 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6306 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6307 guest.wait_vm_boot(Some(50)).unwrap(); 6308 // Check a reboot is triggered by the watchdog 6309 expected_reboot_count += 1; 6310 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6311 6312 #[cfg(target_arch = "x86_64")] 6313 { 6314 // Now pause the VM and remain offline for 30s 6315 assert!(remote_command(&api_socket, "pause", None)); 6316 thread::sleep(std::time::Duration::new(30, 0)); 6317 assert!(remote_command(&api_socket, "resume", None)); 6318 6319 // Check no reboot 6320 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6321 } 6322 }); 6323 6324 let _ = child.kill(); 6325 let output = child.wait_with_output().unwrap(); 6326 6327 handle_child_output(r, &output); 6328 } 6329 6330 #[test] 6331 fn test_pvpanic() { 6332 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 6333 let guest = Guest::new(Box::new(jammy)); 6334 let api_socket = temp_api_path(&guest.tmp_dir); 6335 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6336 6337 let kernel_path = direct_kernel_boot_path(); 6338 6339 let mut cmd = GuestCommand::new(&guest); 6340 cmd.args(["--cpus", "boot=1"]) 6341 .args(["--memory", "size=512M"]) 6342 .args(["--kernel", kernel_path.to_str().unwrap()]) 6343 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6344 .default_disks() 6345 .args(["--net", guest.default_net_string().as_str()]) 6346 .args(["--pvpanic"]) 6347 .args(["--api-socket", &api_socket]) 6348 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6349 .capture_output(); 6350 6351 let mut child = cmd.spawn().unwrap(); 6352 6353 let r = std::panic::catch_unwind(|| { 6354 guest.wait_vm_boot(None).unwrap(); 6355 6356 // Trigger guest a panic 6357 make_guest_panic(&guest); 6358 6359 // Wait a while for guest 6360 thread::sleep(std::time::Duration::new(10, 0)); 6361 6362 let expected_sequential_events = [&MetaEvent { 6363 event: "panic".to_string(), 6364 device_id: None, 6365 }]; 6366 assert!(check_latest_events_exact( 6367 &expected_sequential_events, 6368 &event_path 6369 )); 6370 }); 6371 6372 let _ = child.kill(); 6373 let output = child.wait_with_output().unwrap(); 6374 6375 handle_child_output(r, &output); 6376 } 6377 6378 #[test] 6379 fn test_tap_from_fd() { 6380 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6381 let guest = Guest::new(Box::new(focal)); 6382 let kernel_path = direct_kernel_boot_path(); 6383 6384 // Create a TAP interface with multi-queue enabled 6385 let num_queue_pairs: usize = 2; 6386 6387 use std::str::FromStr; 6388 let taps = net_util::open_tap( 6389 Some("chtap0"), 6390 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6391 None, 6392 &mut None, 6393 None, 6394 num_queue_pairs, 6395 Some(libc::O_RDWR | libc::O_NONBLOCK), 6396 ) 6397 .unwrap(); 6398 6399 let mut child = GuestCommand::new(&guest) 6400 .args(["--cpus", &format!("boot={num_queue_pairs}")]) 6401 .args(["--memory", "size=512M"]) 6402 .args(["--kernel", kernel_path.to_str().unwrap()]) 6403 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6404 .default_disks() 6405 .args([ 6406 "--net", 6407 &format!( 6408 "fd=[{},{}],mac={},num_queues={}", 6409 taps[0].as_raw_fd(), 6410 taps[1].as_raw_fd(), 6411 guest.network.guest_mac, 6412 num_queue_pairs * 2 6413 ), 6414 ]) 6415 .capture_output() 6416 .spawn() 6417 .unwrap(); 6418 6419 let r = std::panic::catch_unwind(|| { 6420 guest.wait_vm_boot(None).unwrap(); 6421 6422 assert_eq!( 6423 guest 6424 .ssh_command("ip -o link | wc -l") 6425 .unwrap() 6426 .trim() 6427 .parse::<u32>() 6428 .unwrap_or_default(), 6429 2 6430 ); 6431 6432 guest.reboot_linux(0, None); 6433 6434 assert_eq!( 6435 guest 6436 .ssh_command("ip -o link | wc -l") 6437 .unwrap() 6438 .trim() 6439 .parse::<u32>() 6440 .unwrap_or_default(), 6441 2 6442 ); 6443 }); 6444 6445 let _ = child.kill(); 6446 let output = child.wait_with_output().unwrap(); 6447 6448 handle_child_output(r, &output); 6449 } 6450 6451 // By design, a guest VM won't be able to connect to the host 6452 // machine when using a macvtap network interface (while it can 6453 // communicate externally). As a workaround, this integration 6454 // test creates two macvtap interfaces in 'bridge' mode on the 6455 // same physical net interface, one for the guest and one for 6456 // the host. With additional setup on the IP address and the 6457 // routing table, it enables the communications between the 6458 // guest VM and the host machine. 6459 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6460 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6461 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6462 let guest = Guest::new(Box::new(focal)); 6463 let api_socket = temp_api_path(&guest.tmp_dir); 6464 6465 #[cfg(target_arch = "x86_64")] 6466 let kernel_path = direct_kernel_boot_path(); 6467 #[cfg(target_arch = "aarch64")] 6468 let kernel_path = edk2_path(); 6469 6470 let phy_net = "eth0"; 6471 6472 // Create a macvtap interface for the guest VM to use 6473 assert!(exec_host_command_status(&format!( 6474 "sudo ip link add link {phy_net} name {guest_macvtap_name} type macvtap mod bridge" 6475 )) 6476 .success()); 6477 assert!(exec_host_command_status(&format!( 6478 "sudo ip link set {} address {} up", 6479 guest_macvtap_name, guest.network.guest_mac 6480 )) 6481 .success()); 6482 assert!( 6483 exec_host_command_status(&format!("sudo ip link show {guest_macvtap_name}")).success() 6484 ); 6485 6486 let tap_index = 6487 fs::read_to_string(format!("/sys/class/net/{guest_macvtap_name}/ifindex")).unwrap(); 6488 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6489 6490 assert!(exec_host_command_status(&format!("sudo chown $UID.$UID {tap_device}")).success()); 6491 6492 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6493 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6494 assert!(tap_fd1 > 0); 6495 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6496 assert!(tap_fd2 > 0); 6497 6498 // Create a macvtap on the same physical net interface for 6499 // the host machine to use 6500 assert!(exec_host_command_status(&format!( 6501 "sudo ip link add link {phy_net} name {host_macvtap_name} type macvtap mod bridge" 6502 )) 6503 .success()); 6504 // Use default mask "255.255.255.0" 6505 assert!(exec_host_command_status(&format!( 6506 "sudo ip address add {}/24 dev {}", 6507 guest.network.host_ip, host_macvtap_name 6508 )) 6509 .success()); 6510 assert!( 6511 exec_host_command_status(&format!("sudo ip link set dev {host_macvtap_name} up")) 6512 .success() 6513 ); 6514 6515 let mut guest_command = GuestCommand::new(&guest); 6516 guest_command 6517 .args(["--cpus", "boot=2"]) 6518 .args(["--memory", "size=512M"]) 6519 .args(["--kernel", kernel_path.to_str().unwrap()]) 6520 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6521 .default_disks() 6522 .args(["--api-socket", &api_socket]); 6523 6524 let net_params = format!( 6525 "fd=[{},{}],mac={},num_queues=4", 6526 tap_fd1, tap_fd2, guest.network.guest_mac 6527 ); 6528 6529 if !hotplug { 6530 guest_command.args(["--net", &net_params]); 6531 } 6532 6533 let mut child = guest_command.capture_output().spawn().unwrap(); 6534 6535 if hotplug { 6536 // Give some time to the VMM process to listen to the API 6537 // socket. This is the only requirement to avoid the following 6538 // call to ch-remote from failing. 6539 thread::sleep(std::time::Duration::new(10, 0)); 6540 // Hotplug the virtio-net device 6541 let (cmd_success, cmd_output) = 6542 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6543 assert!(cmd_success); 6544 #[cfg(target_arch = "x86_64")] 6545 assert!(String::from_utf8_lossy(&cmd_output) 6546 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6547 #[cfg(target_arch = "aarch64")] 6548 assert!(String::from_utf8_lossy(&cmd_output) 6549 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6550 } 6551 6552 // The functional connectivity provided by the virtio-net device 6553 // gets tested through wait_vm_boot() as it expects to receive a 6554 // HTTP request, and through the SSH command as well. 6555 let r = std::panic::catch_unwind(|| { 6556 guest.wait_vm_boot(None).unwrap(); 6557 6558 assert_eq!( 6559 guest 6560 .ssh_command("ip -o link | wc -l") 6561 .unwrap() 6562 .trim() 6563 .parse::<u32>() 6564 .unwrap_or_default(), 6565 2 6566 ); 6567 6568 guest.reboot_linux(0, None); 6569 6570 assert_eq!( 6571 guest 6572 .ssh_command("ip -o link | wc -l") 6573 .unwrap() 6574 .trim() 6575 .parse::<u32>() 6576 .unwrap_or_default(), 6577 2 6578 ); 6579 }); 6580 6581 let _ = child.kill(); 6582 6583 exec_host_command_status(&format!("sudo ip link del {guest_macvtap_name}")); 6584 exec_host_command_status(&format!("sudo ip link del {host_macvtap_name}")); 6585 6586 let output = child.wait_with_output().unwrap(); 6587 6588 handle_child_output(r, &output); 6589 } 6590 6591 #[test] 6592 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6593 fn test_macvtap() { 6594 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6595 } 6596 6597 #[test] 6598 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6599 fn test_macvtap_hotplug() { 6600 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6601 } 6602 6603 #[test] 6604 #[cfg(not(feature = "mshv"))] 6605 fn test_ovs_dpdk() { 6606 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6607 let guest1 = Guest::new(Box::new(focal1)); 6608 6609 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6610 let guest2 = Guest::new(Box::new(focal2)); 6611 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6612 6613 let (mut child1, mut child2) = 6614 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6615 6616 // Create the snapshot directory 6617 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6618 6619 let r = std::panic::catch_unwind(|| { 6620 // Remove one of the two ports from the OVS bridge 6621 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6622 6623 // Spawn a new netcat listener in the first VM 6624 let guest_ip = guest1.network.guest_ip.clone(); 6625 thread::spawn(move || { 6626 ssh_command_ip( 6627 "nc -l 12345", 6628 &guest_ip, 6629 DEFAULT_SSH_RETRIES, 6630 DEFAULT_SSH_TIMEOUT, 6631 ) 6632 .unwrap(); 6633 }); 6634 6635 // Wait for the server to be listening 6636 thread::sleep(std::time::Duration::new(5, 0)); 6637 6638 // Check the connection fails this time 6639 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6640 6641 // Add the OVS port back 6642 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6643 6644 // And finally check the connection is functional again 6645 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6646 6647 // Pause the VM 6648 assert!(remote_command(&api_socket_source, "pause", None)); 6649 6650 // Take a snapshot from the VM 6651 assert!(remote_command( 6652 &api_socket_source, 6653 "snapshot", 6654 Some(format!("file://{snapshot_dir}").as_str()), 6655 )); 6656 6657 // Wait to make sure the snapshot is completed 6658 thread::sleep(std::time::Duration::new(10, 0)); 6659 }); 6660 6661 // Shutdown the source VM 6662 let _ = child2.kill(); 6663 let output = child2.wait_with_output().unwrap(); 6664 handle_child_output(r, &output); 6665 6666 // Remove the vhost-user socket file. 6667 Command::new("rm") 6668 .arg("-f") 6669 .arg("/tmp/dpdkvhostclient2") 6670 .output() 6671 .unwrap(); 6672 6673 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6674 // Restore the VM from the snapshot 6675 let mut child2 = GuestCommand::new(&guest2) 6676 .args(["--api-socket", &api_socket_restored]) 6677 .args([ 6678 "--restore", 6679 format!("source_url=file://{snapshot_dir}").as_str(), 6680 ]) 6681 .capture_output() 6682 .spawn() 6683 .unwrap(); 6684 6685 // Wait for the VM to be restored 6686 thread::sleep(std::time::Duration::new(10, 0)); 6687 6688 let r = std::panic::catch_unwind(|| { 6689 // Resume the VM 6690 assert!(remote_command(&api_socket_restored, "resume", None)); 6691 6692 // Spawn a new netcat listener in the first VM 6693 let guest_ip = guest1.network.guest_ip.clone(); 6694 thread::spawn(move || { 6695 ssh_command_ip( 6696 "nc -l 12345", 6697 &guest_ip, 6698 DEFAULT_SSH_RETRIES, 6699 DEFAULT_SSH_TIMEOUT, 6700 ) 6701 .unwrap(); 6702 }); 6703 6704 // Wait for the server to be listening 6705 thread::sleep(std::time::Duration::new(5, 0)); 6706 6707 // And check the connection is still functional after restore 6708 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6709 }); 6710 6711 let _ = child1.kill(); 6712 let _ = child2.kill(); 6713 6714 let output = child1.wait_with_output().unwrap(); 6715 child2.wait().unwrap(); 6716 6717 cleanup_ovs_dpdk(); 6718 6719 handle_child_output(r, &output); 6720 } 6721 6722 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6723 cleanup_spdk_nvme(); 6724 6725 assert!(exec_host_command_status(&format!( 6726 "mkdir -p {}", 6727 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6728 )) 6729 .success()); 6730 assert!(exec_host_command_status(&format!( 6731 "truncate {} -s 128M", 6732 nvme_dir.join("test-disk.raw").to_str().unwrap() 6733 )) 6734 .success()); 6735 assert!(exec_host_command_status(&format!( 6736 "mkfs.ext4 {}", 6737 nvme_dir.join("test-disk.raw").to_str().unwrap() 6738 )) 6739 .success()); 6740 6741 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6742 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6743 .args(["-i", "0", "-m", "0x1"]) 6744 .spawn() 6745 .unwrap(); 6746 thread::sleep(std::time::Duration::new(2, 0)); 6747 6748 assert!(exec_host_command_status( 6749 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER" 6750 ) 6751 .success()); 6752 assert!(exec_host_command_status(&format!( 6753 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6754 nvme_dir.join("test-disk.raw").to_str().unwrap() 6755 )) 6756 .success()); 6757 assert!(exec_host_command_status( 6758 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6759 ) 6760 .success()); 6761 assert!(exec_host_command_status( 6762 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6763 ) 6764 .success()); 6765 assert!(exec_host_command_status(&format!( 6766 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6767 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6768 )) 6769 .success()); 6770 } 6771 6772 fn cleanup_spdk_nvme() { 6773 exec_host_command_status("pkill -f nvmf_tgt"); 6774 } 6775 6776 #[test] 6777 fn test_vfio_user() { 6778 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6779 let jammy = UbuntuDiskConfig::new(jammy_image); 6780 let guest = Guest::new(Box::new(jammy)); 6781 6782 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6783 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6784 6785 let api_socket = temp_api_path(&guest.tmp_dir); 6786 let mut child = GuestCommand::new(&guest) 6787 .args(["--api-socket", &api_socket]) 6788 .args(["--cpus", "boot=1"]) 6789 .args(["--memory", "size=512M,shared=on,hugepages=on"]) 6790 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6791 .args(["--serial", "tty", "--console", "off"]) 6792 .default_disks() 6793 .default_net() 6794 .capture_output() 6795 .spawn() 6796 .unwrap(); 6797 6798 let r = std::panic::catch_unwind(|| { 6799 guest.wait_vm_boot(None).unwrap(); 6800 6801 // Hotplug the SPDK-NVMe device to the VM 6802 let (cmd_success, cmd_output) = remote_command_w_output( 6803 &api_socket, 6804 "add-user-device", 6805 Some(&format!( 6806 "socket={},id=vfio_user0", 6807 spdk_nvme_dir 6808 .as_path() 6809 .join("nvme-vfio-user/cntrl") 6810 .to_str() 6811 .unwrap(), 6812 )), 6813 ); 6814 assert!(cmd_success); 6815 assert!(String::from_utf8_lossy(&cmd_output) 6816 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6817 6818 thread::sleep(std::time::Duration::new(10, 0)); 6819 6820 // Check both if /dev/nvme exists and if the block size is 128M. 6821 assert_eq!( 6822 guest 6823 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6824 .unwrap() 6825 .trim() 6826 .parse::<u32>() 6827 .unwrap_or_default(), 6828 1 6829 ); 6830 6831 // Check changes persist after reboot 6832 assert_eq!( 6833 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6834 "" 6835 ); 6836 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6837 guest 6838 .ssh_command("echo test123 | sudo tee /mnt/test") 6839 .unwrap(); 6840 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6841 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6842 6843 guest.reboot_linux(0, None); 6844 assert_eq!( 6845 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6846 "" 6847 ); 6848 assert_eq!( 6849 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6850 "test123" 6851 ); 6852 }); 6853 6854 cleanup_spdk_nvme(); 6855 6856 let _ = child.kill(); 6857 let output = child.wait_with_output().unwrap(); 6858 6859 handle_child_output(r, &output); 6860 } 6861 6862 #[test] 6863 #[cfg(target_arch = "x86_64")] 6864 fn test_vdpa_block() { 6865 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6866 if !exec_host_command_status("lsmod | grep vdpa_sim_blk").success() { 6867 return; 6868 } 6869 6870 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6871 let guest = Guest::new(Box::new(focal)); 6872 let api_socket = temp_api_path(&guest.tmp_dir); 6873 6874 let kernel_path = direct_kernel_boot_path(); 6875 6876 let mut child = GuestCommand::new(&guest) 6877 .args(["--cpus", "boot=2"]) 6878 .args(["--memory", "size=512M,hugepages=on"]) 6879 .args(["--kernel", kernel_path.to_str().unwrap()]) 6880 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6881 .default_disks() 6882 .default_net() 6883 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6884 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 6885 .args(["--api-socket", &api_socket]) 6886 .capture_output() 6887 .spawn() 6888 .unwrap(); 6889 6890 let r = std::panic::catch_unwind(|| { 6891 guest.wait_vm_boot(None).unwrap(); 6892 6893 // Check both if /dev/vdc exists and if the block size is 128M. 6894 assert_eq!( 6895 guest 6896 .ssh_command("lsblk | grep vdc | grep -c 128M") 6897 .unwrap() 6898 .trim() 6899 .parse::<u32>() 6900 .unwrap_or_default(), 6901 1 6902 ); 6903 6904 // Check the content of the block device after we wrote to it. 6905 // The vpda-sim-blk should let us read what we previously wrote. 6906 guest 6907 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6908 .unwrap(); 6909 assert_eq!( 6910 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6911 "foobar" 6912 ); 6913 6914 // Hotplug an extra vDPA block device behind the vIOMMU 6915 // Add a new vDPA device to the VM 6916 let (cmd_success, cmd_output) = remote_command_w_output( 6917 &api_socket, 6918 "add-vdpa", 6919 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6920 ); 6921 assert!(cmd_success); 6922 assert!(String::from_utf8_lossy(&cmd_output) 6923 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6924 6925 thread::sleep(std::time::Duration::new(10, 0)); 6926 6927 // Check IOMMU setup 6928 assert!(guest 6929 .does_device_vendor_pair_match("0x1057", "0x1af4") 6930 .unwrap_or_default()); 6931 assert_eq!( 6932 guest 6933 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6934 .unwrap() 6935 .trim(), 6936 "0001:00:01.0" 6937 ); 6938 6939 // Check both if /dev/vdd exists and if the block size is 128M. 6940 assert_eq!( 6941 guest 6942 .ssh_command("lsblk | grep vdd | grep -c 128M") 6943 .unwrap() 6944 .trim() 6945 .parse::<u32>() 6946 .unwrap_or_default(), 6947 1 6948 ); 6949 6950 // Write some content to the block device we've just plugged. 6951 guest 6952 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6953 .unwrap(); 6954 6955 // Check we can read the content back. 6956 assert_eq!( 6957 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6958 "foobar" 6959 ); 6960 6961 // Unplug the device 6962 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6963 assert!(cmd_success); 6964 thread::sleep(std::time::Duration::new(10, 0)); 6965 6966 // Check /dev/vdd doesn't exist anymore 6967 assert_eq!( 6968 guest 6969 .ssh_command("lsblk | grep -c vdd || true") 6970 .unwrap() 6971 .trim() 6972 .parse::<u32>() 6973 .unwrap_or(1), 6974 0 6975 ); 6976 }); 6977 6978 let _ = child.kill(); 6979 let output = child.wait_with_output().unwrap(); 6980 6981 handle_child_output(r, &output); 6982 } 6983 6984 #[test] 6985 #[cfg(target_arch = "x86_64")] 6986 #[ignore = "See #5756"] 6987 fn test_vdpa_net() { 6988 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6989 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6990 return; 6991 } 6992 6993 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6994 let guest = Guest::new(Box::new(focal)); 6995 6996 let kernel_path = direct_kernel_boot_path(); 6997 6998 let mut child = GuestCommand::new(&guest) 6999 .args(["--cpus", "boot=2"]) 7000 .args(["--memory", "size=512M,hugepages=on"]) 7001 .args(["--kernel", kernel_path.to_str().unwrap()]) 7002 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7003 .default_disks() 7004 .default_net() 7005 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 7006 .capture_output() 7007 .spawn() 7008 .unwrap(); 7009 7010 let r = std::panic::catch_unwind(|| { 7011 guest.wait_vm_boot(None).unwrap(); 7012 7013 // Check we can find network interface related to vDPA device 7014 assert_eq!( 7015 guest 7016 .ssh_command("ip -o link | grep -c ens6") 7017 .unwrap() 7018 .trim() 7019 .parse::<u32>() 7020 .unwrap_or(0), 7021 1 7022 ); 7023 7024 guest 7025 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 7026 .unwrap(); 7027 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 7028 7029 // Check there is no packet yet on both TX/RX of the network interface 7030 assert_eq!( 7031 guest 7032 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 7033 .unwrap() 7034 .trim() 7035 .parse::<u32>() 7036 .unwrap_or(0), 7037 2 7038 ); 7039 7040 // Send 6 packets with ping command 7041 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 7042 7043 // Check we can find 6 packets on both TX/RX of the network interface 7044 assert_eq!( 7045 guest 7046 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 7047 .unwrap() 7048 .trim() 7049 .parse::<u32>() 7050 .unwrap_or(0), 7051 2 7052 ); 7053 7054 // No need to check for hotplug as we already tested it through 7055 // test_vdpa_block() 7056 }); 7057 7058 let _ = child.kill(); 7059 let output = child.wait_with_output().unwrap(); 7060 7061 handle_child_output(r, &output); 7062 } 7063 7064 #[test] 7065 #[cfg(target_arch = "x86_64")] 7066 fn test_tpm() { 7067 let focal = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7068 let guest = Guest::new(Box::new(focal)); 7069 7070 let (mut swtpm_command, swtpm_socket_path) = prepare_swtpm_daemon(&guest.tmp_dir); 7071 7072 let mut guest_cmd = GuestCommand::new(&guest); 7073 guest_cmd 7074 .args(["--cpus", "boot=1"]) 7075 .args(["--memory", "size=512M"]) 7076 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7077 .args(["--tpm", &format!("socket={swtpm_socket_path}")]) 7078 .capture_output() 7079 .default_disks() 7080 .default_net(); 7081 7082 // Start swtpm daemon 7083 let mut swtpm_child = swtpm_command.spawn().unwrap(); 7084 thread::sleep(std::time::Duration::new(10, 0)); 7085 let mut child = guest_cmd.spawn().unwrap(); 7086 let r = std::panic::catch_unwind(|| { 7087 guest.wait_vm_boot(None).unwrap(); 7088 assert_eq!( 7089 guest.ssh_command("ls /dev/tpm0").unwrap().trim(), 7090 "/dev/tpm0" 7091 ); 7092 guest.ssh_command("sudo tpm2_selftest -f").unwrap(); 7093 guest 7094 .ssh_command("echo 'hello' > /tmp/checksum_test; ") 7095 .unwrap(); 7096 guest.ssh_command("cmp <(sudo tpm2_pcrevent /tmp/checksum_test | grep sha256 | awk '{print $2}') <(sha256sum /tmp/checksum_test| awk '{print $1}')").unwrap(); 7097 }); 7098 7099 let _ = swtpm_child.kill(); 7100 let _d_out = swtpm_child.wait_with_output().unwrap(); 7101 7102 let _ = child.kill(); 7103 let output = child.wait_with_output().unwrap(); 7104 7105 handle_child_output(r, &output); 7106 } 7107 7108 #[test] 7109 #[cfg(target_arch = "x86_64")] 7110 fn test_double_tty() { 7111 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7112 let guest = Guest::new(Box::new(focal)); 7113 let mut cmd = GuestCommand::new(&guest); 7114 let api_socket = temp_api_path(&guest.tmp_dir); 7115 let tty_str: &str = "console=hvc0 earlyprintk=ttyS0 "; 7116 // linux printk module enable console log. 7117 let con_dis_str: &str = "console [hvc0] enabled"; 7118 // linux printk module disable console log. 7119 let con_enb_str: &str = "bootconsole [earlyser0] disabled"; 7120 7121 let kernel_path = direct_kernel_boot_path(); 7122 7123 cmd.args(["--cpus", "boot=1"]) 7124 .args(["--memory", "size=512M"]) 7125 .args(["--kernel", kernel_path.to_str().unwrap()]) 7126 .args([ 7127 "--cmdline", 7128 DIRECT_KERNEL_BOOT_CMDLINE 7129 .replace("console=hvc0 ", tty_str) 7130 .as_str(), 7131 ]) 7132 .capture_output() 7133 .default_disks() 7134 .default_net() 7135 .args(["--serial", "tty"]) 7136 .args(["--console", "tty"]) 7137 .args(["--api-socket", &api_socket]); 7138 7139 let mut child = cmd.spawn().unwrap(); 7140 7141 let mut r = std::panic::catch_unwind(|| { 7142 guest.wait_vm_boot(None).unwrap(); 7143 }); 7144 7145 let _ = child.kill(); 7146 let output = child.wait_with_output().unwrap(); 7147 7148 if r.is_ok() { 7149 r = std::panic::catch_unwind(|| { 7150 let s = String::from_utf8_lossy(&output.stdout); 7151 assert!(s.contains(tty_str)); 7152 assert!(s.contains(con_dis_str)); 7153 assert!(s.contains(con_enb_str)); 7154 }); 7155 } 7156 7157 handle_child_output(r, &output); 7158 } 7159 } 7160 7161 mod dbus_api { 7162 use crate::*; 7163 7164 // Start cloud-hypervisor with no VM parameters, running both the HTTP 7165 // and DBus APIs. Alternate calls to the external APIs (HTTP and DBus) 7166 // to create a VM, boot it, and verify that it can be shut down and then 7167 // booted again. 7168 #[test] 7169 fn test_api_dbus_and_http_interleaved() { 7170 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7171 let guest = Guest::new(Box::new(focal)); 7172 let dbus_api = TargetApi::new_dbus_api(&guest.tmp_dir); 7173 let http_api = TargetApi::new_http_api(&guest.tmp_dir); 7174 7175 let mut child = GuestCommand::new(&guest) 7176 .args(dbus_api.guest_args()) 7177 .args(http_api.guest_args()) 7178 .capture_output() 7179 .spawn() 7180 .unwrap(); 7181 7182 thread::sleep(std::time::Duration::new(1, 0)); 7183 7184 // Verify API servers are running 7185 assert!(dbus_api.remote_command("ping", None)); 7186 assert!(http_api.remote_command("ping", None)); 7187 7188 // Create the VM first 7189 let cpu_count: u8 = 4; 7190 let request_body = guest.api_create_body( 7191 cpu_count, 7192 direct_kernel_boot_path().to_str().unwrap(), 7193 DIRECT_KERNEL_BOOT_CMDLINE, 7194 ); 7195 7196 let temp_config_path = guest.tmp_dir.as_path().join("config"); 7197 std::fs::write(&temp_config_path, request_body).unwrap(); 7198 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 7199 7200 let r = std::panic::catch_unwind(|| { 7201 // Create the VM 7202 assert!(dbus_api.remote_command("create", Some(create_config),)); 7203 7204 // Then boot it 7205 assert!(http_api.remote_command("boot", None)); 7206 guest.wait_vm_boot(None).unwrap(); 7207 7208 // Check that the VM booted as expected 7209 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7210 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7211 7212 // Sync and shutdown without powering off to prevent filesystem 7213 // corruption. 7214 guest.ssh_command("sync").unwrap(); 7215 guest.ssh_command("sudo shutdown -H now").unwrap(); 7216 7217 // Wait for the guest to be fully shutdown 7218 thread::sleep(std::time::Duration::new(20, 0)); 7219 7220 // Then shutdown the VM 7221 assert!(dbus_api.remote_command("shutdown", None)); 7222 7223 // Then boot it again 7224 assert!(http_api.remote_command("boot", None)); 7225 guest.wait_vm_boot(None).unwrap(); 7226 7227 // Check that the VM booted as expected 7228 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7229 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7230 }); 7231 7232 let _ = child.kill(); 7233 let output = child.wait_with_output().unwrap(); 7234 7235 handle_child_output(r, &output); 7236 } 7237 7238 #[test] 7239 fn test_api_dbus_create_boot() { 7240 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7241 let guest = Guest::new(Box::new(focal)); 7242 7243 _test_api_create_boot(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7244 } 7245 7246 #[test] 7247 fn test_api_dbus_shutdown() { 7248 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7249 let guest = Guest::new(Box::new(focal)); 7250 7251 _test_api_shutdown(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7252 } 7253 7254 #[test] 7255 fn test_api_dbus_delete() { 7256 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7257 let guest = Guest::new(Box::new(focal)); 7258 7259 _test_api_delete(TargetApi::new_dbus_api(&guest.tmp_dir), guest); 7260 } 7261 7262 #[test] 7263 fn test_api_dbus_pause_resume() { 7264 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7265 let guest = Guest::new(Box::new(focal)); 7266 7267 _test_api_pause_resume(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7268 } 7269 } 7270 7271 mod common_sequential { 7272 #[cfg(not(feature = "mshv"))] 7273 use crate::*; 7274 7275 #[test] 7276 #[cfg(not(feature = "mshv"))] 7277 fn test_memory_mergeable_on() { 7278 test_memory_mergeable(true) 7279 } 7280 } 7281 7282 mod windows { 7283 use crate::*; 7284 use once_cell::sync::Lazy; 7285 7286 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 7287 7288 struct WindowsGuest { 7289 guest: Guest, 7290 auth: PasswordAuth, 7291 } 7292 7293 trait FsType { 7294 const FS_FAT: u8; 7295 const FS_NTFS: u8; 7296 } 7297 impl FsType for WindowsGuest { 7298 const FS_FAT: u8 = 0; 7299 const FS_NTFS: u8 = 1; 7300 } 7301 7302 impl WindowsGuest { 7303 fn new() -> Self { 7304 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 7305 let guest = Guest::new(Box::new(disk)); 7306 let auth = PasswordAuth { 7307 username: String::from("administrator"), 7308 password: String::from("Admin123"), 7309 }; 7310 7311 WindowsGuest { guest, auth } 7312 } 7313 7314 fn guest(&self) -> &Guest { 7315 &self.guest 7316 } 7317 7318 fn ssh_cmd(&self, cmd: &str) -> String { 7319 ssh_command_ip_with_auth( 7320 cmd, 7321 &self.auth, 7322 &self.guest.network.guest_ip, 7323 DEFAULT_SSH_RETRIES, 7324 DEFAULT_SSH_TIMEOUT, 7325 ) 7326 .unwrap() 7327 } 7328 7329 fn cpu_count(&self) -> u8 { 7330 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 7331 .trim() 7332 .parse::<u8>() 7333 .unwrap_or(0) 7334 } 7335 7336 fn ram_size(&self) -> usize { 7337 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 7338 .trim() 7339 .parse::<usize>() 7340 .unwrap_or(0) 7341 } 7342 7343 fn netdev_count(&self) -> u8 { 7344 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7345 .trim() 7346 .parse::<u8>() 7347 .unwrap_or(0) 7348 } 7349 7350 fn disk_count(&self) -> u8 { 7351 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7352 .trim() 7353 .parse::<u8>() 7354 .unwrap_or(0) 7355 } 7356 7357 fn reboot(&self) { 7358 let _ = self.ssh_cmd("shutdown /r /t 0"); 7359 } 7360 7361 fn shutdown(&self) { 7362 let _ = self.ssh_cmd("shutdown /s /t 0"); 7363 } 7364 7365 fn run_dnsmasq(&self) -> std::process::Child { 7366 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 7367 let dhcp_host = format!( 7368 "--dhcp-host={},{}", 7369 self.guest.network.guest_mac, self.guest.network.guest_ip 7370 ); 7371 let dhcp_range = format!( 7372 "--dhcp-range=eth,{},{}", 7373 self.guest.network.guest_ip, self.guest.network.guest_ip 7374 ); 7375 7376 Command::new("dnsmasq") 7377 .arg("--no-daemon") 7378 .arg("--log-queries") 7379 .arg(listen_address.as_str()) 7380 .arg("--except-interface=lo") 7381 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 7382 .arg("--conf-file=/dev/null") 7383 .arg(dhcp_host.as_str()) 7384 .arg(dhcp_range.as_str()) 7385 .spawn() 7386 .unwrap() 7387 } 7388 7389 // TODO Cleanup image file explicitly after test, if there's some space issues. 7390 fn disk_new(&self, fs: u8, sz: usize) -> String { 7391 let mut guard = NEXT_DISK_ID.lock().unwrap(); 7392 let id = *guard; 7393 *guard = id + 1; 7394 7395 let img = PathBuf::from(format!("/tmp/test-hotplug-{id}.raw")); 7396 let _ = fs::remove_file(&img); 7397 7398 // Create an image file 7399 let out = Command::new("qemu-img") 7400 .args([ 7401 "create", 7402 "-f", 7403 "raw", 7404 img.to_str().unwrap(), 7405 format!("{sz}m").as_str(), 7406 ]) 7407 .output() 7408 .expect("qemu-img command failed") 7409 .stdout; 7410 println!("{out:?}"); 7411 7412 // Associate image to a loop device 7413 let out = Command::new("losetup") 7414 .args(["--show", "-f", img.to_str().unwrap()]) 7415 .output() 7416 .expect("failed to create loop device") 7417 .stdout; 7418 let _tmp = String::from_utf8_lossy(&out); 7419 let loop_dev = _tmp.trim(); 7420 println!("{out:?}"); 7421 7422 // Create a partition table 7423 // echo 'type=7' | sudo sfdisk "${LOOP}" 7424 let mut child = Command::new("sfdisk") 7425 .args([loop_dev]) 7426 .stdin(Stdio::piped()) 7427 .spawn() 7428 .unwrap(); 7429 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 7430 stdin 7431 .write_all("type=7".as_bytes()) 7432 .expect("failed to write stdin"); 7433 let out = child.wait_with_output().expect("sfdisk failed").stdout; 7434 println!("{out:?}"); 7435 7436 // Disengage the loop device 7437 let out = Command::new("losetup") 7438 .args(["-d", loop_dev]) 7439 .output() 7440 .expect("loop device not found") 7441 .stdout; 7442 println!("{out:?}"); 7443 7444 // Re-associate loop device pointing to the partition only 7445 let out = Command::new("losetup") 7446 .args([ 7447 "--show", 7448 "--offset", 7449 (512 * 2048).to_string().as_str(), 7450 "-f", 7451 img.to_str().unwrap(), 7452 ]) 7453 .output() 7454 .expect("failed to create loop device") 7455 .stdout; 7456 let _tmp = String::from_utf8_lossy(&out); 7457 let loop_dev = _tmp.trim(); 7458 println!("{out:?}"); 7459 7460 // Create filesystem. 7461 let fs_cmd = match fs { 7462 WindowsGuest::FS_FAT => "mkfs.msdos", 7463 WindowsGuest::FS_NTFS => "mkfs.ntfs", 7464 _ => panic!("Unknown filesystem type '{fs}'"), 7465 }; 7466 let out = Command::new(fs_cmd) 7467 .args([&loop_dev]) 7468 .output() 7469 .unwrap_or_else(|_| panic!("{fs_cmd} failed")) 7470 .stdout; 7471 println!("{out:?}"); 7472 7473 // Disengage the loop device 7474 let out = Command::new("losetup") 7475 .args(["-d", loop_dev]) 7476 .output() 7477 .unwrap_or_else(|_| panic!("loop device '{loop_dev}' not found")) 7478 .stdout; 7479 println!("{out:?}"); 7480 7481 img.to_str().unwrap().to_string() 7482 } 7483 7484 fn disks_set_rw(&self) { 7485 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 7486 } 7487 7488 fn disks_online(&self) { 7489 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 7490 } 7491 7492 fn disk_file_put(&self, fname: &str, data: &str) { 7493 let _ = self.ssh_cmd(&format!( 7494 "powershell -Command \"'{data}' | Set-Content -Path {fname}\"" 7495 )); 7496 } 7497 7498 fn disk_file_read(&self, fname: &str) -> String { 7499 self.ssh_cmd(&format!( 7500 "powershell -Command \"Get-Content -Path {fname}\"" 7501 )) 7502 } 7503 7504 fn wait_for_boot(&self) -> bool { 7505 let cmd = "dir /b c:\\ | find \"Windows\""; 7506 let tmo_max = 180; 7507 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 7508 // interval must be small. 7509 let tmo_int = 2; 7510 let out = ssh_command_ip_with_auth( 7511 cmd, 7512 &self.auth, 7513 &self.guest.network.guest_ip, 7514 { 7515 let mut ret = 1; 7516 let mut tmo_acc = 0; 7517 loop { 7518 tmo_acc += tmo_int * ret; 7519 if tmo_acc >= tmo_max { 7520 break; 7521 } 7522 ret += 1; 7523 } 7524 ret 7525 }, 7526 tmo_int, 7527 ) 7528 .unwrap(); 7529 7530 if "Windows" == out.trim() { 7531 return true; 7532 } 7533 7534 false 7535 } 7536 } 7537 7538 fn vcpu_threads_count(pid: u32) -> u8 { 7539 // ps -T -p 12345 | grep vcpu | wc -l 7540 let out = Command::new("ps") 7541 .args(["-T", "-p", format!("{pid}").as_str()]) 7542 .output() 7543 .expect("ps command failed") 7544 .stdout; 7545 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 7546 } 7547 7548 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 7549 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 7550 let out = Command::new("ps") 7551 .args(["-T", "-p", format!("{pid}").as_str()]) 7552 .output() 7553 .expect("ps command failed") 7554 .stdout; 7555 let mut n = 0; 7556 String::from_utf8_lossy(&out) 7557 .split_whitespace() 7558 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 7559 n 7560 } 7561 7562 fn disk_ctrl_threads_count(pid: u32) -> u8 { 7563 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 7564 let out = Command::new("ps") 7565 .args(["-T", "-p", format!("{pid}").as_str()]) 7566 .output() 7567 .expect("ps command failed") 7568 .stdout; 7569 let mut n = 0; 7570 String::from_utf8_lossy(&out) 7571 .split_whitespace() 7572 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 7573 n 7574 } 7575 7576 #[test] 7577 fn test_windows_guest() { 7578 let windows_guest = WindowsGuest::new(); 7579 7580 let mut child = GuestCommand::new(windows_guest.guest()) 7581 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7582 .args(["--memory", "size=4G"]) 7583 .args(["--kernel", edk2_path().to_str().unwrap()]) 7584 .args(["--serial", "tty"]) 7585 .args(["--console", "off"]) 7586 .default_disks() 7587 .default_net() 7588 .capture_output() 7589 .spawn() 7590 .unwrap(); 7591 7592 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7593 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7594 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7595 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7596 7597 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7598 7599 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7600 7601 let r = std::panic::catch_unwind(|| { 7602 // Wait to make sure Windows boots up 7603 assert!(windows_guest.wait_for_boot()); 7604 7605 windows_guest.shutdown(); 7606 }); 7607 7608 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7609 let _ = child.kill(); 7610 let output = child.wait_with_output().unwrap(); 7611 7612 let _ = child_dnsmasq.kill(); 7613 let _ = child_dnsmasq.wait(); 7614 7615 handle_child_output(r, &output); 7616 } 7617 7618 #[test] 7619 fn test_windows_guest_multiple_queues() { 7620 let windows_guest = WindowsGuest::new(); 7621 7622 let mut ovmf_path = dirs::home_dir().unwrap(); 7623 ovmf_path.push("workloads"); 7624 ovmf_path.push(OVMF_NAME); 7625 7626 let mut child = GuestCommand::new(windows_guest.guest()) 7627 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 7628 .args(["--memory", "size=4G"]) 7629 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7630 .args(["--serial", "tty"]) 7631 .args(["--console", "off"]) 7632 .args([ 7633 "--disk", 7634 format!( 7635 "path={},num_queues=4", 7636 windows_guest 7637 .guest() 7638 .disk_config 7639 .disk(DiskType::OperatingSystem) 7640 .unwrap() 7641 ) 7642 .as_str(), 7643 ]) 7644 .args([ 7645 "--net", 7646 format!( 7647 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 7648 windows_guest.guest().network.guest_mac, 7649 windows_guest.guest().network.host_ip 7650 ) 7651 .as_str(), 7652 ]) 7653 .capture_output() 7654 .spawn() 7655 .unwrap(); 7656 7657 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7658 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7659 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7660 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7661 7662 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7663 7664 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7665 7666 let r = std::panic::catch_unwind(|| { 7667 // Wait to make sure Windows boots up 7668 assert!(windows_guest.wait_for_boot()); 7669 7670 windows_guest.shutdown(); 7671 }); 7672 7673 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7674 let _ = child.kill(); 7675 let output = child.wait_with_output().unwrap(); 7676 7677 let _ = child_dnsmasq.kill(); 7678 let _ = child_dnsmasq.wait(); 7679 7680 handle_child_output(r, &output); 7681 } 7682 7683 #[test] 7684 #[cfg(not(feature = "mshv"))] 7685 #[ignore = "See #4327"] 7686 fn test_windows_guest_snapshot_restore() { 7687 let windows_guest = WindowsGuest::new(); 7688 7689 let mut ovmf_path = dirs::home_dir().unwrap(); 7690 ovmf_path.push("workloads"); 7691 ovmf_path.push(OVMF_NAME); 7692 7693 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7694 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 7695 7696 let mut child = GuestCommand::new(windows_guest.guest()) 7697 .args(["--api-socket", &api_socket_source]) 7698 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7699 .args(["--memory", "size=4G"]) 7700 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7701 .args(["--serial", "tty"]) 7702 .args(["--console", "off"]) 7703 .default_disks() 7704 .default_net() 7705 .capture_output() 7706 .spawn() 7707 .unwrap(); 7708 7709 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7710 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7711 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7712 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7713 7714 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7715 7716 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7717 7718 // Wait to make sure Windows boots up 7719 assert!(windows_guest.wait_for_boot()); 7720 7721 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 7722 7723 // Pause the VM 7724 assert!(remote_command(&api_socket_source, "pause", None)); 7725 7726 // Take a snapshot from the VM 7727 assert!(remote_command( 7728 &api_socket_source, 7729 "snapshot", 7730 Some(format!("file://{snapshot_dir}").as_str()), 7731 )); 7732 7733 // Wait to make sure the snapshot is completed 7734 thread::sleep(std::time::Duration::new(30, 0)); 7735 7736 let _ = child.kill(); 7737 child.wait().unwrap(); 7738 7739 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 7740 7741 // Restore the VM from the snapshot 7742 let mut child = GuestCommand::new(windows_guest.guest()) 7743 .args(["--api-socket", &api_socket_restored]) 7744 .args([ 7745 "--restore", 7746 format!("source_url=file://{snapshot_dir}").as_str(), 7747 ]) 7748 .capture_output() 7749 .spawn() 7750 .unwrap(); 7751 7752 // Wait for the VM to be restored 7753 thread::sleep(std::time::Duration::new(20, 0)); 7754 7755 let r = std::panic::catch_unwind(|| { 7756 // Resume the VM 7757 assert!(remote_command(&api_socket_restored, "resume", None)); 7758 7759 windows_guest.shutdown(); 7760 }); 7761 7762 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7763 let _ = child.kill(); 7764 let output = child.wait_with_output().unwrap(); 7765 7766 let _ = child_dnsmasq.kill(); 7767 let _ = child_dnsmasq.wait(); 7768 7769 handle_child_output(r, &output); 7770 } 7771 7772 #[test] 7773 #[cfg(not(feature = "mshv"))] 7774 #[cfg(not(target_arch = "aarch64"))] 7775 fn test_windows_guest_cpu_hotplug() { 7776 let windows_guest = WindowsGuest::new(); 7777 7778 let mut ovmf_path = dirs::home_dir().unwrap(); 7779 ovmf_path.push("workloads"); 7780 ovmf_path.push(OVMF_NAME); 7781 7782 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7783 let api_socket = temp_api_path(&tmp_dir); 7784 7785 let mut child = GuestCommand::new(windows_guest.guest()) 7786 .args(["--api-socket", &api_socket]) 7787 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 7788 .args(["--memory", "size=4G"]) 7789 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7790 .args(["--serial", "tty"]) 7791 .args(["--console", "off"]) 7792 .default_disks() 7793 .default_net() 7794 .capture_output() 7795 .spawn() 7796 .unwrap(); 7797 7798 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7799 7800 let r = std::panic::catch_unwind(|| { 7801 // Wait to make sure Windows boots up 7802 assert!(windows_guest.wait_for_boot()); 7803 7804 let vcpu_num = 2; 7805 // Check the initial number of CPUs the guest sees 7806 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7807 // Check the initial number of vcpu threads in the CH process 7808 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7809 7810 let vcpu_num = 6; 7811 // Hotplug some CPUs 7812 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7813 // Wait to make sure CPUs are added 7814 thread::sleep(std::time::Duration::new(10, 0)); 7815 // Check the guest sees the correct number 7816 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7817 // Check the CH process has the correct number of vcpu threads 7818 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7819 7820 let vcpu_num = 4; 7821 // Remove some CPUs. Note that Windows doesn't support hot-remove. 7822 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7823 // Wait to make sure CPUs are removed 7824 thread::sleep(std::time::Duration::new(10, 0)); 7825 // Reboot to let Windows catch up 7826 windows_guest.reboot(); 7827 // Wait to make sure Windows completely rebooted 7828 thread::sleep(std::time::Duration::new(60, 0)); 7829 // Check the guest sees the correct number 7830 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7831 // Check the CH process has the correct number of vcpu threads 7832 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7833 7834 windows_guest.shutdown(); 7835 }); 7836 7837 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7838 let _ = child.kill(); 7839 let output = child.wait_with_output().unwrap(); 7840 7841 let _ = child_dnsmasq.kill(); 7842 let _ = child_dnsmasq.wait(); 7843 7844 handle_child_output(r, &output); 7845 } 7846 7847 #[test] 7848 #[cfg(not(feature = "mshv"))] 7849 #[cfg(not(target_arch = "aarch64"))] 7850 fn test_windows_guest_ram_hotplug() { 7851 let windows_guest = WindowsGuest::new(); 7852 7853 let mut ovmf_path = dirs::home_dir().unwrap(); 7854 ovmf_path.push("workloads"); 7855 ovmf_path.push(OVMF_NAME); 7856 7857 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7858 let api_socket = temp_api_path(&tmp_dir); 7859 7860 let mut child = GuestCommand::new(windows_guest.guest()) 7861 .args(["--api-socket", &api_socket]) 7862 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7863 .args(["--memory", "size=2G,hotplug_size=5G"]) 7864 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7865 .args(["--serial", "tty"]) 7866 .args(["--console", "off"]) 7867 .default_disks() 7868 .default_net() 7869 .capture_output() 7870 .spawn() 7871 .unwrap(); 7872 7873 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7874 7875 let r = std::panic::catch_unwind(|| { 7876 // Wait to make sure Windows boots up 7877 assert!(windows_guest.wait_for_boot()); 7878 7879 let ram_size = 2 * 1024 * 1024 * 1024; 7880 // Check the initial number of RAM the guest sees 7881 let current_ram_size = windows_guest.ram_size(); 7882 // This size seems to be reserved by the system and thus the 7883 // reported amount differs by this constant value. 7884 let reserved_ram_size = ram_size - current_ram_size; 7885 // Verify that there's not more than 4mb constant diff wasted 7886 // by the reserved ram. 7887 assert!(reserved_ram_size < 4 * 1024 * 1024); 7888 7889 let ram_size = 4 * 1024 * 1024 * 1024; 7890 // Hotplug some RAM 7891 resize_command(&api_socket, None, Some(ram_size), None, None); 7892 // Wait to make sure RAM has been added 7893 thread::sleep(std::time::Duration::new(10, 0)); 7894 // Check the guest sees the correct number 7895 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7896 7897 let ram_size = 3 * 1024 * 1024 * 1024; 7898 // Unplug some RAM. Note that hot-remove most likely won't work. 7899 resize_command(&api_socket, None, Some(ram_size), None, None); 7900 // Wait to make sure RAM has been added 7901 thread::sleep(std::time::Duration::new(10, 0)); 7902 // Reboot to let Windows catch up 7903 windows_guest.reboot(); 7904 // Wait to make sure guest completely rebooted 7905 thread::sleep(std::time::Duration::new(60, 0)); 7906 // Check the guest sees the correct number 7907 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7908 7909 windows_guest.shutdown(); 7910 }); 7911 7912 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7913 let _ = child.kill(); 7914 let output = child.wait_with_output().unwrap(); 7915 7916 let _ = child_dnsmasq.kill(); 7917 let _ = child_dnsmasq.wait(); 7918 7919 handle_child_output(r, &output); 7920 } 7921 7922 #[test] 7923 #[cfg(not(feature = "mshv"))] 7924 fn test_windows_guest_netdev_hotplug() { 7925 let windows_guest = WindowsGuest::new(); 7926 7927 let mut ovmf_path = dirs::home_dir().unwrap(); 7928 ovmf_path.push("workloads"); 7929 ovmf_path.push(OVMF_NAME); 7930 7931 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7932 let api_socket = temp_api_path(&tmp_dir); 7933 7934 let mut child = GuestCommand::new(windows_guest.guest()) 7935 .args(["--api-socket", &api_socket]) 7936 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7937 .args(["--memory", "size=4G"]) 7938 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7939 .args(["--serial", "tty"]) 7940 .args(["--console", "off"]) 7941 .default_disks() 7942 .default_net() 7943 .capture_output() 7944 .spawn() 7945 .unwrap(); 7946 7947 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7948 7949 let r = std::panic::catch_unwind(|| { 7950 // Wait to make sure Windows boots up 7951 assert!(windows_guest.wait_for_boot()); 7952 7953 // Initially present network device 7954 let netdev_num = 1; 7955 assert_eq!(windows_guest.netdev_count(), netdev_num); 7956 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7957 7958 // Hotplug network device 7959 let (cmd_success, cmd_output) = remote_command_w_output( 7960 &api_socket, 7961 "add-net", 7962 Some(windows_guest.guest().default_net_string().as_str()), 7963 ); 7964 assert!(cmd_success); 7965 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 7966 thread::sleep(std::time::Duration::new(5, 0)); 7967 // Verify the device is on the system 7968 let netdev_num = 2; 7969 assert_eq!(windows_guest.netdev_count(), netdev_num); 7970 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7971 7972 // Remove network device 7973 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 7974 assert!(cmd_success); 7975 thread::sleep(std::time::Duration::new(5, 0)); 7976 // Verify the device has been removed 7977 let netdev_num = 1; 7978 assert_eq!(windows_guest.netdev_count(), netdev_num); 7979 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7980 7981 windows_guest.shutdown(); 7982 }); 7983 7984 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7985 let _ = child.kill(); 7986 let output = child.wait_with_output().unwrap(); 7987 7988 let _ = child_dnsmasq.kill(); 7989 let _ = child_dnsmasq.wait(); 7990 7991 handle_child_output(r, &output); 7992 } 7993 7994 #[test] 7995 #[cfg(not(feature = "mshv"))] 7996 #[cfg(not(target_arch = "aarch64"))] 7997 fn test_windows_guest_disk_hotplug() { 7998 let windows_guest = WindowsGuest::new(); 7999 8000 let mut ovmf_path = dirs::home_dir().unwrap(); 8001 ovmf_path.push("workloads"); 8002 ovmf_path.push(OVMF_NAME); 8003 8004 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8005 let api_socket = temp_api_path(&tmp_dir); 8006 8007 let mut child = GuestCommand::new(windows_guest.guest()) 8008 .args(["--api-socket", &api_socket]) 8009 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8010 .args(["--memory", "size=4G"]) 8011 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8012 .args(["--serial", "tty"]) 8013 .args(["--console", "off"]) 8014 .default_disks() 8015 .default_net() 8016 .capture_output() 8017 .spawn() 8018 .unwrap(); 8019 8020 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8021 8022 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 8023 8024 let r = std::panic::catch_unwind(|| { 8025 // Wait to make sure Windows boots up 8026 assert!(windows_guest.wait_for_boot()); 8027 8028 // Initially present disk device 8029 let disk_num = 1; 8030 assert_eq!(windows_guest.disk_count(), disk_num); 8031 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8032 8033 // Hotplug disk device 8034 let (cmd_success, cmd_output) = remote_command_w_output( 8035 &api_socket, 8036 "add-disk", 8037 Some(format!("path={disk},readonly=off").as_str()), 8038 ); 8039 assert!(cmd_success); 8040 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 8041 thread::sleep(std::time::Duration::new(5, 0)); 8042 // Online disk device 8043 windows_guest.disks_set_rw(); 8044 windows_guest.disks_online(); 8045 // Verify the device is on the system 8046 let disk_num = 2; 8047 assert_eq!(windows_guest.disk_count(), disk_num); 8048 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8049 8050 let data = "hello"; 8051 let fname = "d:\\world"; 8052 windows_guest.disk_file_put(fname, data); 8053 8054 // Unmount disk device 8055 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 8056 assert!(cmd_success); 8057 thread::sleep(std::time::Duration::new(5, 0)); 8058 // Verify the device has been removed 8059 let disk_num = 1; 8060 assert_eq!(windows_guest.disk_count(), disk_num); 8061 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8062 8063 // Remount and check the file exists with the expected contents 8064 let (cmd_success, _cmd_output) = remote_command_w_output( 8065 &api_socket, 8066 "add-disk", 8067 Some(format!("path={disk},readonly=off").as_str()), 8068 ); 8069 assert!(cmd_success); 8070 thread::sleep(std::time::Duration::new(5, 0)); 8071 let out = windows_guest.disk_file_read(fname); 8072 assert_eq!(data, out.trim()); 8073 8074 // Intentionally no unmount, it'll happen at shutdown. 8075 8076 windows_guest.shutdown(); 8077 }); 8078 8079 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8080 let _ = child.kill(); 8081 let output = child.wait_with_output().unwrap(); 8082 8083 let _ = child_dnsmasq.kill(); 8084 let _ = child_dnsmasq.wait(); 8085 8086 handle_child_output(r, &output); 8087 } 8088 8089 #[test] 8090 #[cfg(not(feature = "mshv"))] 8091 #[cfg(not(target_arch = "aarch64"))] 8092 fn test_windows_guest_disk_hotplug_multi() { 8093 let windows_guest = WindowsGuest::new(); 8094 8095 let mut ovmf_path = dirs::home_dir().unwrap(); 8096 ovmf_path.push("workloads"); 8097 ovmf_path.push(OVMF_NAME); 8098 8099 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8100 let api_socket = temp_api_path(&tmp_dir); 8101 8102 let mut child = GuestCommand::new(windows_guest.guest()) 8103 .args(["--api-socket", &api_socket]) 8104 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8105 .args(["--memory", "size=2G"]) 8106 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8107 .args(["--serial", "tty"]) 8108 .args(["--console", "off"]) 8109 .default_disks() 8110 .default_net() 8111 .capture_output() 8112 .spawn() 8113 .unwrap(); 8114 8115 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8116 8117 // Predefined data to used at various test stages 8118 let disk_test_data: [[String; 4]; 2] = [ 8119 [ 8120 "_disk2".to_string(), 8121 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 8122 "d:\\world".to_string(), 8123 "hello".to_string(), 8124 ], 8125 [ 8126 "_disk3".to_string(), 8127 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 8128 "e:\\hello".to_string(), 8129 "world".to_string(), 8130 ], 8131 ]; 8132 8133 let r = std::panic::catch_unwind(|| { 8134 // Wait to make sure Windows boots up 8135 assert!(windows_guest.wait_for_boot()); 8136 8137 // Initially present disk device 8138 let disk_num = 1; 8139 assert_eq!(windows_guest.disk_count(), disk_num); 8140 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8141 8142 for it in &disk_test_data { 8143 let disk_id = it[0].as_str(); 8144 let disk = it[1].as_str(); 8145 // Hotplug disk device 8146 let (cmd_success, cmd_output) = remote_command_w_output( 8147 &api_socket, 8148 "add-disk", 8149 Some(format!("path={disk},readonly=off").as_str()), 8150 ); 8151 assert!(cmd_success); 8152 assert!(String::from_utf8_lossy(&cmd_output) 8153 .contains(format!("\"id\":\"{disk_id}\"").as_str())); 8154 thread::sleep(std::time::Duration::new(5, 0)); 8155 // Online disk devices 8156 windows_guest.disks_set_rw(); 8157 windows_guest.disks_online(); 8158 } 8159 // Verify the devices are on the system 8160 let disk_num = (disk_test_data.len() + 1) as u8; 8161 assert_eq!(windows_guest.disk_count(), disk_num); 8162 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8163 8164 // Put test data 8165 for it in &disk_test_data { 8166 let fname = it[2].as_str(); 8167 let data = it[3].as_str(); 8168 windows_guest.disk_file_put(fname, data); 8169 } 8170 8171 // Unmount disk devices 8172 for it in &disk_test_data { 8173 let disk_id = it[0].as_str(); 8174 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 8175 assert!(cmd_success); 8176 thread::sleep(std::time::Duration::new(5, 0)); 8177 } 8178 8179 // Verify the devices have been removed 8180 let disk_num = 1; 8181 assert_eq!(windows_guest.disk_count(), disk_num); 8182 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8183 8184 // Remount 8185 for it in &disk_test_data { 8186 let disk = it[1].as_str(); 8187 let (cmd_success, _cmd_output) = remote_command_w_output( 8188 &api_socket, 8189 "add-disk", 8190 Some(format!("path={disk},readonly=off").as_str()), 8191 ); 8192 assert!(cmd_success); 8193 thread::sleep(std::time::Duration::new(5, 0)); 8194 } 8195 8196 // Check the files exists with the expected contents 8197 for it in &disk_test_data { 8198 let fname = it[2].as_str(); 8199 let data = it[3].as_str(); 8200 let out = windows_guest.disk_file_read(fname); 8201 assert_eq!(data, out.trim()); 8202 } 8203 8204 // Intentionally no unmount, it'll happen at shutdown. 8205 8206 windows_guest.shutdown(); 8207 }); 8208 8209 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8210 let _ = child.kill(); 8211 let output = child.wait_with_output().unwrap(); 8212 8213 let _ = child_dnsmasq.kill(); 8214 let _ = child_dnsmasq.wait(); 8215 8216 handle_child_output(r, &output); 8217 } 8218 8219 #[test] 8220 #[cfg(not(feature = "mshv"))] 8221 #[cfg(not(target_arch = "aarch64"))] 8222 fn test_windows_guest_netdev_multi() { 8223 let windows_guest = WindowsGuest::new(); 8224 8225 let mut ovmf_path = dirs::home_dir().unwrap(); 8226 ovmf_path.push("workloads"); 8227 ovmf_path.push(OVMF_NAME); 8228 8229 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8230 let api_socket = temp_api_path(&tmp_dir); 8231 8232 let mut child = GuestCommand::new(windows_guest.guest()) 8233 .args(["--api-socket", &api_socket]) 8234 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8235 .args(["--memory", "size=4G"]) 8236 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8237 .args(["--serial", "tty"]) 8238 .args(["--console", "off"]) 8239 .default_disks() 8240 // The multi net dev config is borrowed from test_multiple_network_interfaces 8241 .args([ 8242 "--net", 8243 windows_guest.guest().default_net_string().as_str(), 8244 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 8245 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 8246 ]) 8247 .capture_output() 8248 .spawn() 8249 .unwrap(); 8250 8251 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8252 8253 let r = std::panic::catch_unwind(|| { 8254 // Wait to make sure Windows boots up 8255 assert!(windows_guest.wait_for_boot()); 8256 8257 let netdev_num = 3; 8258 assert_eq!(windows_guest.netdev_count(), netdev_num); 8259 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8260 8261 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 8262 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 8263 8264 windows_guest.shutdown(); 8265 }); 8266 8267 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8268 let _ = child.kill(); 8269 let output = child.wait_with_output().unwrap(); 8270 8271 let _ = child_dnsmasq.kill(); 8272 let _ = child_dnsmasq.wait(); 8273 8274 handle_child_output(r, &output); 8275 } 8276 } 8277 8278 #[cfg(target_arch = "x86_64")] 8279 mod sgx { 8280 use crate::*; 8281 8282 #[test] 8283 fn test_sgx() { 8284 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 8285 let jammy = UbuntuDiskConfig::new(jammy_image); 8286 let guest = Guest::new(Box::new(jammy)); 8287 8288 let mut child = GuestCommand::new(&guest) 8289 .args(["--cpus", "boot=1"]) 8290 .args(["--memory", "size=512M"]) 8291 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8292 .default_disks() 8293 .default_net() 8294 .args(["--sgx-epc", "id=epc0,size=64M"]) 8295 .capture_output() 8296 .spawn() 8297 .unwrap(); 8298 8299 let r = std::panic::catch_unwind(|| { 8300 guest.wait_vm_boot(None).unwrap(); 8301 8302 // Check if SGX is correctly detected in the guest. 8303 guest.check_sgx_support().unwrap(); 8304 8305 // Validate the SGX EPC section is 64MiB. 8306 assert_eq!( 8307 guest 8308 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 8309 .unwrap() 8310 .trim(), 8311 "0x0000000004000000" 8312 ); 8313 }); 8314 8315 let _ = child.kill(); 8316 let output = child.wait_with_output().unwrap(); 8317 8318 handle_child_output(r, &output); 8319 } 8320 } 8321 8322 #[cfg(target_arch = "x86_64")] 8323 mod vfio { 8324 use crate::*; 8325 8326 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 8327 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8328 let guest = Guest::new(Box::new(jammy)); 8329 let api_socket = temp_api_path(&guest.tmp_dir); 8330 8331 let mut child = GuestCommand::new(&guest) 8332 .args(["--cpus", "boot=4"]) 8333 .args([ 8334 "--memory", 8335 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(), 8336 ]) 8337 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8338 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8339 .args(["--api-socket", &api_socket]) 8340 .default_disks() 8341 .default_net() 8342 .capture_output() 8343 .spawn() 8344 .unwrap(); 8345 8346 let r = std::panic::catch_unwind(|| { 8347 guest.wait_vm_boot(None).unwrap(); 8348 8349 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8350 8351 guest.enable_memory_hotplug(); 8352 8353 // Add RAM to the VM 8354 let desired_ram = 6 << 30; 8355 resize_command(&api_socket, None, Some(desired_ram), None, None); 8356 thread::sleep(std::time::Duration::new(30, 0)); 8357 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8358 8359 // Check the VFIO device works when RAM is increased to 6GiB 8360 guest.check_nvidia_gpu(); 8361 }); 8362 8363 let _ = child.kill(); 8364 let output = child.wait_with_output().unwrap(); 8365 8366 handle_child_output(r, &output); 8367 } 8368 8369 #[test] 8370 fn test_nvidia_card_memory_hotplug_acpi() { 8371 test_nvidia_card_memory_hotplug("acpi") 8372 } 8373 8374 #[test] 8375 fn test_nvidia_card_memory_hotplug_virtio_mem() { 8376 test_nvidia_card_memory_hotplug("virtio-mem") 8377 } 8378 8379 #[test] 8380 fn test_nvidia_card_pci_hotplug() { 8381 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8382 let guest = Guest::new(Box::new(jammy)); 8383 let api_socket = temp_api_path(&guest.tmp_dir); 8384 8385 let mut child = GuestCommand::new(&guest) 8386 .args(["--cpus", "boot=4"]) 8387 .args(["--memory", "size=4G"]) 8388 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8389 .args(["--api-socket", &api_socket]) 8390 .default_disks() 8391 .default_net() 8392 .capture_output() 8393 .spawn() 8394 .unwrap(); 8395 8396 let r = std::panic::catch_unwind(|| { 8397 guest.wait_vm_boot(None).unwrap(); 8398 8399 // Hotplug the card to the VM 8400 let (cmd_success, cmd_output) = remote_command_w_output( 8401 &api_socket, 8402 "add-device", 8403 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 8404 ); 8405 assert!(cmd_success); 8406 assert!(String::from_utf8_lossy(&cmd_output) 8407 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 8408 8409 thread::sleep(std::time::Duration::new(10, 0)); 8410 8411 // Check the VFIO device works after hotplug 8412 guest.check_nvidia_gpu(); 8413 }); 8414 8415 let _ = child.kill(); 8416 let output = child.wait_with_output().unwrap(); 8417 8418 handle_child_output(r, &output); 8419 } 8420 8421 #[test] 8422 fn test_nvidia_card_reboot() { 8423 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8424 let guest = Guest::new(Box::new(jammy)); 8425 let api_socket = temp_api_path(&guest.tmp_dir); 8426 8427 let mut child = GuestCommand::new(&guest) 8428 .args(["--cpus", "boot=4"]) 8429 .args(["--memory", "size=4G"]) 8430 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8431 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8432 .args(["--api-socket", &api_socket]) 8433 .default_disks() 8434 .default_net() 8435 .capture_output() 8436 .spawn() 8437 .unwrap(); 8438 8439 let r = std::panic::catch_unwind(|| { 8440 guest.wait_vm_boot(None).unwrap(); 8441 8442 // Check the VFIO device works after boot 8443 guest.check_nvidia_gpu(); 8444 8445 guest.reboot_linux(0, None); 8446 8447 // Check the VFIO device works after reboot 8448 guest.check_nvidia_gpu(); 8449 }); 8450 8451 let _ = child.kill(); 8452 let output = child.wait_with_output().unwrap(); 8453 8454 handle_child_output(r, &output); 8455 } 8456 } 8457 8458 mod live_migration { 8459 use crate::*; 8460 8461 fn start_live_migration( 8462 migration_socket: &str, 8463 src_api_socket: &str, 8464 dest_api_socket: &str, 8465 local: bool, 8466 ) -> bool { 8467 // Start to receive migration from the destintion VM 8468 let mut receive_migration = Command::new(clh_command("ch-remote")) 8469 .args([ 8470 &format!("--api-socket={dest_api_socket}"), 8471 "receive-migration", 8472 &format! {"unix:{migration_socket}"}, 8473 ]) 8474 .stderr(Stdio::piped()) 8475 .stdout(Stdio::piped()) 8476 .spawn() 8477 .unwrap(); 8478 // Give it '1s' to make sure the 'migration_socket' file is properly created 8479 thread::sleep(std::time::Duration::new(1, 0)); 8480 // Start to send migration from the source VM 8481 8482 let mut args = [ 8483 format!("--api-socket={}", &src_api_socket), 8484 "send-migration".to_string(), 8485 format! {"unix:{migration_socket}"}, 8486 ] 8487 .to_vec(); 8488 8489 if local { 8490 args.insert(2, "--local".to_string()); 8491 } 8492 8493 let mut send_migration = Command::new(clh_command("ch-remote")) 8494 .args(&args) 8495 .stderr(Stdio::piped()) 8496 .stdout(Stdio::piped()) 8497 .spawn() 8498 .unwrap(); 8499 8500 // The 'send-migration' command should be executed successfully within the given timeout 8501 let send_success = if let Some(status) = send_migration 8502 .wait_timeout(std::time::Duration::from_secs(30)) 8503 .unwrap() 8504 { 8505 status.success() 8506 } else { 8507 false 8508 }; 8509 8510 if !send_success { 8511 let _ = send_migration.kill(); 8512 let output = send_migration.wait_with_output().unwrap(); 8513 eprintln!( 8514 "\n\n==== Start 'send_migration' output ==== \ 8515 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8516 \n\n==== End 'send_migration' output ====\n\n", 8517 String::from_utf8_lossy(&output.stdout), 8518 String::from_utf8_lossy(&output.stderr) 8519 ); 8520 } 8521 8522 // The 'receive-migration' command should be executed successfully within the given timeout 8523 let receive_success = if let Some(status) = receive_migration 8524 .wait_timeout(std::time::Duration::from_secs(30)) 8525 .unwrap() 8526 { 8527 status.success() 8528 } else { 8529 false 8530 }; 8531 8532 if !receive_success { 8533 let _ = receive_migration.kill(); 8534 let output = receive_migration.wait_with_output().unwrap(); 8535 eprintln!( 8536 "\n\n==== Start 'receive_migration' output ==== \ 8537 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8538 \n\n==== End 'receive_migration' output ====\n\n", 8539 String::from_utf8_lossy(&output.stdout), 8540 String::from_utf8_lossy(&output.stderr) 8541 ); 8542 } 8543 8544 send_success && receive_success 8545 } 8546 8547 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 8548 let mut src_vm = src_vm; 8549 let mut dest_vm = dest_vm; 8550 8551 let _ = src_vm.kill(); 8552 let src_output = src_vm.wait_with_output().unwrap(); 8553 eprintln!( 8554 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8555 String::from_utf8_lossy(&src_output.stdout) 8556 ); 8557 eprintln!( 8558 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8559 String::from_utf8_lossy(&src_output.stderr) 8560 ); 8561 let _ = dest_vm.kill(); 8562 let dest_output = dest_vm.wait_with_output().unwrap(); 8563 eprintln!( 8564 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8565 String::from_utf8_lossy(&dest_output.stdout) 8566 ); 8567 eprintln!( 8568 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8569 String::from_utf8_lossy(&dest_output.stderr) 8570 ); 8571 8572 if let Some(ovs_vm) = ovs_vm { 8573 let mut ovs_vm = ovs_vm; 8574 let _ = ovs_vm.kill(); 8575 let ovs_output = ovs_vm.wait_with_output().unwrap(); 8576 eprintln!( 8577 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 8578 String::from_utf8_lossy(&ovs_output.stdout) 8579 ); 8580 eprintln!( 8581 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 8582 String::from_utf8_lossy(&ovs_output.stderr) 8583 ); 8584 8585 cleanup_ovs_dpdk(); 8586 } 8587 8588 panic!("Test failed: {message}") 8589 } 8590 8591 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 8592 // same host. It ensures the following behaviors: 8593 // 1. The source VM is up and functional (including various virtio-devices are working properly); 8594 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 8595 // 3. The source VM terminated gracefully after live migration; 8596 // 4. The destination VM is functional (including various virtio-devices are working properly) after 8597 // live migration; 8598 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 8599 fn _test_live_migration(upgrade_test: bool, local: bool) { 8600 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8601 let guest = Guest::new(Box::new(focal)); 8602 let kernel_path = direct_kernel_boot_path(); 8603 let console_text = String::from("On a branch floating down river a cricket, singing."); 8604 let net_id = "net123"; 8605 let net_params = format!( 8606 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8607 net_id, guest.network.guest_mac, guest.network.host_ip 8608 ); 8609 8610 let memory_param: &[&str] = if local { 8611 &["--memory", "size=4G,shared=on"] 8612 } else { 8613 &["--memory", "size=4G"] 8614 }; 8615 8616 let boot_vcpus = 2; 8617 let max_vcpus = 4; 8618 8619 let pmem_temp_file = TempFile::new().unwrap(); 8620 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8621 std::process::Command::new("mkfs.ext4") 8622 .arg(pmem_temp_file.as_path()) 8623 .output() 8624 .expect("Expect creating disk image to succeed"); 8625 let pmem_path = String::from("/dev/pmem0"); 8626 8627 // Start the source VM 8628 let src_vm_path = if !upgrade_test { 8629 clh_command("cloud-hypervisor") 8630 } else { 8631 cloud_hypervisor_release_path() 8632 }; 8633 let src_api_socket = temp_api_path(&guest.tmp_dir); 8634 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8635 src_vm_cmd 8636 .args([ 8637 "--cpus", 8638 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8639 ]) 8640 .args(memory_param) 8641 .args(["--kernel", kernel_path.to_str().unwrap()]) 8642 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8643 .default_disks() 8644 .args(["--net", net_params.as_str()]) 8645 .args(["--api-socket", &src_api_socket]) 8646 .args([ 8647 "--pmem", 8648 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8649 ]); 8650 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8651 8652 // Start the destination VM 8653 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8654 dest_api_socket.push_str(".dest"); 8655 let mut dest_child = GuestCommand::new(&guest) 8656 .args(["--api-socket", &dest_api_socket]) 8657 .capture_output() 8658 .spawn() 8659 .unwrap(); 8660 8661 let r = std::panic::catch_unwind(|| { 8662 guest.wait_vm_boot(None).unwrap(); 8663 8664 // Make sure the source VM is functaionl 8665 // Check the number of vCPUs 8666 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8667 8668 // Check the guest RAM 8669 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8670 8671 // Check the guest virtio-devices, e.g. block, rng, console, and net 8672 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8673 8674 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8675 // to make sure that removing and adding back the virtio-net device does 8676 // not break the live-migration support for virtio-pci. 8677 #[cfg(target_arch = "x86_64")] 8678 { 8679 assert!(remote_command( 8680 &src_api_socket, 8681 "remove-device", 8682 Some(net_id), 8683 )); 8684 thread::sleep(std::time::Duration::new(10, 0)); 8685 8686 // Plug the virtio-net device again 8687 assert!(remote_command( 8688 &src_api_socket, 8689 "add-net", 8690 Some(net_params.as_str()), 8691 )); 8692 thread::sleep(std::time::Duration::new(10, 0)); 8693 } 8694 8695 // Start the live-migration 8696 let migration_socket = String::from( 8697 guest 8698 .tmp_dir 8699 .as_path() 8700 .join("live-migration.sock") 8701 .to_str() 8702 .unwrap(), 8703 ); 8704 8705 assert!( 8706 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8707 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8708 ); 8709 }); 8710 8711 // Check and report any errors occurred during the live-migration 8712 if r.is_err() { 8713 print_and_panic( 8714 src_child, 8715 dest_child, 8716 None, 8717 "Error occurred during live-migration", 8718 ); 8719 } 8720 8721 // Check the source vm has been terminated successful (give it '3s' to settle) 8722 thread::sleep(std::time::Duration::new(3, 0)); 8723 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8724 print_and_panic( 8725 src_child, 8726 dest_child, 8727 None, 8728 "source VM was not terminated successfully.", 8729 ); 8730 }; 8731 8732 // Post live-migration check to make sure the destination VM is funcational 8733 let r = std::panic::catch_unwind(|| { 8734 // Perform same checks to validate VM has been properly migrated 8735 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8736 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8737 8738 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8739 }); 8740 8741 // Clean-up the destination VM and make sure it terminated correctly 8742 let _ = dest_child.kill(); 8743 let dest_output = dest_child.wait_with_output().unwrap(); 8744 handle_child_output(r, &dest_output); 8745 8746 // Check the destination VM has the expected 'concole_text' from its output 8747 let r = std::panic::catch_unwind(|| { 8748 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8749 }); 8750 handle_child_output(r, &dest_output); 8751 } 8752 8753 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 8754 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8755 let guest = Guest::new(Box::new(focal)); 8756 let kernel_path = direct_kernel_boot_path(); 8757 let console_text = String::from("On a branch floating down river a cricket, singing."); 8758 let net_id = "net123"; 8759 let net_params = format!( 8760 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8761 net_id, guest.network.guest_mac, guest.network.host_ip 8762 ); 8763 8764 let memory_param: &[&str] = if local { 8765 &[ 8766 "--memory", 8767 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 8768 "--balloon", 8769 "size=0", 8770 ] 8771 } else { 8772 &[ 8773 "--memory", 8774 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 8775 "--balloon", 8776 "size=0", 8777 ] 8778 }; 8779 8780 let boot_vcpus = 2; 8781 let max_vcpus = 4; 8782 8783 let pmem_temp_file = TempFile::new().unwrap(); 8784 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8785 std::process::Command::new("mkfs.ext4") 8786 .arg(pmem_temp_file.as_path()) 8787 .output() 8788 .expect("Expect creating disk image to succeed"); 8789 let pmem_path = String::from("/dev/pmem0"); 8790 8791 // Start the source VM 8792 let src_vm_path = if !upgrade_test { 8793 clh_command("cloud-hypervisor") 8794 } else { 8795 cloud_hypervisor_release_path() 8796 }; 8797 let src_api_socket = temp_api_path(&guest.tmp_dir); 8798 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8799 src_vm_cmd 8800 .args([ 8801 "--cpus", 8802 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8803 ]) 8804 .args(memory_param) 8805 .args(["--kernel", kernel_path.to_str().unwrap()]) 8806 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8807 .default_disks() 8808 .args(["--net", net_params.as_str()]) 8809 .args(["--api-socket", &src_api_socket]) 8810 .args([ 8811 "--pmem", 8812 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8813 ]); 8814 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8815 8816 // Start the destination VM 8817 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8818 dest_api_socket.push_str(".dest"); 8819 let mut dest_child = GuestCommand::new(&guest) 8820 .args(["--api-socket", &dest_api_socket]) 8821 .capture_output() 8822 .spawn() 8823 .unwrap(); 8824 8825 let r = std::panic::catch_unwind(|| { 8826 guest.wait_vm_boot(None).unwrap(); 8827 8828 // Make sure the source VM is functaionl 8829 // Check the number of vCPUs 8830 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8831 8832 // Check the guest RAM 8833 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8834 // Increase the guest RAM 8835 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 8836 thread::sleep(std::time::Duration::new(5, 0)); 8837 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8838 // Use balloon to remove RAM from the VM 8839 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 8840 thread::sleep(std::time::Duration::new(5, 0)); 8841 let total_memory = guest.get_total_memory().unwrap_or_default(); 8842 assert!(total_memory > 4_800_000); 8843 assert!(total_memory < 5_760_000); 8844 8845 // Check the guest virtio-devices, e.g. block, rng, console, and net 8846 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8847 8848 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8849 // to make sure that removing and adding back the virtio-net device does 8850 // not break the live-migration support for virtio-pci. 8851 #[cfg(target_arch = "x86_64")] 8852 { 8853 assert!(remote_command( 8854 &src_api_socket, 8855 "remove-device", 8856 Some(net_id), 8857 )); 8858 thread::sleep(std::time::Duration::new(10, 0)); 8859 8860 // Plug the virtio-net device again 8861 assert!(remote_command( 8862 &src_api_socket, 8863 "add-net", 8864 Some(net_params.as_str()), 8865 )); 8866 thread::sleep(std::time::Duration::new(10, 0)); 8867 } 8868 8869 // Start the live-migration 8870 let migration_socket = String::from( 8871 guest 8872 .tmp_dir 8873 .as_path() 8874 .join("live-migration.sock") 8875 .to_str() 8876 .unwrap(), 8877 ); 8878 8879 assert!( 8880 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8881 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8882 ); 8883 }); 8884 8885 // Check and report any errors occurred during the live-migration 8886 if r.is_err() { 8887 print_and_panic( 8888 src_child, 8889 dest_child, 8890 None, 8891 "Error occurred during live-migration", 8892 ); 8893 } 8894 8895 // Check the source vm has been terminated successful (give it '3s' to settle) 8896 thread::sleep(std::time::Duration::new(3, 0)); 8897 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8898 print_and_panic( 8899 src_child, 8900 dest_child, 8901 None, 8902 "source VM was not terminated successfully.", 8903 ); 8904 }; 8905 8906 // Post live-migration check to make sure the destination VM is funcational 8907 let r = std::panic::catch_unwind(|| { 8908 // Perform same checks to validate VM has been properly migrated 8909 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8910 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8911 8912 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8913 8914 // Perform checks on guest RAM using balloon 8915 let total_memory = guest.get_total_memory().unwrap_or_default(); 8916 assert!(total_memory > 4_800_000); 8917 assert!(total_memory < 5_760_000); 8918 // Deflate balloon to restore entire RAM to the VM 8919 resize_command(&dest_api_socket, None, None, Some(0), None); 8920 thread::sleep(std::time::Duration::new(5, 0)); 8921 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8922 // Decrease guest RAM with virtio-mem 8923 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 8924 thread::sleep(std::time::Duration::new(5, 0)); 8925 let total_memory = guest.get_total_memory().unwrap_or_default(); 8926 assert!(total_memory > 4_800_000); 8927 assert!(total_memory < 5_760_000); 8928 }); 8929 8930 // Clean-up the destination VM and make sure it terminated correctly 8931 let _ = dest_child.kill(); 8932 let dest_output = dest_child.wait_with_output().unwrap(); 8933 handle_child_output(r, &dest_output); 8934 8935 // Check the destination VM has the expected 'concole_text' from its output 8936 let r = std::panic::catch_unwind(|| { 8937 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8938 }); 8939 handle_child_output(r, &dest_output); 8940 } 8941 8942 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 8943 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8944 let guest = Guest::new(Box::new(focal)); 8945 let kernel_path = direct_kernel_boot_path(); 8946 let console_text = String::from("On a branch floating down river a cricket, singing."); 8947 let net_id = "net123"; 8948 let net_params = format!( 8949 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8950 net_id, guest.network.guest_mac, guest.network.host_ip 8951 ); 8952 8953 let memory_param: &[&str] = if local { 8954 &[ 8955 "--memory", 8956 "size=0,hotplug_method=virtio-mem,shared=on", 8957 "--memory-zone", 8958 "id=mem0,size=1G,hotplug_size=4G,shared=on", 8959 "id=mem1,size=1G,hotplug_size=4G,shared=on", 8960 "id=mem2,size=2G,hotplug_size=4G,shared=on", 8961 "--numa", 8962 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 8963 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 8964 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 8965 ] 8966 } else { 8967 &[ 8968 "--memory", 8969 "size=0,hotplug_method=virtio-mem", 8970 "--memory-zone", 8971 "id=mem0,size=1G,hotplug_size=4G", 8972 "id=mem1,size=1G,hotplug_size=4G", 8973 "id=mem2,size=2G,hotplug_size=4G", 8974 "--numa", 8975 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 8976 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 8977 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 8978 ] 8979 }; 8980 8981 let boot_vcpus = 6; 8982 let max_vcpus = 12; 8983 8984 let pmem_temp_file = TempFile::new().unwrap(); 8985 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8986 std::process::Command::new("mkfs.ext4") 8987 .arg(pmem_temp_file.as_path()) 8988 .output() 8989 .expect("Expect creating disk image to succeed"); 8990 let pmem_path = String::from("/dev/pmem0"); 8991 8992 // Start the source VM 8993 let src_vm_path = if !upgrade_test { 8994 clh_command("cloud-hypervisor") 8995 } else { 8996 cloud_hypervisor_release_path() 8997 }; 8998 let src_api_socket = temp_api_path(&guest.tmp_dir); 8999 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9000 src_vm_cmd 9001 .args([ 9002 "--cpus", 9003 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9004 ]) 9005 .args(memory_param) 9006 .args(["--kernel", kernel_path.to_str().unwrap()]) 9007 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9008 .default_disks() 9009 .args(["--net", net_params.as_str()]) 9010 .args(["--api-socket", &src_api_socket]) 9011 .args([ 9012 "--pmem", 9013 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9014 ]); 9015 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9016 9017 // Start the destination VM 9018 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9019 dest_api_socket.push_str(".dest"); 9020 let mut dest_child = GuestCommand::new(&guest) 9021 .args(["--api-socket", &dest_api_socket]) 9022 .capture_output() 9023 .spawn() 9024 .unwrap(); 9025 9026 let r = std::panic::catch_unwind(|| { 9027 guest.wait_vm_boot(None).unwrap(); 9028 9029 // Make sure the source VM is functaionl 9030 // Check the number of vCPUs 9031 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9032 9033 // Check the guest RAM 9034 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 9035 9036 // Check the guest virtio-devices, e.g. block, rng, console, and net 9037 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9038 9039 // Check the NUMA parameters are applied correctly and resize 9040 // each zone to test the case where we migrate a VM with the 9041 // virtio-mem regions being used. 9042 { 9043 guest.check_numa_common( 9044 Some(&[960_000, 960_000, 1_920_000]), 9045 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9046 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9047 ); 9048 9049 // AArch64 currently does not support hotplug, and therefore we only 9050 // test hotplug-related function on x86_64 here. 9051 #[cfg(target_arch = "x86_64")] 9052 { 9053 guest.enable_memory_hotplug(); 9054 9055 // Resize every memory zone and check each associated NUMA node 9056 // has been assigned the right amount of memory. 9057 resize_zone_command(&src_api_socket, "mem0", "2G"); 9058 resize_zone_command(&src_api_socket, "mem1", "2G"); 9059 resize_zone_command(&src_api_socket, "mem2", "3G"); 9060 thread::sleep(std::time::Duration::new(5, 0)); 9061 9062 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 9063 } 9064 } 9065 9066 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9067 // to make sure that removing and adding back the virtio-net device does 9068 // not break the live-migration support for virtio-pci. 9069 #[cfg(target_arch = "x86_64")] 9070 { 9071 assert!(remote_command( 9072 &src_api_socket, 9073 "remove-device", 9074 Some(net_id), 9075 )); 9076 thread::sleep(std::time::Duration::new(10, 0)); 9077 9078 // Plug the virtio-net device again 9079 assert!(remote_command( 9080 &src_api_socket, 9081 "add-net", 9082 Some(net_params.as_str()), 9083 )); 9084 thread::sleep(std::time::Duration::new(10, 0)); 9085 } 9086 9087 // Start the live-migration 9088 let migration_socket = String::from( 9089 guest 9090 .tmp_dir 9091 .as_path() 9092 .join("live-migration.sock") 9093 .to_str() 9094 .unwrap(), 9095 ); 9096 9097 assert!( 9098 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9099 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9100 ); 9101 }); 9102 9103 // Check and report any errors occurred during the live-migration 9104 if r.is_err() { 9105 print_and_panic( 9106 src_child, 9107 dest_child, 9108 None, 9109 "Error occurred during live-migration", 9110 ); 9111 } 9112 9113 // Check the source vm has been terminated successful (give it '3s' to settle) 9114 thread::sleep(std::time::Duration::new(3, 0)); 9115 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9116 print_and_panic( 9117 src_child, 9118 dest_child, 9119 None, 9120 "source VM was not terminated successfully.", 9121 ); 9122 }; 9123 9124 // Post live-migration check to make sure the destination VM is funcational 9125 let r = std::panic::catch_unwind(|| { 9126 // Perform same checks to validate VM has been properly migrated 9127 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9128 #[cfg(target_arch = "x86_64")] 9129 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 9130 #[cfg(target_arch = "aarch64")] 9131 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9132 9133 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9134 9135 // Perform NUMA related checks 9136 { 9137 #[cfg(target_arch = "aarch64")] 9138 { 9139 guest.check_numa_common( 9140 Some(&[960_000, 960_000, 1_920_000]), 9141 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9142 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9143 ); 9144 } 9145 9146 // AArch64 currently does not support hotplug, and therefore we only 9147 // test hotplug-related function on x86_64 here. 9148 #[cfg(target_arch = "x86_64")] 9149 { 9150 guest.check_numa_common( 9151 Some(&[1_920_000, 1_920_000, 2_880_000]), 9152 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9153 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9154 ); 9155 9156 guest.enable_memory_hotplug(); 9157 9158 // Resize every memory zone and check each associated NUMA node 9159 // has been assigned the right amount of memory. 9160 resize_zone_command(&dest_api_socket, "mem0", "4G"); 9161 resize_zone_command(&dest_api_socket, "mem1", "4G"); 9162 resize_zone_command(&dest_api_socket, "mem2", "4G"); 9163 // Resize to the maximum amount of CPUs and check each NUMA 9164 // node has been assigned the right CPUs set. 9165 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 9166 thread::sleep(std::time::Duration::new(5, 0)); 9167 9168 guest.check_numa_common( 9169 Some(&[3_840_000, 3_840_000, 3_840_000]), 9170 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 9171 None, 9172 ); 9173 } 9174 } 9175 }); 9176 9177 // Clean-up the destination VM and make sure it terminated correctly 9178 let _ = dest_child.kill(); 9179 let dest_output = dest_child.wait_with_output().unwrap(); 9180 handle_child_output(r, &dest_output); 9181 9182 // Check the destination VM has the expected 'concole_text' from its output 9183 let r = std::panic::catch_unwind(|| { 9184 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9185 }); 9186 handle_child_output(r, &dest_output); 9187 } 9188 9189 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 9190 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9191 let guest = Guest::new(Box::new(focal)); 9192 let kernel_path = direct_kernel_boot_path(); 9193 let console_text = String::from("On a branch floating down river a cricket, singing."); 9194 let net_id = "net123"; 9195 let net_params = format!( 9196 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9197 net_id, guest.network.guest_mac, guest.network.host_ip 9198 ); 9199 9200 let memory_param: &[&str] = if local { 9201 &["--memory", "size=4G,shared=on"] 9202 } else { 9203 &["--memory", "size=4G"] 9204 }; 9205 9206 let boot_vcpus = 2; 9207 let max_vcpus = 4; 9208 9209 let pmem_temp_file = TempFile::new().unwrap(); 9210 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9211 std::process::Command::new("mkfs.ext4") 9212 .arg(pmem_temp_file.as_path()) 9213 .output() 9214 .expect("Expect creating disk image to succeed"); 9215 let pmem_path = String::from("/dev/pmem0"); 9216 9217 // Start the source VM 9218 let src_vm_path = if !upgrade_test { 9219 clh_command("cloud-hypervisor") 9220 } else { 9221 cloud_hypervisor_release_path() 9222 }; 9223 let src_api_socket = temp_api_path(&guest.tmp_dir); 9224 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9225 src_vm_cmd 9226 .args([ 9227 "--cpus", 9228 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9229 ]) 9230 .args(memory_param) 9231 .args(["--kernel", kernel_path.to_str().unwrap()]) 9232 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9233 .default_disks() 9234 .args(["--net", net_params.as_str()]) 9235 .args(["--api-socket", &src_api_socket]) 9236 .args([ 9237 "--pmem", 9238 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9239 ]) 9240 .args(["--watchdog"]); 9241 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9242 9243 // Start the destination VM 9244 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9245 dest_api_socket.push_str(".dest"); 9246 let mut dest_child = GuestCommand::new(&guest) 9247 .args(["--api-socket", &dest_api_socket]) 9248 .capture_output() 9249 .spawn() 9250 .unwrap(); 9251 9252 let r = std::panic::catch_unwind(|| { 9253 guest.wait_vm_boot(None).unwrap(); 9254 9255 // Make sure the source VM is functaionl 9256 // Check the number of vCPUs 9257 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9258 // Check the guest RAM 9259 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9260 // Check the guest virtio-devices, e.g. block, rng, console, and net 9261 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9262 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9263 // to make sure that removing and adding back the virtio-net device does 9264 // not break the live-migration support for virtio-pci. 9265 #[cfg(target_arch = "x86_64")] 9266 { 9267 assert!(remote_command( 9268 &src_api_socket, 9269 "remove-device", 9270 Some(net_id), 9271 )); 9272 thread::sleep(std::time::Duration::new(10, 0)); 9273 9274 // Plug the virtio-net device again 9275 assert!(remote_command( 9276 &src_api_socket, 9277 "add-net", 9278 Some(net_params.as_str()), 9279 )); 9280 thread::sleep(std::time::Duration::new(10, 0)); 9281 } 9282 9283 // Enable watchdog and ensure its functional 9284 let mut expected_reboot_count = 1; 9285 // Enable the watchdog with a 15s timeout 9286 enable_guest_watchdog(&guest, 15); 9287 // Reboot and check that systemd has activated the watchdog 9288 guest.ssh_command("sudo reboot").unwrap(); 9289 guest.wait_vm_boot(None).unwrap(); 9290 expected_reboot_count += 1; 9291 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9292 assert_eq!( 9293 guest 9294 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 9295 .unwrap() 9296 .trim() 9297 .parse::<u32>() 9298 .unwrap_or_default(), 9299 2 9300 ); 9301 // Allow some normal time to elapse to check we don't get spurious reboots 9302 thread::sleep(std::time::Duration::new(40, 0)); 9303 // Check no reboot 9304 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9305 9306 // Start the live-migration 9307 let migration_socket = String::from( 9308 guest 9309 .tmp_dir 9310 .as_path() 9311 .join("live-migration.sock") 9312 .to_str() 9313 .unwrap(), 9314 ); 9315 9316 assert!( 9317 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9318 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9319 ); 9320 }); 9321 9322 // Check and report any errors occurred during the live-migration 9323 if r.is_err() { 9324 print_and_panic( 9325 src_child, 9326 dest_child, 9327 None, 9328 "Error occurred during live-migration", 9329 ); 9330 } 9331 9332 // Check the source vm has been terminated successful (give it '3s' to settle) 9333 thread::sleep(std::time::Duration::new(3, 0)); 9334 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9335 print_and_panic( 9336 src_child, 9337 dest_child, 9338 None, 9339 "source VM was not terminated successfully.", 9340 ); 9341 }; 9342 9343 // Post live-migration check to make sure the destination VM is funcational 9344 let r = std::panic::catch_unwind(|| { 9345 // Perform same checks to validate VM has been properly migrated 9346 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9347 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9348 9349 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9350 9351 // Perform checks on watchdog 9352 let mut expected_reboot_count = 2; 9353 9354 // Allow some normal time to elapse to check we don't get spurious reboots 9355 thread::sleep(std::time::Duration::new(40, 0)); 9356 // Check no reboot 9357 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9358 9359 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 9360 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 9361 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 9362 guest.wait_vm_boot(Some(50)).unwrap(); 9363 // Check a reboot is triggered by the watchdog 9364 expected_reboot_count += 1; 9365 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9366 9367 #[cfg(target_arch = "x86_64")] 9368 { 9369 // Now pause the VM and remain offline for 30s 9370 assert!(remote_command(&dest_api_socket, "pause", None)); 9371 thread::sleep(std::time::Duration::new(30, 0)); 9372 assert!(remote_command(&dest_api_socket, "resume", None)); 9373 9374 // Check no reboot 9375 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9376 } 9377 }); 9378 9379 // Clean-up the destination VM and make sure it terminated correctly 9380 let _ = dest_child.kill(); 9381 let dest_output = dest_child.wait_with_output().unwrap(); 9382 handle_child_output(r, &dest_output); 9383 9384 // Check the destination VM has the expected 'concole_text' from its output 9385 let r = std::panic::catch_unwind(|| { 9386 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9387 }); 9388 handle_child_output(r, &dest_output); 9389 } 9390 9391 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 9392 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9393 let ovs_guest = Guest::new(Box::new(ovs_focal)); 9394 9395 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9396 let migration_guest = Guest::new(Box::new(migration_focal)); 9397 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 9398 9399 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 9400 let (mut ovs_child, mut src_child) = 9401 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 9402 9403 // Start the destination VM 9404 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 9405 dest_api_socket.push_str(".dest"); 9406 let mut dest_child = GuestCommand::new(&migration_guest) 9407 .args(["--api-socket", &dest_api_socket]) 9408 .capture_output() 9409 .spawn() 9410 .unwrap(); 9411 9412 let r = std::panic::catch_unwind(|| { 9413 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 9414 thread::sleep(std::time::Duration::new(1, 0)); 9415 9416 // Start the live-migration 9417 let migration_socket = String::from( 9418 migration_guest 9419 .tmp_dir 9420 .as_path() 9421 .join("live-migration.sock") 9422 .to_str() 9423 .unwrap(), 9424 ); 9425 9426 assert!( 9427 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9428 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9429 ); 9430 }); 9431 9432 // Check and report any errors occurred during the live-migration 9433 if r.is_err() { 9434 print_and_panic( 9435 src_child, 9436 dest_child, 9437 Some(ovs_child), 9438 "Error occurred during live-migration", 9439 ); 9440 } 9441 9442 // Check the source vm has been terminated successful (give it '3s' to settle) 9443 thread::sleep(std::time::Duration::new(3, 0)); 9444 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9445 print_and_panic( 9446 src_child, 9447 dest_child, 9448 Some(ovs_child), 9449 "source VM was not terminated successfully.", 9450 ); 9451 }; 9452 9453 // Post live-migration check to make sure the destination VM is funcational 9454 let r = std::panic::catch_unwind(|| { 9455 // Perform same checks to validate VM has been properly migrated 9456 // Spawn a new netcat listener in the OVS VM 9457 let guest_ip = ovs_guest.network.guest_ip.clone(); 9458 thread::spawn(move || { 9459 ssh_command_ip( 9460 "nc -l 12345", 9461 &guest_ip, 9462 DEFAULT_SSH_RETRIES, 9463 DEFAULT_SSH_TIMEOUT, 9464 ) 9465 .unwrap(); 9466 }); 9467 9468 // Wait for the server to be listening 9469 thread::sleep(std::time::Duration::new(5, 0)); 9470 9471 // And check the connection is still functional after live-migration 9472 migration_guest 9473 .ssh_command("nc -vz 172.100.0.1 12345") 9474 .unwrap(); 9475 }); 9476 9477 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 9478 let _ = dest_child.kill(); 9479 let _ = ovs_child.kill(); 9480 let dest_output = dest_child.wait_with_output().unwrap(); 9481 let ovs_output = ovs_child.wait_with_output().unwrap(); 9482 9483 cleanup_ovs_dpdk(); 9484 9485 handle_child_output(r, &dest_output); 9486 handle_child_output(Ok(()), &ovs_output); 9487 } 9488 9489 mod live_migration_parallel { 9490 use super::*; 9491 #[test] 9492 fn test_live_migration_basic() { 9493 _test_live_migration(false, false) 9494 } 9495 9496 #[test] 9497 fn test_live_migration_local() { 9498 _test_live_migration(false, true) 9499 } 9500 9501 #[test] 9502 #[cfg(not(feature = "mshv"))] 9503 fn test_live_migration_numa() { 9504 _test_live_migration_numa(false, false) 9505 } 9506 9507 #[test] 9508 #[cfg(not(feature = "mshv"))] 9509 fn test_live_migration_numa_local() { 9510 _test_live_migration_numa(false, true) 9511 } 9512 9513 #[test] 9514 fn test_live_migration_watchdog() { 9515 _test_live_migration_watchdog(false, false) 9516 } 9517 9518 #[test] 9519 fn test_live_migration_watchdog_local() { 9520 _test_live_migration_watchdog(false, true) 9521 } 9522 9523 #[test] 9524 fn test_live_migration_balloon() { 9525 _test_live_migration_balloon(false, false) 9526 } 9527 9528 #[test] 9529 fn test_live_migration_balloon_local() { 9530 _test_live_migration_balloon(false, true) 9531 } 9532 9533 #[test] 9534 #[ignore = "See #5791"] 9535 fn test_live_upgrade_basic() { 9536 _test_live_migration(true, false) 9537 } 9538 9539 #[test] 9540 #[ignore = "See #5791"] 9541 fn test_live_upgrade_local() { 9542 _test_live_migration(true, true) 9543 } 9544 9545 #[test] 9546 #[cfg(not(feature = "mshv"))] 9547 #[ignore = "See #5791"] 9548 fn test_live_upgrade_numa() { 9549 _test_live_migration_numa(true, false) 9550 } 9551 9552 #[test] 9553 #[cfg(not(feature = "mshv"))] 9554 #[ignore = "See #5791"] 9555 fn test_live_upgrade_numa_local() { 9556 _test_live_migration_numa(true, true) 9557 } 9558 9559 #[test] 9560 #[ignore = "See #5791"] 9561 fn test_live_upgrade_watchdog() { 9562 _test_live_migration_watchdog(true, false) 9563 } 9564 9565 #[test] 9566 #[ignore = "See #5791"] 9567 fn test_live_upgrade_watchdog_local() { 9568 _test_live_migration_watchdog(true, true) 9569 } 9570 9571 #[test] 9572 #[ignore = "See #5791"] 9573 fn test_live_upgrade_balloon() { 9574 _test_live_migration_balloon(true, false) 9575 } 9576 9577 #[test] 9578 #[ignore = "See #5791"] 9579 fn test_live_upgrade_balloon_local() { 9580 _test_live_migration_balloon(true, true) 9581 } 9582 } 9583 9584 mod live_migration_sequential { 9585 #[cfg(target_arch = "x86_64")] 9586 #[cfg(not(feature = "mshv"))] 9587 use super::*; 9588 9589 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 9590 #[test] 9591 #[cfg(target_arch = "x86_64")] 9592 #[cfg(not(feature = "mshv"))] 9593 fn test_live_migration_ovs_dpdk() { 9594 _test_live_migration_ovs_dpdk(false, false); 9595 } 9596 9597 #[test] 9598 #[cfg(target_arch = "x86_64")] 9599 #[cfg(not(feature = "mshv"))] 9600 fn test_live_migration_ovs_dpdk_local() { 9601 _test_live_migration_ovs_dpdk(false, true); 9602 } 9603 9604 #[test] 9605 #[cfg(target_arch = "x86_64")] 9606 #[cfg(not(feature = "mshv"))] 9607 #[ignore = "See #5791"] 9608 fn test_live_upgrade_ovs_dpdk() { 9609 _test_live_migration_ovs_dpdk(true, false); 9610 } 9611 9612 #[test] 9613 #[cfg(target_arch = "x86_64")] 9614 #[cfg(not(feature = "mshv"))] 9615 #[ignore = "See #5791"] 9616 fn test_live_upgrade_ovs_dpdk_local() { 9617 _test_live_migration_ovs_dpdk(true, true); 9618 } 9619 } 9620 } 9621 9622 #[cfg(target_arch = "aarch64")] 9623 mod aarch64_acpi { 9624 use crate::*; 9625 9626 #[test] 9627 fn test_simple_launch_acpi() { 9628 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9629 9630 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 9631 let guest = Guest::new(disk_config); 9632 9633 let mut child = GuestCommand::new(&guest) 9634 .args(["--cpus", "boot=1"]) 9635 .args(["--memory", "size=512M"]) 9636 .args(["--kernel", edk2_path().to_str().unwrap()]) 9637 .default_disks() 9638 .default_net() 9639 .args(["--serial", "tty", "--console", "off"]) 9640 .capture_output() 9641 .spawn() 9642 .unwrap(); 9643 9644 let r = std::panic::catch_unwind(|| { 9645 guest.wait_vm_boot(Some(120)).unwrap(); 9646 9647 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 9648 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 9649 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 9650 }); 9651 9652 let _ = child.kill(); 9653 let output = child.wait_with_output().unwrap(); 9654 9655 handle_child_output(r, &output); 9656 }); 9657 } 9658 9659 #[test] 9660 fn test_guest_numa_nodes_acpi() { 9661 _test_guest_numa_nodes(true); 9662 } 9663 9664 #[test] 9665 fn test_cpu_topology_421_acpi() { 9666 test_cpu_topology(4, 2, 1, true); 9667 } 9668 9669 #[test] 9670 fn test_cpu_topology_142_acpi() { 9671 test_cpu_topology(1, 4, 2, true); 9672 } 9673 9674 #[test] 9675 fn test_cpu_topology_262_acpi() { 9676 test_cpu_topology(2, 6, 2, true); 9677 } 9678 9679 #[test] 9680 fn test_power_button_acpi() { 9681 _test_power_button(true); 9682 } 9683 9684 #[test] 9685 fn test_virtio_iommu() { 9686 _test_virtio_iommu(true) 9687 } 9688 } 9689 9690 mod rate_limiter { 9691 use super::*; 9692 9693 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 9694 // compared to given 'limit' rate. 9695 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 9696 let upper_limit = limit * (1_f64 + difference); 9697 let lower_limit = limit * (1_f64 - difference); 9698 9699 if measured > lower_limit && measured < upper_limit { 9700 return true; 9701 } 9702 9703 eprintln!( 9704 "\n\n==== Start 'check_rate_limit' failed ==== \ 9705 \n\nmeasured={measured}, , lower_limit={lower_limit}, upper_limit={upper_limit} \ 9706 \n\n==== End 'check_rate_limit' failed ====\n\n" 9707 ); 9708 9709 false 9710 } 9711 9712 fn _test_rate_limiter_net(rx: bool) { 9713 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9714 let guest = Guest::new(Box::new(focal)); 9715 9716 let test_timeout = 10; 9717 let num_queues = 2; 9718 let queue_size = 256; 9719 let bw_size = 10485760_u64; // bytes 9720 let bw_refill_time = 100; // ms 9721 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 9722 9723 let net_params = format!( 9724 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 9725 guest.network.guest_mac, 9726 guest.network.host_ip, 9727 num_queues, 9728 queue_size, 9729 bw_size, 9730 bw_refill_time, 9731 ); 9732 9733 let mut child = GuestCommand::new(&guest) 9734 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 9735 .args(["--memory", "size=4G"]) 9736 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9737 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9738 .default_disks() 9739 .args(["--net", net_params.as_str()]) 9740 .capture_output() 9741 .spawn() 9742 .unwrap(); 9743 9744 let r = std::panic::catch_unwind(|| { 9745 guest.wait_vm_boot(None).unwrap(); 9746 let measured_bps = 9747 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, true) 9748 .unwrap(); 9749 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 9750 }); 9751 9752 let _ = child.kill(); 9753 let output = child.wait_with_output().unwrap(); 9754 handle_child_output(r, &output); 9755 } 9756 9757 #[test] 9758 fn test_rate_limiter_net_rx() { 9759 _test_rate_limiter_net(true); 9760 } 9761 9762 #[test] 9763 fn test_rate_limiter_net_tx() { 9764 _test_rate_limiter_net(false); 9765 } 9766 9767 fn _test_rate_limiter_block(bandwidth: bool) { 9768 let test_timeout = 10; 9769 let num_queues = 1; 9770 let fio_ops = FioOps::RandRW; 9771 9772 let bw_size = if bandwidth { 9773 10485760_u64 // bytes 9774 } else { 9775 100_u64 // I/O 9776 }; 9777 let bw_refill_time = 100; // ms 9778 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 9779 9780 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9781 let guest = Guest::new(Box::new(focal)); 9782 let api_socket = temp_api_path(&guest.tmp_dir); 9783 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 9784 let blk_rate_limiter_test_img = 9785 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 9786 9787 // Create the test block image 9788 assert!(exec_host_command_output(&format!( 9789 "dd if=/dev/zero of={blk_rate_limiter_test_img} bs=1M count=1024" 9790 )) 9791 .status 9792 .success()); 9793 9794 let test_blk_params = if bandwidth { 9795 format!( 9796 "path={blk_rate_limiter_test_img},bw_size={bw_size},bw_refill_time={bw_refill_time}" 9797 ) 9798 } else { 9799 format!( 9800 "path={blk_rate_limiter_test_img},ops_size={bw_size},ops_refill_time={bw_refill_time}" 9801 ) 9802 }; 9803 9804 let mut child = GuestCommand::new(&guest) 9805 .args(["--cpus", &format!("boot={num_queues}")]) 9806 .args(["--memory", "size=4G"]) 9807 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9808 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9809 .args([ 9810 "--disk", 9811 format!( 9812 "path={}", 9813 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 9814 ) 9815 .as_str(), 9816 format!( 9817 "path={}", 9818 guest.disk_config.disk(DiskType::CloudInit).unwrap() 9819 ) 9820 .as_str(), 9821 test_blk_params.as_str(), 9822 ]) 9823 .default_net() 9824 .args(["--api-socket", &api_socket]) 9825 .capture_output() 9826 .spawn() 9827 .unwrap(); 9828 9829 let r = std::panic::catch_unwind(|| { 9830 guest.wait_vm_boot(None).unwrap(); 9831 9832 let fio_command = format!( 9833 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 9834 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 9835 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 9836 ); 9837 let output = guest.ssh_command(&fio_command).unwrap(); 9838 9839 // Parse fio output 9840 let measured_rate = if bandwidth { 9841 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 9842 } else { 9843 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 9844 }; 9845 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 9846 }); 9847 9848 let _ = child.kill(); 9849 let output = child.wait_with_output().unwrap(); 9850 handle_child_output(r, &output); 9851 } 9852 9853 #[test] 9854 fn test_rate_limiter_block_bandwidth() { 9855 _test_rate_limiter_block(true) 9856 } 9857 9858 #[test] 9859 fn test_rate_limiter_block_iops() { 9860 _test_rate_limiter_block(false) 9861 } 9862 } 9863