1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 use std::sync::Mutex; 28 use std::thread; 29 use test_infra::*; 30 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 31 use wait_timeout::ChildExt; 32 33 // Constant taken from the VMM crate. 34 const MAX_NUM_PCI_SEGMENTS: u16 = 96; 35 36 #[cfg(target_arch = "x86_64")] 37 mod x86_64 { 38 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 39 pub const JAMMY_NVIDIA_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-nvidia.raw"; 40 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 41 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 42 "focal-server-cloudimg-amd64-custom-20210609-0-backing.qcow2"; 43 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 44 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 45 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20230119-0.raw"; 46 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2022-amd64-2.raw"; 47 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 48 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 49 } 50 51 #[cfg(target_arch = "x86_64")] 52 use x86_64::*; 53 54 #[cfg(target_arch = "aarch64")] 55 mod aarch64 { 56 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 57 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 58 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 59 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 60 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 61 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 62 "focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2"; 63 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 64 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 65 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 66 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 67 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 68 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 69 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 70 } 71 72 #[cfg(target_arch = "aarch64")] 73 use aarch64::*; 74 75 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 76 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 77 78 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 79 80 // This enum exists to make it more convenient to 81 // implement test for both D-Bus and REST APIs. 82 enum TargetApi { 83 // API socket 84 HttpApi(String), 85 // well known service name, object path 86 DBusApi(String, String), 87 } 88 89 impl TargetApi { 90 fn new_http_api(tmp_dir: &TempDir) -> Self { 91 Self::HttpApi(temp_api_path(tmp_dir)) 92 } 93 94 fn new_dbus_api(tmp_dir: &TempDir) -> Self { 95 // `tmp_dir` is in the form of "/tmp/chXXXXXX" 96 // and we take the `chXXXXXX` part as a unique identifier for the guest 97 let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); 98 99 Self::DBusApi( 100 format!("org.cloudhypervisor.{id}"), 101 format!("/org/cloudhypervisor/{id}"), 102 ) 103 } 104 105 fn guest_args(&self) -> Vec<String> { 106 match self { 107 TargetApi::HttpApi(api_socket) => { 108 vec![format!("--api-socket={}", api_socket.as_str())] 109 } 110 TargetApi::DBusApi(service_name, object_path) => { 111 vec![ 112 format!("--dbus-service-name={}", service_name.as_str()), 113 format!("--dbus-object-path={}", object_path.as_str()), 114 ] 115 } 116 } 117 } 118 119 fn remote_args(&self) -> Vec<String> { 120 // `guest_args` and `remote_args` are consistent with each other 121 self.guest_args() 122 } 123 124 fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { 125 let mut cmd = Command::new(clh_command("ch-remote")); 126 cmd.args(self.remote_args()); 127 cmd.arg(command); 128 129 if let Some(arg) = arg { 130 cmd.arg(arg); 131 } 132 133 let output = cmd.output().unwrap(); 134 if output.status.success() { 135 true 136 } else { 137 eprintln!("Error running ch-remote command: {:?}", &cmd); 138 let stderr = String::from_utf8_lossy(&output.stderr); 139 eprintln!("stderr: {stderr}"); 140 false 141 } 142 } 143 } 144 145 // Start cloud-hypervisor with no VM parameters, only the API server running. 146 // From the API: Create a VM, boot it and check that it looks as expected. 147 fn _test_api_create_boot(target_api: TargetApi, guest: Guest) { 148 let mut child = GuestCommand::new(&guest) 149 .args(target_api.guest_args()) 150 .capture_output() 151 .spawn() 152 .unwrap(); 153 154 thread::sleep(std::time::Duration::new(1, 0)); 155 156 // Verify API server is running 157 assert!(target_api.remote_command("ping", None)); 158 159 // Create the VM first 160 let cpu_count: u8 = 4; 161 let request_body = guest.api_create_body( 162 cpu_count, 163 direct_kernel_boot_path().to_str().unwrap(), 164 DIRECT_KERNEL_BOOT_CMDLINE, 165 ); 166 167 let temp_config_path = guest.tmp_dir.as_path().join("config"); 168 std::fs::write(&temp_config_path, request_body).unwrap(); 169 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 170 171 assert!(target_api.remote_command("create", Some(create_config),)); 172 173 // Then boot it 174 assert!(target_api.remote_command("boot", None)); 175 thread::sleep(std::time::Duration::new(20, 0)); 176 177 let r = std::panic::catch_unwind(|| { 178 // Check that the VM booted as expected 179 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 180 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 181 }); 182 183 let _ = child.kill(); 184 let output = child.wait_with_output().unwrap(); 185 186 handle_child_output(r, &output); 187 } 188 189 // Start cloud-hypervisor with no VM parameters, only the API server running. 190 // From the API: Create a VM, boot it and check it can be shutdown and then 191 // booted again 192 fn _test_api_shutdown(target_api: TargetApi, guest: Guest) { 193 let mut child = GuestCommand::new(&guest) 194 .args(target_api.guest_args()) 195 .capture_output() 196 .spawn() 197 .unwrap(); 198 199 thread::sleep(std::time::Duration::new(1, 0)); 200 201 // Verify API server is running 202 assert!(target_api.remote_command("ping", None)); 203 204 // Create the VM first 205 let cpu_count: u8 = 4; 206 let request_body = guest.api_create_body( 207 cpu_count, 208 direct_kernel_boot_path().to_str().unwrap(), 209 DIRECT_KERNEL_BOOT_CMDLINE, 210 ); 211 212 let temp_config_path = guest.tmp_dir.as_path().join("config"); 213 std::fs::write(&temp_config_path, request_body).unwrap(); 214 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 215 216 let r = std::panic::catch_unwind(|| { 217 assert!(target_api.remote_command("create", Some(create_config))); 218 219 // Then boot it 220 assert!(target_api.remote_command("boot", None)); 221 222 guest.wait_vm_boot(None).unwrap(); 223 224 // Check that the VM booted as expected 225 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 226 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 227 228 // Sync and shutdown without powering off to prevent filesystem 229 // corruption. 230 guest.ssh_command("sync").unwrap(); 231 guest.ssh_command("sudo shutdown -H now").unwrap(); 232 233 // Wait for the guest to be fully shutdown 234 thread::sleep(std::time::Duration::new(20, 0)); 235 236 // Then shut it down 237 assert!(target_api.remote_command("shutdown", None)); 238 239 // Then boot it again 240 assert!(target_api.remote_command("boot", None)); 241 242 guest.wait_vm_boot(None).unwrap(); 243 244 // Check that the VM booted as expected 245 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 246 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 247 }); 248 249 let _ = child.kill(); 250 let output = child.wait_with_output().unwrap(); 251 252 handle_child_output(r, &output); 253 } 254 255 // Start cloud-hypervisor with no VM parameters, only the API server running. 256 // From the API: Create a VM, boot it and check it can be deleted and then recreated 257 // booted again. 258 fn _test_api_delete(target_api: TargetApi, guest: Guest) { 259 let mut child = GuestCommand::new(&guest) 260 .args(target_api.guest_args()) 261 .capture_output() 262 .spawn() 263 .unwrap(); 264 265 thread::sleep(std::time::Duration::new(1, 0)); 266 267 // Verify API server is running 268 assert!(target_api.remote_command("ping", None)); 269 270 // Create the VM first 271 let cpu_count: u8 = 4; 272 let request_body = guest.api_create_body( 273 cpu_count, 274 direct_kernel_boot_path().to_str().unwrap(), 275 DIRECT_KERNEL_BOOT_CMDLINE, 276 ); 277 let temp_config_path = guest.tmp_dir.as_path().join("config"); 278 std::fs::write(&temp_config_path, request_body).unwrap(); 279 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 280 281 let r = std::panic::catch_unwind(|| { 282 assert!(target_api.remote_command("create", Some(create_config))); 283 284 // Then boot it 285 assert!(target_api.remote_command("boot", None)); 286 287 guest.wait_vm_boot(None).unwrap(); 288 289 // Check that the VM booted as expected 290 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 291 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 292 293 // Sync and shutdown without powering off to prevent filesystem 294 // corruption. 295 guest.ssh_command("sync").unwrap(); 296 guest.ssh_command("sudo shutdown -H now").unwrap(); 297 298 // Wait for the guest to be fully shutdown 299 thread::sleep(std::time::Duration::new(20, 0)); 300 301 // Then delete it 302 assert!(target_api.remote_command("delete", None)); 303 304 assert!(target_api.remote_command("create", Some(create_config))); 305 306 // Then boot it again 307 assert!(target_api.remote_command("boot", None)); 308 309 guest.wait_vm_boot(None).unwrap(); 310 311 // Check that the VM booted as expected 312 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 313 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 314 }); 315 316 let _ = child.kill(); 317 let output = child.wait_with_output().unwrap(); 318 319 handle_child_output(r, &output); 320 } 321 322 // Start cloud-hypervisor with no VM parameters, only the API server running. 323 // From the API: Create a VM, boot it and check that it looks as expected. 324 // Then we pause the VM, check that it's no longer available. 325 // Finally we resume the VM and check that it's available. 326 fn _test_api_pause_resume(target_api: TargetApi, guest: Guest) { 327 let mut child = GuestCommand::new(&guest) 328 .args(target_api.guest_args()) 329 .capture_output() 330 .spawn() 331 .unwrap(); 332 333 thread::sleep(std::time::Duration::new(1, 0)); 334 335 // Verify API server is running 336 assert!(target_api.remote_command("ping", None)); 337 338 // Create the VM first 339 let cpu_count: u8 = 4; 340 let request_body = guest.api_create_body( 341 cpu_count, 342 direct_kernel_boot_path().to_str().unwrap(), 343 DIRECT_KERNEL_BOOT_CMDLINE, 344 ); 345 346 let temp_config_path = guest.tmp_dir.as_path().join("config"); 347 std::fs::write(&temp_config_path, request_body).unwrap(); 348 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 349 350 assert!(target_api.remote_command("create", Some(create_config))); 351 352 // Then boot it 353 assert!(target_api.remote_command("boot", None)); 354 thread::sleep(std::time::Duration::new(20, 0)); 355 356 let r = std::panic::catch_unwind(|| { 357 // Check that the VM booted as expected 358 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 359 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 360 361 // We now pause the VM 362 assert!(target_api.remote_command("pause", None)); 363 364 // Check pausing again fails 365 assert!(!target_api.remote_command("pause", None)); 366 367 thread::sleep(std::time::Duration::new(2, 0)); 368 369 // SSH into the VM should fail 370 assert!(ssh_command_ip( 371 "grep -c processor /proc/cpuinfo", 372 &guest.network.guest_ip, 373 2, 374 5 375 ) 376 .is_err()); 377 378 // Resume the VM 379 assert!(target_api.remote_command("resume", None)); 380 381 // Check resuming again fails 382 assert!(!target_api.remote_command("resume", None)); 383 384 thread::sleep(std::time::Duration::new(2, 0)); 385 386 // Now we should be able to SSH back in and get the right number of CPUs 387 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 388 }); 389 390 let _ = child.kill(); 391 let output = child.wait_with_output().unwrap(); 392 393 handle_child_output(r, &output); 394 } 395 396 fn _test_pty_interaction(pty_path: PathBuf) { 397 let mut cf = std::fs::OpenOptions::new() 398 .write(true) 399 .read(true) 400 .open(pty_path) 401 .unwrap(); 402 403 // Some dumb sleeps but we don't want to write 404 // before the console is up and we don't want 405 // to try and write the next line before the 406 // login process is ready. 407 thread::sleep(std::time::Duration::new(5, 0)); 408 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 409 thread::sleep(std::time::Duration::new(2, 0)); 410 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 411 thread::sleep(std::time::Duration::new(2, 0)); 412 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 413 thread::sleep(std::time::Duration::new(2, 0)); 414 415 // read pty and ensure they have a login shell 416 // some fairly hacky workarounds to avoid looping 417 // forever in case the channel is blocked getting output 418 let ptyc = pty_read(cf); 419 let mut empty = 0; 420 let mut prev = String::new(); 421 loop { 422 thread::sleep(std::time::Duration::new(2, 0)); 423 match ptyc.try_recv() { 424 Ok(line) => { 425 empty = 0; 426 prev = prev + &line; 427 if prev.contains("test_pty_console") { 428 break; 429 } 430 } 431 Err(mpsc::TryRecvError::Empty) => { 432 empty += 1; 433 assert!(empty <= 5, "No login on pty"); 434 } 435 _ => { 436 panic!("No login on pty") 437 } 438 } 439 } 440 } 441 442 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 443 let mut workload_path = dirs::home_dir().unwrap(); 444 workload_path.push("workloads"); 445 446 let mut virtiofsd_path = workload_path; 447 virtiofsd_path.push("virtiofsd"); 448 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 449 450 let virtiofsd_socket_path = 451 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 452 453 // Start the daemon 454 let child = Command::new(virtiofsd_path.as_str()) 455 .args(["--shared-dir", shared_dir]) 456 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 457 .args(["--cache", "never"]) 458 .spawn() 459 .unwrap(); 460 461 thread::sleep(std::time::Duration::new(10, 0)); 462 463 (child, virtiofsd_socket_path) 464 } 465 466 fn prepare_vubd( 467 tmp_dir: &TempDir, 468 blk_img: &str, 469 num_queues: usize, 470 rdonly: bool, 471 direct: bool, 472 ) -> (std::process::Child, String) { 473 let mut workload_path = dirs::home_dir().unwrap(); 474 workload_path.push("workloads"); 475 476 let mut blk_file_path = workload_path; 477 blk_file_path.push(blk_img); 478 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 479 480 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 481 482 // Start the daemon 483 let child = Command::new(clh_command("vhost_user_block")) 484 .args([ 485 "--block-backend", 486 format!( 487 "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" 488 ) 489 .as_str(), 490 ]) 491 .spawn() 492 .unwrap(); 493 494 thread::sleep(std::time::Duration::new(10, 0)); 495 496 (child, vubd_socket_path) 497 } 498 499 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 500 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 501 } 502 503 fn temp_api_path(tmp_dir: &TempDir) -> String { 504 String::from( 505 tmp_dir 506 .as_path() 507 .join("cloud-hypervisor.sock") 508 .to_str() 509 .unwrap(), 510 ) 511 } 512 513 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 514 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 515 } 516 517 // Creates the directory and returns the path. 518 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 519 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 520 std::fs::create_dir(&snapshot_dir).unwrap(); 521 snapshot_dir 522 } 523 524 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 525 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 526 vmcore_file 527 } 528 529 // Creates the path for direct kernel boot and return the path. 530 // For x86_64, this function returns the vmlinux kernel path. 531 // For AArch64, this function returns the PE kernel path. 532 fn direct_kernel_boot_path() -> PathBuf { 533 let mut workload_path = dirs::home_dir().unwrap(); 534 workload_path.push("workloads"); 535 536 let mut kernel_path = workload_path; 537 #[cfg(target_arch = "x86_64")] 538 kernel_path.push("vmlinux"); 539 #[cfg(target_arch = "aarch64")] 540 kernel_path.push("Image"); 541 542 kernel_path 543 } 544 545 fn edk2_path() -> PathBuf { 546 let mut workload_path = dirs::home_dir().unwrap(); 547 workload_path.push("workloads"); 548 let mut edk2_path = workload_path; 549 edk2_path.push(OVMF_NAME); 550 551 edk2_path 552 } 553 554 fn cloud_hypervisor_release_path() -> String { 555 let mut workload_path = dirs::home_dir().unwrap(); 556 workload_path.push("workloads"); 557 558 let mut ch_release_path = workload_path; 559 #[cfg(target_arch = "x86_64")] 560 ch_release_path.push("cloud-hypervisor-static"); 561 #[cfg(target_arch = "aarch64")] 562 ch_release_path.push("cloud-hypervisor-static-aarch64"); 563 564 ch_release_path.into_os_string().into_string().unwrap() 565 } 566 567 fn prepare_vhost_user_net_daemon( 568 tmp_dir: &TempDir, 569 ip: &str, 570 tap: Option<&str>, 571 mtu: Option<u16>, 572 num_queues: usize, 573 client_mode: bool, 574 ) -> (std::process::Command, String) { 575 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 576 577 // Start the daemon 578 let mut net_params = format!( 579 "ip={ip},mask=255.255.255.0,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" 580 ); 581 582 if let Some(tap) = tap { 583 net_params.push_str(format!(",tap={tap}").as_str()); 584 } 585 586 if let Some(mtu) = mtu { 587 net_params.push_str(format!(",mtu={mtu}").as_str()); 588 } 589 590 let mut command = Command::new(clh_command("vhost_user_net")); 591 command.args(["--net-backend", net_params.as_str()]); 592 593 (command, vunet_socket_path) 594 } 595 596 fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { 597 let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); 598 let swtpm_socket_path = String::from( 599 tmp_dir 600 .as_path() 601 .join("swtpm") 602 .join("swtpm.sock") 603 .to_str() 604 .unwrap(), 605 ); 606 std::fs::create_dir(&swtpm_tpm_dir).unwrap(); 607 608 let mut swtpm_command = Command::new("swtpm"); 609 let swtpm_args = [ 610 "socket", 611 "--tpmstate", 612 &format!("dir={swtpm_tpm_dir}"), 613 "--ctrl", 614 &format!("type=unixio,path={swtpm_socket_path}"), 615 "--flags", 616 "startup-clear", 617 "--tpm2", 618 ]; 619 swtpm_command.args(swtpm_args); 620 621 (swtpm_command, swtpm_socket_path) 622 } 623 624 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 625 let mut cmd = Command::new(clh_command("ch-remote")); 626 cmd.args([&format!("--api-socket={api_socket}"), command]); 627 628 if let Some(arg) = arg { 629 cmd.arg(arg); 630 } 631 let output = cmd.output().unwrap(); 632 if output.status.success() { 633 true 634 } else { 635 eprintln!("Error running ch-remote command: {:?}", &cmd); 636 let stderr = String::from_utf8_lossy(&output.stderr); 637 eprintln!("stderr: {stderr}"); 638 false 639 } 640 } 641 642 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 643 let mut cmd = Command::new(clh_command("ch-remote")); 644 cmd.args([&format!("--api-socket={api_socket}"), command]); 645 646 if let Some(arg) = arg { 647 cmd.arg(arg); 648 } 649 650 let output = cmd.output().expect("Failed to launch ch-remote"); 651 652 (output.status.success(), output.stdout) 653 } 654 655 fn resize_command( 656 api_socket: &str, 657 desired_vcpus: Option<u8>, 658 desired_ram: Option<usize>, 659 desired_balloon: Option<usize>, 660 event_file: Option<&str>, 661 ) -> bool { 662 let mut cmd = Command::new(clh_command("ch-remote")); 663 cmd.args([&format!("--api-socket={api_socket}"), "resize"]); 664 665 if let Some(desired_vcpus) = desired_vcpus { 666 cmd.arg(format!("--cpus={desired_vcpus}")); 667 } 668 669 if let Some(desired_ram) = desired_ram { 670 cmd.arg(format!("--memory={desired_ram}")); 671 } 672 673 if let Some(desired_balloon) = desired_balloon { 674 cmd.arg(format!("--balloon={desired_balloon}")); 675 } 676 677 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 678 679 if let Some(event_path) = event_file { 680 let latest_events = [ 681 &MetaEvent { 682 event: "resizing".to_string(), 683 device_id: None, 684 }, 685 &MetaEvent { 686 event: "resized".to_string(), 687 device_id: None, 688 }, 689 ]; 690 assert!(check_latest_events_exact(&latest_events, event_path)); 691 } 692 693 ret 694 } 695 696 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 697 let mut cmd = Command::new(clh_command("ch-remote")); 698 cmd.args([ 699 &format!("--api-socket={api_socket}"), 700 "resize-zone", 701 &format!("--id={id}"), 702 &format!("--size={desired_size}"), 703 ]); 704 705 cmd.status().expect("Failed to launch ch-remote").success() 706 } 707 708 // setup OVS-DPDK bridge and ports 709 fn setup_ovs_dpdk() { 710 // setup OVS-DPDK 711 assert!(exec_host_command_status("service openvswitch-switch start").success()); 712 assert!(exec_host_command_status("ovs-vsctl init").success()); 713 assert!( 714 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 715 .success() 716 ); 717 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 718 719 // Create OVS-DPDK bridge and ports 720 assert!(exec_host_command_status( 721 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 722 ) 723 .success()); 724 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 725 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 726 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 727 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 728 } 729 fn cleanup_ovs_dpdk() { 730 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 731 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 732 } 733 // Setup two guests and ensure they are connected through ovs-dpdk 734 fn setup_ovs_dpdk_guests( 735 guest1: &Guest, 736 guest2: &Guest, 737 api_socket: &str, 738 release_binary: bool, 739 ) -> (Child, Child) { 740 setup_ovs_dpdk(); 741 742 let clh_path = if !release_binary { 743 clh_command("cloud-hypervisor") 744 } else { 745 cloud_hypervisor_release_path() 746 }; 747 748 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 749 .args(["--cpus", "boot=2"]) 750 .args(["--memory", "size=0,shared=on"]) 751 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 752 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 753 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 754 .default_disks() 755 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 756 .capture_output() 757 .spawn() 758 .unwrap(); 759 760 #[cfg(target_arch = "x86_64")] 761 let guest_net_iface = "ens5"; 762 #[cfg(target_arch = "aarch64")] 763 let guest_net_iface = "enp0s5"; 764 765 let r = std::panic::catch_unwind(|| { 766 guest1.wait_vm_boot(None).unwrap(); 767 768 guest1 769 .ssh_command(&format!( 770 "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" 771 )) 772 .unwrap(); 773 guest1 774 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 775 .unwrap(); 776 777 let guest_ip = guest1.network.guest_ip.clone(); 778 thread::spawn(move || { 779 ssh_command_ip( 780 "nc -l 12345", 781 &guest_ip, 782 DEFAULT_SSH_RETRIES, 783 DEFAULT_SSH_TIMEOUT, 784 ) 785 .unwrap(); 786 }); 787 }); 788 if r.is_err() { 789 cleanup_ovs_dpdk(); 790 791 let _ = child1.kill(); 792 let output = child1.wait_with_output().unwrap(); 793 handle_child_output(r, &output); 794 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 795 } 796 797 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 798 .args(["--api-socket", api_socket]) 799 .args(["--cpus", "boot=2"]) 800 .args(["--memory", "size=0,shared=on"]) 801 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 802 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 803 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 804 .default_disks() 805 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 806 .capture_output() 807 .spawn() 808 .unwrap(); 809 810 let r = std::panic::catch_unwind(|| { 811 guest2.wait_vm_boot(None).unwrap(); 812 813 guest2 814 .ssh_command(&format!( 815 "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" 816 )) 817 .unwrap(); 818 guest2 819 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 820 .unwrap(); 821 822 // Check the connection works properly between the two VMs 823 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 824 }); 825 if r.is_err() { 826 cleanup_ovs_dpdk(); 827 828 let _ = child1.kill(); 829 let _ = child2.kill(); 830 let output = child2.wait_with_output().unwrap(); 831 handle_child_output(r, &output); 832 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 833 } 834 835 (child1, child2) 836 } 837 838 enum FwType { 839 Ovmf, 840 RustHypervisorFirmware, 841 } 842 843 fn fw_path(_fw_type: FwType) -> String { 844 let mut workload_path = dirs::home_dir().unwrap(); 845 workload_path.push("workloads"); 846 847 let mut fw_path = workload_path; 848 #[cfg(target_arch = "aarch64")] 849 fw_path.push("CLOUDHV_EFI.fd"); 850 #[cfg(target_arch = "x86_64")] 851 { 852 match _fw_type { 853 FwType::Ovmf => fw_path.push(OVMF_NAME), 854 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 855 } 856 } 857 858 fw_path.to_str().unwrap().to_string() 859 } 860 861 #[derive(Debug)] 862 struct MetaEvent { 863 event: String, 864 device_id: Option<String>, 865 } 866 867 impl MetaEvent { 868 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 869 let mut matched = false; 870 if v["event"].as_str().unwrap() == self.event { 871 if let Some(device_id) = &self.device_id { 872 if v["properties"]["id"].as_str().unwrap() == device_id { 873 matched = true 874 } 875 } else { 876 matched = true; 877 } 878 } 879 matched 880 } 881 } 882 883 // Parse the event_monitor file based on the format that each event 884 // is followed by a double newline 885 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 886 let content = fs::read(event_file).unwrap(); 887 let mut ret = Vec::new(); 888 for entry in String::from_utf8_lossy(&content) 889 .trim() 890 .split("\n\n") 891 .collect::<Vec<&str>>() 892 { 893 ret.push(serde_json::from_str(entry).unwrap()); 894 } 895 896 ret 897 } 898 899 // Return true if all events from the input 'expected_events' are matched sequentially 900 // with events from the 'event_file' 901 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 902 let json_events = parse_event_file(event_file); 903 let len = expected_events.len(); 904 let mut idx = 0; 905 for e in &json_events { 906 if idx == len { 907 break; 908 } 909 if expected_events[idx].match_with_json_event(e) { 910 idx += 1; 911 } 912 } 913 914 let ret = idx == len; 915 916 if !ret { 917 eprintln!( 918 "\n\n==== Start 'check_sequential_events' failed ==== \ 919 \n\nexpected_events={:?}\nactual_events={:?} \ 920 \n\n==== End 'check_sequential_events' failed ====", 921 expected_events, json_events, 922 ); 923 } 924 925 ret 926 } 927 928 // Return true if all events from the input 'expected_events' are matched exactly 929 // with events from the 'event_file' 930 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 931 let json_events = parse_event_file(event_file); 932 assert!(expected_events.len() <= json_events.len()); 933 let json_events = &json_events[..expected_events.len()]; 934 935 for (idx, e) in json_events.iter().enumerate() { 936 if !expected_events[idx].match_with_json_event(e) { 937 eprintln!( 938 "\n\n==== Start 'check_sequential_events_exact' failed ==== \ 939 \n\nexpected_events={:?}\nactual_events={:?} \ 940 \n\n==== End 'check_sequential_events_exact' failed ====", 941 expected_events, json_events, 942 ); 943 944 return false; 945 } 946 } 947 948 true 949 } 950 951 // Return true if events from the input 'latest_events' are matched exactly 952 // with the most recent events from the 'event_file' 953 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 954 let json_events = parse_event_file(event_file); 955 assert!(latest_events.len() <= json_events.len()); 956 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 957 958 for (idx, e) in json_events.iter().enumerate() { 959 if !latest_events[idx].match_with_json_event(e) { 960 eprintln!( 961 "\n\n==== Start 'check_latest_events_exact' failed ==== \ 962 \n\nexpected_events={:?}\nactual_events={:?} \ 963 \n\n==== End 'check_latest_events_exact' failed ====", 964 latest_events, json_events, 965 ); 966 967 return false; 968 } 969 } 970 971 true 972 } 973 974 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 975 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 976 let guest = Guest::new(Box::new(focal)); 977 let total_vcpus = threads_per_core * cores_per_package * packages; 978 let direct_kernel_boot_path = direct_kernel_boot_path(); 979 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 980 let fw_path = fw_path(FwType::RustHypervisorFirmware); 981 if use_fw { 982 kernel_path = fw_path.as_str(); 983 } 984 985 let mut child = GuestCommand::new(&guest) 986 .args([ 987 "--cpus", 988 &format!( 989 "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" 990 ), 991 ]) 992 .args(["--memory", "size=512M"]) 993 .args(["--kernel", kernel_path]) 994 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 995 .default_disks() 996 .default_net() 997 .capture_output() 998 .spawn() 999 .unwrap(); 1000 1001 let r = std::panic::catch_unwind(|| { 1002 guest.wait_vm_boot(None).unwrap(); 1003 assert_eq!( 1004 guest.get_cpu_count().unwrap_or_default(), 1005 u32::from(total_vcpus) 1006 ); 1007 assert_eq!( 1008 guest 1009 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1010 .unwrap() 1011 .trim() 1012 .parse::<u8>() 1013 .unwrap_or(0), 1014 threads_per_core 1015 ); 1016 1017 assert_eq!( 1018 guest 1019 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1020 .unwrap() 1021 .trim() 1022 .parse::<u8>() 1023 .unwrap_or(0), 1024 cores_per_package 1025 ); 1026 1027 assert_eq!( 1028 guest 1029 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1030 .unwrap() 1031 .trim() 1032 .parse::<u8>() 1033 .unwrap_or(0), 1034 packages 1035 ); 1036 }); 1037 1038 let _ = child.kill(); 1039 let output = child.wait_with_output().unwrap(); 1040 1041 handle_child_output(r, &output); 1042 } 1043 1044 #[allow(unused_variables)] 1045 fn _test_guest_numa_nodes(acpi: bool) { 1046 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1047 let guest = Guest::new(Box::new(focal)); 1048 let api_socket = temp_api_path(&guest.tmp_dir); 1049 #[cfg(target_arch = "x86_64")] 1050 let kernel_path = direct_kernel_boot_path(); 1051 #[cfg(target_arch = "aarch64")] 1052 let kernel_path = if acpi { 1053 edk2_path() 1054 } else { 1055 direct_kernel_boot_path() 1056 }; 1057 1058 let mut child = GuestCommand::new(&guest) 1059 .args(["--cpus", "boot=6,max=12"]) 1060 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 1061 .args([ 1062 "--memory-zone", 1063 "id=mem0,size=1G,hotplug_size=3G", 1064 "id=mem1,size=2G,hotplug_size=3G", 1065 "id=mem2,size=3G,hotplug_size=3G", 1066 ]) 1067 .args([ 1068 "--numa", 1069 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 1070 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 1071 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 1072 ]) 1073 .args(["--kernel", kernel_path.to_str().unwrap()]) 1074 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1075 .args(["--api-socket", &api_socket]) 1076 .capture_output() 1077 .default_disks() 1078 .default_net() 1079 .spawn() 1080 .unwrap(); 1081 1082 let r = std::panic::catch_unwind(|| { 1083 guest.wait_vm_boot(None).unwrap(); 1084 1085 guest.check_numa_common( 1086 Some(&[960_000, 1_920_000, 2_880_000]), 1087 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 1088 Some(&["10 15 20", "20 10 25", "25 30 10"]), 1089 ); 1090 1091 // AArch64 currently does not support hotplug, and therefore we only 1092 // test hotplug-related function on x86_64 here. 1093 #[cfg(target_arch = "x86_64")] 1094 { 1095 guest.enable_memory_hotplug(); 1096 1097 // Resize every memory zone and check each associated NUMA node 1098 // has been assigned the right amount of memory. 1099 resize_zone_command(&api_socket, "mem0", "4G"); 1100 resize_zone_command(&api_socket, "mem1", "4G"); 1101 resize_zone_command(&api_socket, "mem2", "4G"); 1102 // Resize to the maximum amount of CPUs and check each NUMA 1103 // node has been assigned the right CPUs set. 1104 resize_command(&api_socket, Some(12), None, None, None); 1105 thread::sleep(std::time::Duration::new(5, 0)); 1106 1107 guest.check_numa_common( 1108 Some(&[3_840_000, 3_840_000, 3_840_000]), 1109 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 1110 None, 1111 ); 1112 } 1113 }); 1114 1115 let _ = child.kill(); 1116 let output = child.wait_with_output().unwrap(); 1117 1118 handle_child_output(r, &output); 1119 } 1120 1121 #[allow(unused_variables)] 1122 fn _test_power_button(acpi: bool) { 1123 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1124 let guest = Guest::new(Box::new(focal)); 1125 let mut cmd = GuestCommand::new(&guest); 1126 let api_socket = temp_api_path(&guest.tmp_dir); 1127 1128 #[cfg(target_arch = "x86_64")] 1129 let kernel_path = direct_kernel_boot_path(); 1130 #[cfg(target_arch = "aarch64")] 1131 let kernel_path = if acpi { 1132 edk2_path() 1133 } else { 1134 direct_kernel_boot_path() 1135 }; 1136 1137 cmd.args(["--cpus", "boot=1"]) 1138 .args(["--memory", "size=512M"]) 1139 .args(["--kernel", kernel_path.to_str().unwrap()]) 1140 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1141 .capture_output() 1142 .default_disks() 1143 .default_net() 1144 .args(["--api-socket", &api_socket]); 1145 1146 let child = cmd.spawn().unwrap(); 1147 1148 let r = std::panic::catch_unwind(|| { 1149 guest.wait_vm_boot(None).unwrap(); 1150 assert!(remote_command(&api_socket, "power-button", None)); 1151 }); 1152 1153 let output = child.wait_with_output().unwrap(); 1154 assert!(output.status.success()); 1155 handle_child_output(r, &output); 1156 } 1157 1158 type PrepareNetDaemon = dyn Fn( 1159 &TempDir, 1160 &str, 1161 Option<&str>, 1162 Option<u16>, 1163 usize, 1164 bool, 1165 ) -> (std::process::Command, String); 1166 1167 fn test_vhost_user_net( 1168 tap: Option<&str>, 1169 num_queues: usize, 1170 prepare_daemon: &PrepareNetDaemon, 1171 generate_host_mac: bool, 1172 client_mode_daemon: bool, 1173 ) { 1174 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1175 let guest = Guest::new(Box::new(focal)); 1176 let api_socket = temp_api_path(&guest.tmp_dir); 1177 1178 let kernel_path = direct_kernel_boot_path(); 1179 1180 let host_mac = if generate_host_mac { 1181 Some(MacAddr::local_random()) 1182 } else { 1183 None 1184 }; 1185 1186 let mtu = Some(3000); 1187 1188 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 1189 &guest.tmp_dir, 1190 &guest.network.host_ip, 1191 tap, 1192 mtu, 1193 num_queues, 1194 client_mode_daemon, 1195 ); 1196 1197 let net_params = format!( 1198 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 1199 guest.network.guest_mac, 1200 vunet_socket_path, 1201 num_queues, 1202 if let Some(host_mac) = host_mac { 1203 format!(",host_mac={host_mac}") 1204 } else { 1205 "".to_owned() 1206 }, 1207 if client_mode_daemon { 1208 "server" 1209 } else { 1210 "client" 1211 }, 1212 ); 1213 1214 let mut ch_command = GuestCommand::new(&guest); 1215 ch_command 1216 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 1217 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1218 .args(["--kernel", kernel_path.to_str().unwrap()]) 1219 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1220 .default_disks() 1221 .args(["--net", net_params.as_str()]) 1222 .args(["--api-socket", &api_socket]) 1223 .capture_output(); 1224 1225 let mut daemon_child: std::process::Child; 1226 let mut child: std::process::Child; 1227 1228 if client_mode_daemon { 1229 child = ch_command.spawn().unwrap(); 1230 // Make sure the VMM is waiting for the backend to connect 1231 thread::sleep(std::time::Duration::new(10, 0)); 1232 daemon_child = daemon_command.spawn().unwrap(); 1233 } else { 1234 daemon_child = daemon_command.spawn().unwrap(); 1235 // Make sure the backend is waiting for the VMM to connect 1236 thread::sleep(std::time::Duration::new(10, 0)); 1237 child = ch_command.spawn().unwrap(); 1238 } 1239 1240 let r = std::panic::catch_unwind(|| { 1241 guest.wait_vm_boot(None).unwrap(); 1242 1243 if let Some(tap_name) = tap { 1244 let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); 1245 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1246 } 1247 1248 if let Some(host_mac) = tap { 1249 let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); 1250 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1251 } 1252 1253 #[cfg(target_arch = "aarch64")] 1254 let iface = "enp0s4"; 1255 #[cfg(target_arch = "x86_64")] 1256 let iface = "ens4"; 1257 1258 assert_eq!( 1259 guest 1260 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 1261 .unwrap() 1262 .trim(), 1263 "3000" 1264 ); 1265 1266 // 1 network interface + default localhost ==> 2 interfaces 1267 // It's important to note that this test is fully exercising the 1268 // vhost-user-net implementation and the associated backend since 1269 // it does not define any --net network interface. That means all 1270 // the ssh communication in that test happens through the network 1271 // interface backed by vhost-user-net. 1272 assert_eq!( 1273 guest 1274 .ssh_command("ip -o link | wc -l") 1275 .unwrap() 1276 .trim() 1277 .parse::<u32>() 1278 .unwrap_or_default(), 1279 2 1280 ); 1281 1282 // The following pci devices will appear on guest with PCI-MSI 1283 // interrupt vectors assigned. 1284 // 1 virtio-console with 3 vectors: config, Rx, Tx 1285 // 1 virtio-blk with 2 vectors: config, Request 1286 // 1 virtio-blk with 2 vectors: config, Request 1287 // 1 virtio-rng with 2 vectors: config, Request 1288 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1289 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1290 // Based on the above, the total vectors should 14. 1291 #[cfg(target_arch = "x86_64")] 1292 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1293 #[cfg(target_arch = "aarch64")] 1294 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1295 assert_eq!( 1296 guest 1297 .ssh_command(grep_cmd) 1298 .unwrap() 1299 .trim() 1300 .parse::<u32>() 1301 .unwrap_or_default(), 1302 10 + (num_queues as u32) 1303 ); 1304 1305 // ACPI feature is needed. 1306 #[cfg(target_arch = "x86_64")] 1307 { 1308 guest.enable_memory_hotplug(); 1309 1310 // Add RAM to the VM 1311 let desired_ram = 1024 << 20; 1312 resize_command(&api_socket, None, Some(desired_ram), None, None); 1313 1314 thread::sleep(std::time::Duration::new(10, 0)); 1315 1316 // Here by simply checking the size (through ssh), we validate 1317 // the connection is still working, which means vhost-user-net 1318 // keeps working after the resize. 1319 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1320 } 1321 }); 1322 1323 let _ = child.kill(); 1324 let output = child.wait_with_output().unwrap(); 1325 1326 thread::sleep(std::time::Duration::new(5, 0)); 1327 let _ = daemon_child.kill(); 1328 let _ = daemon_child.wait(); 1329 1330 handle_child_output(r, &output); 1331 } 1332 1333 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1334 1335 fn test_vhost_user_blk( 1336 num_queues: usize, 1337 readonly: bool, 1338 direct: bool, 1339 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1340 ) { 1341 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1342 let guest = Guest::new(Box::new(focal)); 1343 let api_socket = temp_api_path(&guest.tmp_dir); 1344 1345 let kernel_path = direct_kernel_boot_path(); 1346 1347 let (blk_params, daemon_child) = { 1348 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1349 // Start the daemon 1350 let (daemon_child, vubd_socket_path) = 1351 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1352 1353 ( 1354 format!( 1355 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1356 ), 1357 Some(daemon_child), 1358 ) 1359 }; 1360 1361 let mut child = GuestCommand::new(&guest) 1362 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1363 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1364 .args(["--kernel", kernel_path.to_str().unwrap()]) 1365 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1366 .args([ 1367 "--disk", 1368 format!( 1369 "path={}", 1370 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1371 ) 1372 .as_str(), 1373 format!( 1374 "path={}", 1375 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1376 ) 1377 .as_str(), 1378 blk_params.as_str(), 1379 ]) 1380 .default_net() 1381 .args(["--api-socket", &api_socket]) 1382 .capture_output() 1383 .spawn() 1384 .unwrap(); 1385 1386 let r = std::panic::catch_unwind(|| { 1387 guest.wait_vm_boot(None).unwrap(); 1388 1389 // Check both if /dev/vdc exists and if the block size is 16M. 1390 assert_eq!( 1391 guest 1392 .ssh_command("lsblk | grep vdc | grep -c 16M") 1393 .unwrap() 1394 .trim() 1395 .parse::<u32>() 1396 .unwrap_or_default(), 1397 1 1398 ); 1399 1400 // Check if this block is RO or RW. 1401 assert_eq!( 1402 guest 1403 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1404 .unwrap() 1405 .trim() 1406 .parse::<u32>() 1407 .unwrap_or_default(), 1408 readonly as u32 1409 ); 1410 1411 // Check if the number of queues in /sys/block/vdc/mq matches the 1412 // expected num_queues. 1413 assert_eq!( 1414 guest 1415 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1416 .unwrap() 1417 .trim() 1418 .parse::<u32>() 1419 .unwrap_or_default(), 1420 num_queues as u32 1421 ); 1422 1423 // Mount the device 1424 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1425 guest.ssh_command("mkdir mount_image").unwrap(); 1426 guest 1427 .ssh_command( 1428 format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), 1429 ) 1430 .unwrap(); 1431 1432 // Check the content of the block device. The file "foo" should 1433 // contain "bar". 1434 assert_eq!( 1435 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1436 "bar" 1437 ); 1438 1439 // ACPI feature is needed. 1440 #[cfg(target_arch = "x86_64")] 1441 { 1442 guest.enable_memory_hotplug(); 1443 1444 // Add RAM to the VM 1445 let desired_ram = 1024 << 20; 1446 resize_command(&api_socket, None, Some(desired_ram), None, None); 1447 1448 thread::sleep(std::time::Duration::new(10, 0)); 1449 1450 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1451 1452 // Check again the content of the block device after the resize 1453 // has been performed. 1454 assert_eq!( 1455 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1456 "bar" 1457 ); 1458 } 1459 1460 // Unmount the device 1461 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1462 guest.ssh_command("rm -r mount_image").unwrap(); 1463 }); 1464 1465 let _ = child.kill(); 1466 let output = child.wait_with_output().unwrap(); 1467 1468 if let Some(mut daemon_child) = daemon_child { 1469 thread::sleep(std::time::Duration::new(5, 0)); 1470 let _ = daemon_child.kill(); 1471 let _ = daemon_child.wait(); 1472 } 1473 1474 handle_child_output(r, &output); 1475 } 1476 1477 fn test_boot_from_vhost_user_blk( 1478 num_queues: usize, 1479 readonly: bool, 1480 direct: bool, 1481 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1482 ) { 1483 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1484 let guest = Guest::new(Box::new(focal)); 1485 1486 let kernel_path = direct_kernel_boot_path(); 1487 1488 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1489 1490 let (blk_boot_params, daemon_child) = { 1491 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1492 // Start the daemon 1493 let (daemon_child, vubd_socket_path) = prepare_daemon( 1494 &guest.tmp_dir, 1495 disk_path.as_str(), 1496 num_queues, 1497 readonly, 1498 direct, 1499 ); 1500 1501 ( 1502 format!( 1503 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1504 ), 1505 Some(daemon_child), 1506 ) 1507 }; 1508 1509 let mut child = GuestCommand::new(&guest) 1510 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1511 .args(["--memory", "size=512M,shared=on"]) 1512 .args(["--kernel", kernel_path.to_str().unwrap()]) 1513 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1514 .args([ 1515 "--disk", 1516 blk_boot_params.as_str(), 1517 format!( 1518 "path={}", 1519 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1520 ) 1521 .as_str(), 1522 ]) 1523 .default_net() 1524 .capture_output() 1525 .spawn() 1526 .unwrap(); 1527 1528 let r = std::panic::catch_unwind(|| { 1529 guest.wait_vm_boot(None).unwrap(); 1530 1531 // Just check the VM booted correctly. 1532 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1533 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1534 }); 1535 let _ = child.kill(); 1536 let output = child.wait_with_output().unwrap(); 1537 1538 if let Some(mut daemon_child) = daemon_child { 1539 thread::sleep(std::time::Duration::new(5, 0)); 1540 let _ = daemon_child.kill(); 1541 let _ = daemon_child.wait(); 1542 } 1543 1544 handle_child_output(r, &output); 1545 } 1546 1547 fn _test_virtio_fs( 1548 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1549 hotplug: bool, 1550 pci_segment: Option<u16>, 1551 ) { 1552 #[cfg(target_arch = "aarch64")] 1553 let focal_image = if hotplug { 1554 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1555 } else { 1556 FOCAL_IMAGE_NAME.to_string() 1557 }; 1558 #[cfg(target_arch = "x86_64")] 1559 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1560 let focal = UbuntuDiskConfig::new(focal_image); 1561 let guest = Guest::new(Box::new(focal)); 1562 let api_socket = temp_api_path(&guest.tmp_dir); 1563 1564 let mut workload_path = dirs::home_dir().unwrap(); 1565 workload_path.push("workloads"); 1566 1567 let mut shared_dir = workload_path; 1568 shared_dir.push("shared_dir"); 1569 1570 #[cfg(target_arch = "x86_64")] 1571 let kernel_path = direct_kernel_boot_path(); 1572 #[cfg(target_arch = "aarch64")] 1573 let kernel_path = if hotplug { 1574 edk2_path() 1575 } else { 1576 direct_kernel_boot_path() 1577 }; 1578 1579 let (mut daemon_child, virtiofsd_socket_path) = 1580 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1581 1582 let mut guest_command = GuestCommand::new(&guest); 1583 guest_command 1584 .args(["--cpus", "boot=1"]) 1585 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1586 .args(["--kernel", kernel_path.to_str().unwrap()]) 1587 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1588 .default_disks() 1589 .default_net() 1590 .args(["--api-socket", &api_socket]); 1591 if pci_segment.is_some() { 1592 guest_command.args([ 1593 "--platform", 1594 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 1595 ]); 1596 } 1597 1598 let fs_params = format!( 1599 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1600 virtiofsd_socket_path, 1601 if let Some(pci_segment) = pci_segment { 1602 format!(",pci_segment={pci_segment}") 1603 } else { 1604 "".to_owned() 1605 } 1606 ); 1607 1608 if !hotplug { 1609 guest_command.args(["--fs", fs_params.as_str()]); 1610 } 1611 1612 let mut child = guest_command.capture_output().spawn().unwrap(); 1613 1614 let r = std::panic::catch_unwind(|| { 1615 guest.wait_vm_boot(None).unwrap(); 1616 1617 if hotplug { 1618 // Add fs to the VM 1619 let (cmd_success, cmd_output) = 1620 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1621 assert!(cmd_success); 1622 1623 if let Some(pci_segment) = pci_segment { 1624 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1625 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1626 ))); 1627 } else { 1628 assert!(String::from_utf8_lossy(&cmd_output) 1629 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1630 } 1631 1632 thread::sleep(std::time::Duration::new(10, 0)); 1633 } 1634 1635 // Mount shared directory through virtio_fs filesystem 1636 guest 1637 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1638 .unwrap(); 1639 1640 // Check file1 exists and its content is "foo" 1641 assert_eq!( 1642 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1643 "foo" 1644 ); 1645 // Check file2 does not exist 1646 guest 1647 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1648 .unwrap(); 1649 1650 // Check file3 exists and its content is "bar" 1651 assert_eq!( 1652 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1653 "bar" 1654 ); 1655 1656 // ACPI feature is needed. 1657 #[cfg(target_arch = "x86_64")] 1658 { 1659 guest.enable_memory_hotplug(); 1660 1661 // Add RAM to the VM 1662 let desired_ram = 1024 << 20; 1663 resize_command(&api_socket, None, Some(desired_ram), None, None); 1664 1665 thread::sleep(std::time::Duration::new(30, 0)); 1666 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1667 1668 // After the resize, check again that file1 exists and its 1669 // content is "foo". 1670 assert_eq!( 1671 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1672 "foo" 1673 ); 1674 } 1675 1676 if hotplug { 1677 // Remove from VM 1678 guest.ssh_command("sudo umount mount_dir").unwrap(); 1679 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1680 } 1681 }); 1682 1683 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1684 thread::sleep(std::time::Duration::new(10, 0)); 1685 let (daemon_child, virtiofsd_socket_path) = 1686 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1687 1688 let r = std::panic::catch_unwind(|| { 1689 thread::sleep(std::time::Duration::new(10, 0)); 1690 let fs_params = format!( 1691 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1692 virtiofsd_socket_path, 1693 if let Some(pci_segment) = pci_segment { 1694 format!(",pci_segment={pci_segment}") 1695 } else { 1696 "".to_owned() 1697 } 1698 ); 1699 1700 // Add back and check it works 1701 let (cmd_success, cmd_output) = 1702 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1703 assert!(cmd_success); 1704 if let Some(pci_segment) = pci_segment { 1705 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1706 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1707 ))); 1708 } else { 1709 assert!(String::from_utf8_lossy(&cmd_output) 1710 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1711 } 1712 1713 thread::sleep(std::time::Duration::new(10, 0)); 1714 // Mount shared directory through virtio_fs filesystem 1715 guest 1716 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1717 .unwrap(); 1718 1719 // Check file1 exists and its content is "foo" 1720 assert_eq!( 1721 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1722 "foo" 1723 ); 1724 }); 1725 1726 (r, Some(daemon_child)) 1727 } else { 1728 (r, None) 1729 }; 1730 1731 let _ = child.kill(); 1732 let output = child.wait_with_output().unwrap(); 1733 1734 let _ = daemon_child.kill(); 1735 let _ = daemon_child.wait(); 1736 1737 if let Some(mut daemon_child) = hotplug_daemon_child { 1738 let _ = daemon_child.kill(); 1739 let _ = daemon_child.wait(); 1740 } 1741 1742 handle_child_output(r, &output); 1743 } 1744 1745 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1746 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1747 let guest = Guest::new(Box::new(focal)); 1748 1749 let kernel_path = direct_kernel_boot_path(); 1750 1751 let pmem_temp_file = TempFile::new().unwrap(); 1752 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1753 1754 std::process::Command::new("mkfs.ext4") 1755 .arg(pmem_temp_file.as_path()) 1756 .output() 1757 .expect("Expect creating disk image to succeed"); 1758 1759 let mut child = GuestCommand::new(&guest) 1760 .args(["--cpus", "boot=1"]) 1761 .args(["--memory", "size=512M"]) 1762 .args(["--kernel", kernel_path.to_str().unwrap()]) 1763 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1764 .default_disks() 1765 .default_net() 1766 .args([ 1767 "--pmem", 1768 format!( 1769 "file={}{}{}", 1770 pmem_temp_file.as_path().to_str().unwrap(), 1771 if specify_size { ",size=128M" } else { "" }, 1772 if discard_writes { 1773 ",discard_writes=on" 1774 } else { 1775 "" 1776 } 1777 ) 1778 .as_str(), 1779 ]) 1780 .capture_output() 1781 .spawn() 1782 .unwrap(); 1783 1784 let r = std::panic::catch_unwind(|| { 1785 guest.wait_vm_boot(None).unwrap(); 1786 1787 // Check for the presence of /dev/pmem0 1788 assert_eq!( 1789 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1790 "/dev/pmem0" 1791 ); 1792 1793 // Check changes persist after reboot 1794 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1795 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1796 guest 1797 .ssh_command("echo test123 | sudo tee /mnt/test") 1798 .unwrap(); 1799 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1800 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1801 1802 guest.reboot_linux(0, None); 1803 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1804 assert_eq!( 1805 guest 1806 .ssh_command("sudo cat /mnt/test || true") 1807 .unwrap() 1808 .trim(), 1809 if discard_writes { "" } else { "test123" } 1810 ); 1811 }); 1812 1813 let _ = child.kill(); 1814 let output = child.wait_with_output().unwrap(); 1815 1816 handle_child_output(r, &output); 1817 } 1818 1819 fn get_fd_count(pid: u32) -> usize { 1820 fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() 1821 } 1822 1823 fn _test_virtio_vsock(hotplug: bool) { 1824 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1825 let guest = Guest::new(Box::new(focal)); 1826 1827 #[cfg(target_arch = "x86_64")] 1828 let kernel_path = direct_kernel_boot_path(); 1829 #[cfg(target_arch = "aarch64")] 1830 let kernel_path = if hotplug { 1831 edk2_path() 1832 } else { 1833 direct_kernel_boot_path() 1834 }; 1835 1836 let socket = temp_vsock_path(&guest.tmp_dir); 1837 let api_socket = temp_api_path(&guest.tmp_dir); 1838 1839 let mut cmd = GuestCommand::new(&guest); 1840 cmd.args(["--api-socket", &api_socket]); 1841 cmd.args(["--cpus", "boot=1"]); 1842 cmd.args(["--memory", "size=512M"]); 1843 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1844 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1845 cmd.default_disks(); 1846 cmd.default_net(); 1847 1848 if !hotplug { 1849 cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); 1850 } 1851 1852 let mut child = cmd.capture_output().spawn().unwrap(); 1853 1854 let r = std::panic::catch_unwind(|| { 1855 guest.wait_vm_boot(None).unwrap(); 1856 1857 if hotplug { 1858 let (cmd_success, cmd_output) = remote_command_w_output( 1859 &api_socket, 1860 "add-vsock", 1861 Some(format!("cid=3,socket={socket},id=test0").as_str()), 1862 ); 1863 assert!(cmd_success); 1864 assert!(String::from_utf8_lossy(&cmd_output) 1865 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1866 thread::sleep(std::time::Duration::new(10, 0)); 1867 // Check adding a second one fails 1868 assert!(!remote_command( 1869 &api_socket, 1870 "add-vsock", 1871 Some("cid=1234,socket=/tmp/fail") 1872 )); 1873 } 1874 1875 // Validate vsock works as expected. 1876 guest.check_vsock(socket.as_str()); 1877 guest.reboot_linux(0, None); 1878 // Validate vsock still works after a reboot. 1879 guest.check_vsock(socket.as_str()); 1880 1881 if hotplug { 1882 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1883 } 1884 }); 1885 1886 let _ = child.kill(); 1887 let output = child.wait_with_output().unwrap(); 1888 1889 handle_child_output(r, &output); 1890 } 1891 1892 fn get_ksm_pages_shared() -> u32 { 1893 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1894 .unwrap() 1895 .trim() 1896 .parse::<u32>() 1897 .unwrap() 1898 } 1899 1900 fn test_memory_mergeable(mergeable: bool) { 1901 let memory_param = if mergeable { 1902 "mergeable=on" 1903 } else { 1904 "mergeable=off" 1905 }; 1906 1907 // We are assuming the rest of the system in our CI is not using mergeable memory 1908 let ksm_ps_init = get_ksm_pages_shared(); 1909 assert!(ksm_ps_init == 0); 1910 1911 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1912 let guest1 = Guest::new(Box::new(focal1)); 1913 let mut child1 = GuestCommand::new(&guest1) 1914 .args(["--cpus", "boot=1"]) 1915 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1916 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1917 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1918 .default_disks() 1919 .args(["--net", guest1.default_net_string().as_str()]) 1920 .args(["--serial", "tty", "--console", "off"]) 1921 .capture_output() 1922 .spawn() 1923 .unwrap(); 1924 1925 let r = std::panic::catch_unwind(|| { 1926 guest1.wait_vm_boot(None).unwrap(); 1927 }); 1928 if r.is_err() { 1929 let _ = child1.kill(); 1930 let output = child1.wait_with_output().unwrap(); 1931 handle_child_output(r, &output); 1932 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1933 } 1934 1935 let ksm_ps_guest1 = get_ksm_pages_shared(); 1936 1937 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1938 let guest2 = Guest::new(Box::new(focal2)); 1939 let mut child2 = GuestCommand::new(&guest2) 1940 .args(["--cpus", "boot=1"]) 1941 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1942 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1943 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1944 .default_disks() 1945 .args(["--net", guest2.default_net_string().as_str()]) 1946 .args(["--serial", "tty", "--console", "off"]) 1947 .capture_output() 1948 .spawn() 1949 .unwrap(); 1950 1951 let r = std::panic::catch_unwind(|| { 1952 guest2.wait_vm_boot(None).unwrap(); 1953 let ksm_ps_guest2 = get_ksm_pages_shared(); 1954 1955 if mergeable { 1956 println!( 1957 "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" 1958 ); 1959 // We are expecting the number of shared pages to increase as the number of VM increases 1960 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1961 } else { 1962 assert!(ksm_ps_guest1 == 0); 1963 assert!(ksm_ps_guest2 == 0); 1964 } 1965 }); 1966 1967 let _ = child1.kill(); 1968 let _ = child2.kill(); 1969 1970 let output = child1.wait_with_output().unwrap(); 1971 child2.wait().unwrap(); 1972 1973 handle_child_output(r, &output); 1974 } 1975 1976 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 1977 let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); 1978 let reader = io::BufReader::new(smaps); 1979 1980 let mut skip_map: bool = false; 1981 let mut region_name: String = "".to_string(); 1982 let mut region_maps = HashMap::new(); 1983 for line in reader.lines() { 1984 let l = line.unwrap(); 1985 1986 if l.contains('-') { 1987 let values: Vec<&str> = l.split_whitespace().collect(); 1988 region_name = values.last().unwrap().trim().to_string(); 1989 if region_name == "0" { 1990 region_name = "anonymous".to_string() 1991 } 1992 } 1993 1994 // Each section begins with something that looks like: 1995 // Size: 2184 kB 1996 if l.starts_with("Size:") { 1997 let values: Vec<&str> = l.split_whitespace().collect(); 1998 let map_size = values[1].parse::<u32>().unwrap(); 1999 // We skip the assigned guest RAM map, its RSS is only 2000 // dependent on the guest actual memory usage. 2001 // Everything else can be added to the VMM overhead. 2002 skip_map = map_size >= guest_memory_size; 2003 continue; 2004 } 2005 2006 // If this is a map we're taking into account, then we only 2007 // count the RSS. The sum of all counted RSS is the VMM overhead. 2008 if !skip_map && l.starts_with("Rss:") { 2009 let values: Vec<&str> = l.split_whitespace().collect(); 2010 let value = values[1].trim().parse::<u32>().unwrap(); 2011 *region_maps.entry(region_name.clone()).or_insert(0) += value; 2012 } 2013 } 2014 2015 region_maps 2016 } 2017 2018 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 2019 let mut total = 0; 2020 2021 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 2022 eprintln!("{region_name}: {value}"); 2023 total += value; 2024 } 2025 2026 total 2027 } 2028 2029 fn process_rss_kib(pid: u32) -> usize { 2030 let command = format!("ps -q {pid} -o rss="); 2031 let rss = exec_host_command_output(&command); 2032 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 2033 } 2034 2035 // 10MB is our maximum accepted overhead. 2036 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 2037 2038 #[derive(PartialEq, Eq, PartialOrd)] 2039 struct Counters { 2040 rx_bytes: u64, 2041 rx_frames: u64, 2042 tx_bytes: u64, 2043 tx_frames: u64, 2044 read_bytes: u64, 2045 write_bytes: u64, 2046 read_ops: u64, 2047 write_ops: u64, 2048 } 2049 2050 fn get_counters(api_socket: &str) -> Counters { 2051 // Get counters 2052 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 2053 assert!(cmd_success); 2054 2055 let counters: HashMap<&str, HashMap<&str, u64>> = 2056 serde_json::from_slice(&cmd_output).unwrap_or_default(); 2057 2058 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 2059 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 2060 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 2061 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 2062 2063 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 2064 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 2065 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 2066 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 2067 2068 Counters { 2069 rx_bytes, 2070 rx_frames, 2071 tx_bytes, 2072 tx_frames, 2073 read_bytes, 2074 write_bytes, 2075 read_ops, 2076 write_ops, 2077 } 2078 } 2079 2080 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 2081 let (tx, rx) = mpsc::channel::<String>(); 2082 thread::spawn(move || loop { 2083 thread::sleep(std::time::Duration::new(1, 0)); 2084 let mut buf = [0; 512]; 2085 match pty.read(&mut buf) { 2086 Ok(_) => { 2087 let output = std::str::from_utf8(&buf).unwrap().to_string(); 2088 match tx.send(output) { 2089 Ok(_) => (), 2090 Err(_) => break, 2091 } 2092 } 2093 Err(_) => break, 2094 } 2095 }); 2096 rx 2097 } 2098 2099 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 2100 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2101 assert!(cmd_success); 2102 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2103 assert_eq!("Pty", info["config"][pty_type]["mode"]); 2104 PathBuf::from( 2105 info["config"][pty_type]["file"] 2106 .as_str() 2107 .expect("Missing pty path"), 2108 ) 2109 } 2110 2111 // VFIO test network setup. 2112 // We reserve a different IP class for it: 172.18.0.0/24. 2113 #[cfg(target_arch = "x86_64")] 2114 fn setup_vfio_network_interfaces() { 2115 // 'vfio-br0' 2116 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 2117 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 2118 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 2119 // 'vfio-tap0' 2120 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 2121 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 2122 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 2123 // 'vfio-tap1' 2124 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 2125 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 2126 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 2127 // 'vfio-tap2' 2128 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 2129 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 2130 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 2131 // 'vfio-tap3' 2132 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 2133 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 2134 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 2135 } 2136 2137 // Tear VFIO test network down 2138 #[cfg(target_arch = "x86_64")] 2139 fn cleanup_vfio_network_interfaces() { 2140 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 2141 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 2142 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 2143 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 2144 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 2145 } 2146 2147 fn balloon_size(api_socket: &str) -> u64 { 2148 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2149 assert!(cmd_success); 2150 2151 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2152 let total_mem = &info["config"]["memory"]["size"] 2153 .to_string() 2154 .parse::<u64>() 2155 .unwrap(); 2156 let actual_mem = &info["memory_actual_size"] 2157 .to_string() 2158 .parse::<u64>() 2159 .unwrap(); 2160 total_mem - actual_mem 2161 } 2162 2163 fn vm_state(api_socket: &str) -> String { 2164 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2165 assert!(cmd_success); 2166 2167 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2168 let state = &info["state"].as_str().unwrap(); 2169 2170 state.to_string() 2171 } 2172 2173 // This test validates that it can find the virtio-iommu device at first. 2174 // It also verifies that both disks and the network card are attached to 2175 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 2176 // The last interesting part of this test is that it exercises the network 2177 // interface attached to the virtual IOMMU since this is the one used to 2178 // send all commands through SSH. 2179 fn _test_virtio_iommu(acpi: bool) { 2180 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 2181 // Focal image is still old. 2182 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 2183 // the kernel binary has been updated. 2184 #[cfg(target_arch = "aarch64")] 2185 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 2186 #[cfg(target_arch = "x86_64")] 2187 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2188 let focal = UbuntuDiskConfig::new(focal_image); 2189 let guest = Guest::new(Box::new(focal)); 2190 2191 #[cfg(target_arch = "x86_64")] 2192 let kernel_path = direct_kernel_boot_path(); 2193 #[cfg(target_arch = "aarch64")] 2194 let kernel_path = if acpi { 2195 edk2_path() 2196 } else { 2197 direct_kernel_boot_path() 2198 }; 2199 2200 let mut child = GuestCommand::new(&guest) 2201 .args(["--cpus", "boot=1"]) 2202 .args(["--memory", "size=512M"]) 2203 .args(["--kernel", kernel_path.to_str().unwrap()]) 2204 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2205 .args([ 2206 "--disk", 2207 format!( 2208 "path={},iommu=on", 2209 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2210 ) 2211 .as_str(), 2212 format!( 2213 "path={},iommu=on", 2214 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2215 ) 2216 .as_str(), 2217 ]) 2218 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 2219 .capture_output() 2220 .spawn() 2221 .unwrap(); 2222 2223 let r = std::panic::catch_unwind(|| { 2224 guest.wait_vm_boot(None).unwrap(); 2225 2226 // Verify the virtio-iommu device is present. 2227 assert!(guest 2228 .does_device_vendor_pair_match("0x1057", "0x1af4") 2229 .unwrap_or_default()); 2230 2231 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 2232 // different with ACPI. 2233 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 2234 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 2235 // and appear under folder '/sys/kernel/iommu_groups/'. 2236 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 2237 // which is the console. The first disk "0000:00:02.0" is in group '1'. 2238 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 2239 // contains "0000:00:02.0" which is the first disk. 2240 // 2241 // Verify the iommu group of the first disk. 2242 let iommu_group = !acpi as i32; 2243 assert_eq!( 2244 guest 2245 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2246 .unwrap() 2247 .trim(), 2248 "0000:00:02.0" 2249 ); 2250 2251 // Verify the iommu group of the second disk. 2252 let iommu_group = if acpi { 1 } else { 2 }; 2253 assert_eq!( 2254 guest 2255 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2256 .unwrap() 2257 .trim(), 2258 "0000:00:03.0" 2259 ); 2260 2261 // Verify the iommu group of the network card. 2262 let iommu_group = if acpi { 2 } else { 3 }; 2263 assert_eq!( 2264 guest 2265 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2266 .unwrap() 2267 .trim(), 2268 "0000:00:04.0" 2269 ); 2270 }); 2271 2272 let _ = child.kill(); 2273 let output = child.wait_with_output().unwrap(); 2274 2275 handle_child_output(r, &output); 2276 } 2277 2278 fn get_reboot_count(guest: &Guest) -> u32 { 2279 guest 2280 .ssh_command("sudo last | grep -c reboot") 2281 .unwrap() 2282 .trim() 2283 .parse::<u32>() 2284 .unwrap_or_default() 2285 } 2286 2287 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 2288 // Check for PCI device 2289 assert!(guest 2290 .does_device_vendor_pair_match("0x1063", "0x1af4") 2291 .unwrap_or_default()); 2292 2293 // Enable systemd watchdog 2294 guest 2295 .ssh_command(&format!( 2296 "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" 2297 )) 2298 .unwrap(); 2299 } 2300 2301 fn make_guest_panic(guest: &Guest) { 2302 // Check for pvpanic device 2303 assert!(guest 2304 .does_device_vendor_pair_match("0x0011", "0x1b36") 2305 .unwrap_or_default()); 2306 2307 // Trigger guest a panic 2308 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 2309 } 2310 2311 mod common_parallel { 2312 use std::{fs::OpenOptions, io::SeekFrom}; 2313 2314 use crate::*; 2315 2316 #[test] 2317 #[cfg(target_arch = "x86_64")] 2318 fn test_focal_hypervisor_fw() { 2319 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 2320 } 2321 2322 #[test] 2323 #[cfg(target_arch = "x86_64")] 2324 fn test_focal_ovmf() { 2325 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 2326 } 2327 2328 #[cfg(target_arch = "x86_64")] 2329 fn test_simple_launch(fw_path: String, disk_path: &str) { 2330 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 2331 let guest = Guest::new(disk_config); 2332 let event_path = temp_event_monitor_path(&guest.tmp_dir); 2333 2334 let mut child = GuestCommand::new(&guest) 2335 .args(["--cpus", "boot=1"]) 2336 .args(["--memory", "size=512M"]) 2337 .args(["--kernel", fw_path.as_str()]) 2338 .default_disks() 2339 .default_net() 2340 .args(["--serial", "tty", "--console", "off"]) 2341 .args(["--event-monitor", format!("path={event_path}").as_str()]) 2342 .capture_output() 2343 .spawn() 2344 .unwrap(); 2345 2346 let r = std::panic::catch_unwind(|| { 2347 guest.wait_vm_boot(Some(120)).unwrap(); 2348 2349 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2350 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2351 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2352 2353 let expected_sequential_events = [ 2354 &MetaEvent { 2355 event: "starting".to_string(), 2356 device_id: None, 2357 }, 2358 &MetaEvent { 2359 event: "booting".to_string(), 2360 device_id: None, 2361 }, 2362 &MetaEvent { 2363 event: "booted".to_string(), 2364 device_id: None, 2365 }, 2366 &MetaEvent { 2367 event: "activated".to_string(), 2368 device_id: Some("_disk0".to_string()), 2369 }, 2370 &MetaEvent { 2371 event: "reset".to_string(), 2372 device_id: Some("_disk0".to_string()), 2373 }, 2374 ]; 2375 assert!(check_sequential_events( 2376 &expected_sequential_events, 2377 &event_path 2378 )); 2379 2380 // It's been observed on the Bionic image that udev and snapd 2381 // services can cause some delay in the VM's shutdown. Disabling 2382 // them improves the reliability of this test. 2383 let _ = guest.ssh_command("sudo systemctl disable udev"); 2384 let _ = guest.ssh_command("sudo systemctl stop udev"); 2385 let _ = guest.ssh_command("sudo systemctl disable snapd"); 2386 let _ = guest.ssh_command("sudo systemctl stop snapd"); 2387 2388 guest.ssh_command("sudo poweroff").unwrap(); 2389 thread::sleep(std::time::Duration::new(20, 0)); 2390 let latest_events = [ 2391 &MetaEvent { 2392 event: "shutdown".to_string(), 2393 device_id: None, 2394 }, 2395 &MetaEvent { 2396 event: "deleted".to_string(), 2397 device_id: None, 2398 }, 2399 &MetaEvent { 2400 event: "shutdown".to_string(), 2401 device_id: None, 2402 }, 2403 ]; 2404 assert!(check_latest_events_exact(&latest_events, &event_path)); 2405 }); 2406 2407 let _ = child.kill(); 2408 let output = child.wait_with_output().unwrap(); 2409 2410 handle_child_output(r, &output); 2411 } 2412 2413 #[test] 2414 fn test_multi_cpu() { 2415 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2416 let jammy = UbuntuDiskConfig::new(jammy_image); 2417 let guest = Guest::new(Box::new(jammy)); 2418 2419 let mut cmd = GuestCommand::new(&guest); 2420 cmd.args(["--cpus", "boot=2,max=4"]) 2421 .args(["--memory", "size=512M"]) 2422 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2423 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2424 .capture_output() 2425 .default_disks() 2426 .default_net(); 2427 2428 let mut child = cmd.spawn().unwrap(); 2429 2430 let r = std::panic::catch_unwind(|| { 2431 guest.wait_vm_boot(Some(120)).unwrap(); 2432 2433 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2434 2435 assert_eq!( 2436 guest 2437 .ssh_command( 2438 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2439 ) 2440 .unwrap() 2441 .trim(), 2442 "smp: Brought up 1 node, 2 CPUs" 2443 ); 2444 }); 2445 2446 let _ = child.kill(); 2447 let output = child.wait_with_output().unwrap(); 2448 2449 handle_child_output(r, &output); 2450 } 2451 2452 #[test] 2453 fn test_cpu_topology_421() { 2454 test_cpu_topology(4, 2, 1, false); 2455 } 2456 2457 #[test] 2458 fn test_cpu_topology_142() { 2459 test_cpu_topology(1, 4, 2, false); 2460 } 2461 2462 #[test] 2463 fn test_cpu_topology_262() { 2464 test_cpu_topology(2, 6, 2, false); 2465 } 2466 2467 #[test] 2468 #[cfg(target_arch = "x86_64")] 2469 #[cfg(not(feature = "mshv"))] 2470 fn test_cpu_physical_bits() { 2471 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2472 let guest = Guest::new(Box::new(focal)); 2473 let max_phys_bits: u8 = 36; 2474 let mut child = GuestCommand::new(&guest) 2475 .args(["--cpus", &format!("max_phys_bits={max_phys_bits}")]) 2476 .args(["--memory", "size=512M"]) 2477 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2478 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2479 .default_disks() 2480 .default_net() 2481 .capture_output() 2482 .spawn() 2483 .unwrap(); 2484 2485 let r = std::panic::catch_unwind(|| { 2486 guest.wait_vm_boot(None).unwrap(); 2487 2488 assert!( 2489 guest 2490 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2491 .unwrap() 2492 .trim() 2493 .parse::<u8>() 2494 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2495 ); 2496 }); 2497 2498 let _ = child.kill(); 2499 let output = child.wait_with_output().unwrap(); 2500 2501 handle_child_output(r, &output); 2502 } 2503 2504 #[test] 2505 fn test_cpu_affinity() { 2506 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2507 let guest = Guest::new(Box::new(focal)); 2508 2509 // We need the host to have at least 4 CPUs if we want to be able 2510 // to run this test. 2511 let host_cpus_count = exec_host_command_output("nproc"); 2512 assert!( 2513 String::from_utf8_lossy(&host_cpus_count.stdout) 2514 .trim() 2515 .parse::<u16>() 2516 .unwrap_or(0) 2517 >= 4 2518 ); 2519 2520 let mut child = GuestCommand::new(&guest) 2521 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2522 .args(["--memory", "size=512M"]) 2523 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2524 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2525 .default_disks() 2526 .default_net() 2527 .capture_output() 2528 .spawn() 2529 .unwrap(); 2530 2531 let r = std::panic::catch_unwind(|| { 2532 guest.wait_vm_boot(None).unwrap(); 2533 let pid = child.id(); 2534 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2535 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2536 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2537 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2538 }); 2539 2540 let _ = child.kill(); 2541 let output = child.wait_with_output().unwrap(); 2542 2543 handle_child_output(r, &output); 2544 } 2545 2546 #[test] 2547 #[cfg(not(feature = "mshv"))] 2548 fn test_large_vm() { 2549 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2550 let guest = Guest::new(Box::new(focal)); 2551 let mut cmd = GuestCommand::new(&guest); 2552 cmd.args(["--cpus", "boot=48"]) 2553 .args(["--memory", "size=5120M"]) 2554 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2555 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2556 .args(["--serial", "tty"]) 2557 .args(["--console", "off"]) 2558 .capture_output() 2559 .default_disks() 2560 .default_net(); 2561 2562 let mut child = cmd.spawn().unwrap(); 2563 2564 guest.wait_vm_boot(None).unwrap(); 2565 2566 let r = std::panic::catch_unwind(|| { 2567 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2568 assert_eq!( 2569 guest 2570 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2571 .unwrap() 2572 .trim(), 2573 "0-47" 2574 ); 2575 2576 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2577 }); 2578 2579 let _ = child.kill(); 2580 let output = child.wait_with_output().unwrap(); 2581 2582 handle_child_output(r, &output); 2583 } 2584 2585 #[test] 2586 #[cfg(not(feature = "mshv"))] 2587 fn test_huge_memory() { 2588 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2589 let guest = Guest::new(Box::new(focal)); 2590 let mut cmd = GuestCommand::new(&guest); 2591 cmd.args(["--cpus", "boot=1"]) 2592 .args(["--memory", "size=128G"]) 2593 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2594 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2595 .capture_output() 2596 .default_disks() 2597 .default_net(); 2598 2599 let mut child = cmd.spawn().unwrap(); 2600 2601 guest.wait_vm_boot(Some(120)).unwrap(); 2602 2603 let r = std::panic::catch_unwind(|| { 2604 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2605 }); 2606 2607 let _ = child.kill(); 2608 let output = child.wait_with_output().unwrap(); 2609 2610 handle_child_output(r, &output); 2611 } 2612 2613 #[test] 2614 fn test_power_button() { 2615 _test_power_button(false); 2616 } 2617 2618 #[test] 2619 #[cfg(not(feature = "mshv"))] 2620 fn test_user_defined_memory_regions() { 2621 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2622 let guest = Guest::new(Box::new(focal)); 2623 let api_socket = temp_api_path(&guest.tmp_dir); 2624 2625 let kernel_path = direct_kernel_boot_path(); 2626 2627 let mut child = GuestCommand::new(&guest) 2628 .args(["--cpus", "boot=1"]) 2629 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2630 .args([ 2631 "--memory-zone", 2632 "id=mem0,size=1G,hotplug_size=2G", 2633 "id=mem1,size=1G,shared=on", 2634 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2635 ]) 2636 .args(["--kernel", kernel_path.to_str().unwrap()]) 2637 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2638 .args(["--api-socket", &api_socket]) 2639 .capture_output() 2640 .default_disks() 2641 .default_net() 2642 .spawn() 2643 .unwrap(); 2644 2645 let r = std::panic::catch_unwind(|| { 2646 guest.wait_vm_boot(None).unwrap(); 2647 2648 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2649 2650 guest.enable_memory_hotplug(); 2651 2652 resize_zone_command(&api_socket, "mem0", "3G"); 2653 thread::sleep(std::time::Duration::new(5, 0)); 2654 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2655 resize_zone_command(&api_socket, "mem2", "3G"); 2656 thread::sleep(std::time::Duration::new(5, 0)); 2657 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2658 resize_zone_command(&api_socket, "mem0", "2G"); 2659 thread::sleep(std::time::Duration::new(5, 0)); 2660 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2661 resize_zone_command(&api_socket, "mem2", "2G"); 2662 thread::sleep(std::time::Duration::new(5, 0)); 2663 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2664 2665 guest.reboot_linux(0, None); 2666 2667 // Check the amount of RAM after reboot 2668 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2669 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2670 2671 // Check if we can still resize down to the initial 'boot'size 2672 resize_zone_command(&api_socket, "mem0", "1G"); 2673 thread::sleep(std::time::Duration::new(5, 0)); 2674 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2675 resize_zone_command(&api_socket, "mem2", "1G"); 2676 thread::sleep(std::time::Duration::new(5, 0)); 2677 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2678 }); 2679 2680 let _ = child.kill(); 2681 let output = child.wait_with_output().unwrap(); 2682 2683 handle_child_output(r, &output); 2684 } 2685 2686 #[test] 2687 #[cfg(not(feature = "mshv"))] 2688 fn test_guest_numa_nodes() { 2689 _test_guest_numa_nodes(false); 2690 } 2691 2692 #[test] 2693 #[cfg(target_arch = "x86_64")] 2694 fn test_iommu_segments() { 2695 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2696 let focal = UbuntuDiskConfig::new(focal_image); 2697 let guest = Guest::new(Box::new(focal)); 2698 2699 // Prepare another disk file for the virtio-disk device 2700 let test_disk_path = String::from( 2701 guest 2702 .tmp_dir 2703 .as_path() 2704 .join("test-disk.raw") 2705 .to_str() 2706 .unwrap(), 2707 ); 2708 assert!( 2709 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2710 ); 2711 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2712 2713 let api_socket = temp_api_path(&guest.tmp_dir); 2714 let mut cmd = GuestCommand::new(&guest); 2715 2716 cmd.args(["--cpus", "boot=1"]) 2717 .args(["--api-socket", &api_socket]) 2718 .args(["--memory", "size=512M"]) 2719 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2720 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2721 .args([ 2722 "--platform", 2723 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS},iommu_segments=[1]"), 2724 ]) 2725 .default_disks() 2726 .capture_output() 2727 .default_net(); 2728 2729 let mut child = cmd.spawn().unwrap(); 2730 2731 guest.wait_vm_boot(None).unwrap(); 2732 2733 let r = std::panic::catch_unwind(|| { 2734 let (cmd_success, cmd_output) = remote_command_w_output( 2735 &api_socket, 2736 "add-disk", 2737 Some( 2738 format!( 2739 "path={},id=test0,pci_segment=1,iommu=on", 2740 test_disk_path.as_str() 2741 ) 2742 .as_str(), 2743 ), 2744 ); 2745 assert!(cmd_success); 2746 assert!(String::from_utf8_lossy(&cmd_output) 2747 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2748 2749 // Check IOMMU setup 2750 assert!(guest 2751 .does_device_vendor_pair_match("0x1057", "0x1af4") 2752 .unwrap_or_default()); 2753 assert_eq!( 2754 guest 2755 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2756 .unwrap() 2757 .trim(), 2758 "0001:00:01.0" 2759 ); 2760 }); 2761 2762 let _ = child.kill(); 2763 let output = child.wait_with_output().unwrap(); 2764 2765 handle_child_output(r, &output); 2766 } 2767 2768 #[test] 2769 fn test_pci_msi() { 2770 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2771 let guest = Guest::new(Box::new(focal)); 2772 let mut cmd = GuestCommand::new(&guest); 2773 cmd.args(["--cpus", "boot=1"]) 2774 .args(["--memory", "size=512M"]) 2775 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2776 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2777 .capture_output() 2778 .default_disks() 2779 .default_net(); 2780 2781 let mut child = cmd.spawn().unwrap(); 2782 2783 guest.wait_vm_boot(None).unwrap(); 2784 2785 #[cfg(target_arch = "x86_64")] 2786 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2787 #[cfg(target_arch = "aarch64")] 2788 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2789 2790 let r = std::panic::catch_unwind(|| { 2791 assert_eq!( 2792 guest 2793 .ssh_command(grep_cmd) 2794 .unwrap() 2795 .trim() 2796 .parse::<u32>() 2797 .unwrap_or_default(), 2798 12 2799 ); 2800 }); 2801 2802 let _ = child.kill(); 2803 let output = child.wait_with_output().unwrap(); 2804 2805 handle_child_output(r, &output); 2806 } 2807 2808 #[test] 2809 fn test_virtio_net_ctrl_queue() { 2810 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2811 let guest = Guest::new(Box::new(focal)); 2812 let mut cmd = GuestCommand::new(&guest); 2813 cmd.args(["--cpus", "boot=1"]) 2814 .args(["--memory", "size=512M"]) 2815 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2816 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2817 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2818 .capture_output() 2819 .default_disks(); 2820 2821 let mut child = cmd.spawn().unwrap(); 2822 2823 guest.wait_vm_boot(None).unwrap(); 2824 2825 #[cfg(target_arch = "aarch64")] 2826 let iface = "enp0s4"; 2827 #[cfg(target_arch = "x86_64")] 2828 let iface = "ens4"; 2829 2830 let r = std::panic::catch_unwind(|| { 2831 assert_eq!( 2832 guest 2833 .ssh_command( 2834 format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() 2835 ) 2836 .unwrap() 2837 .trim(), 2838 "success" 2839 ); 2840 assert_eq!( 2841 guest 2842 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 2843 .unwrap() 2844 .trim(), 2845 "3000" 2846 ); 2847 }); 2848 2849 let _ = child.kill(); 2850 let output = child.wait_with_output().unwrap(); 2851 2852 handle_child_output(r, &output); 2853 } 2854 2855 #[test] 2856 fn test_pci_multiple_segments() { 2857 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2858 let guest = Guest::new(Box::new(focal)); 2859 2860 // Prepare another disk file for the virtio-disk device 2861 let test_disk_path = String::from( 2862 guest 2863 .tmp_dir 2864 .as_path() 2865 .join("test-disk.raw") 2866 .to_str() 2867 .unwrap(), 2868 ); 2869 assert!( 2870 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2871 ); 2872 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2873 2874 let mut cmd = GuestCommand::new(&guest); 2875 cmd.args(["--cpus", "boot=1"]) 2876 .args(["--memory", "size=512M"]) 2877 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2878 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2879 .args([ 2880 "--platform", 2881 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 2882 ]) 2883 .args([ 2884 "--disk", 2885 format!( 2886 "path={}", 2887 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2888 ) 2889 .as_str(), 2890 format!( 2891 "path={}", 2892 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2893 ) 2894 .as_str(), 2895 format!("path={test_disk_path},pci_segment=15").as_str(), 2896 ]) 2897 .capture_output() 2898 .default_net(); 2899 2900 let mut child = cmd.spawn().unwrap(); 2901 2902 guest.wait_vm_boot(None).unwrap(); 2903 2904 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2905 2906 let r = std::panic::catch_unwind(|| { 2907 // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. 2908 assert_eq!( 2909 guest 2910 .ssh_command(grep_cmd) 2911 .unwrap() 2912 .trim() 2913 .parse::<u16>() 2914 .unwrap_or_default(), 2915 MAX_NUM_PCI_SEGMENTS 2916 ); 2917 2918 // Check both if /dev/vdc exists and if the block size is 4M. 2919 assert_eq!( 2920 guest 2921 .ssh_command("lsblk | grep vdc | grep -c 4M") 2922 .unwrap() 2923 .trim() 2924 .parse::<u32>() 2925 .unwrap_or_default(), 2926 1 2927 ); 2928 2929 // Mount the device. 2930 guest.ssh_command("mkdir mount_image").unwrap(); 2931 guest 2932 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 2933 .unwrap(); 2934 // Grant all users with write permission. 2935 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 2936 2937 // Write something to the device. 2938 guest 2939 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 2940 .unwrap(); 2941 2942 // Check the content of the block device. The file "foo" should 2943 // contain "bar". 2944 assert_eq!( 2945 guest 2946 .ssh_command("sudo cat mount_image/foo") 2947 .unwrap() 2948 .trim(), 2949 "bar" 2950 ); 2951 }); 2952 2953 let _ = child.kill(); 2954 let output = child.wait_with_output().unwrap(); 2955 2956 handle_child_output(r, &output); 2957 } 2958 2959 #[test] 2960 fn test_pci_multiple_segments_numa_node() { 2961 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2962 let guest = Guest::new(Box::new(focal)); 2963 let api_socket = temp_api_path(&guest.tmp_dir); 2964 #[cfg(target_arch = "x86_64")] 2965 let kernel_path = direct_kernel_boot_path(); 2966 #[cfg(target_arch = "aarch64")] 2967 let kernel_path = edk2_path(); 2968 2969 // Prepare another disk file for the virtio-disk device 2970 let test_disk_path = String::from( 2971 guest 2972 .tmp_dir 2973 .as_path() 2974 .join("test-disk.raw") 2975 .to_str() 2976 .unwrap(), 2977 ); 2978 assert!( 2979 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2980 ); 2981 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2982 const TEST_DISK_NODE: u16 = 1; 2983 2984 let mut child = GuestCommand::new(&guest) 2985 .args(["--platform", "num_pci_segments=2"]) 2986 .args(["--cpus", "boot=2"]) 2987 .args(["--memory", "size=0"]) 2988 .args(["--memory-zone", "id=mem0,size=256M", "id=mem1,size=256M"]) 2989 .args([ 2990 "--numa", 2991 "guest_numa_id=0,cpus=[0],distances=[1@20],memory_zones=mem0,pci_segments=[0]", 2992 "guest_numa_id=1,cpus=[1],distances=[0@20],memory_zones=mem1,pci_segments=[1]", 2993 ]) 2994 .args(["--kernel", kernel_path.to_str().unwrap()]) 2995 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2996 .args(["--api-socket", &api_socket]) 2997 .capture_output() 2998 .args([ 2999 "--disk", 3000 format!( 3001 "path={}", 3002 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3003 ) 3004 .as_str(), 3005 format!( 3006 "path={}", 3007 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3008 ) 3009 .as_str(), 3010 format!("path={test_disk_path},pci_segment={TEST_DISK_NODE}").as_str(), 3011 ]) 3012 .default_net() 3013 .spawn() 3014 .unwrap(); 3015 3016 let cmd = "cat /sys/block/vdc/device/../numa_node"; 3017 3018 let r = std::panic::catch_unwind(|| { 3019 guest.wait_vm_boot(None).unwrap(); 3020 3021 assert_eq!( 3022 guest 3023 .ssh_command(cmd) 3024 .unwrap() 3025 .trim() 3026 .parse::<u16>() 3027 .unwrap_or_default(), 3028 TEST_DISK_NODE 3029 ); 3030 }); 3031 3032 let _ = child.kill(); 3033 let output = child.wait_with_output().unwrap(); 3034 3035 handle_child_output(r, &output); 3036 } 3037 3038 #[test] 3039 fn test_direct_kernel_boot() { 3040 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3041 let guest = Guest::new(Box::new(focal)); 3042 3043 let kernel_path = direct_kernel_boot_path(); 3044 3045 let mut child = GuestCommand::new(&guest) 3046 .args(["--cpus", "boot=1"]) 3047 .args(["--memory", "size=512M"]) 3048 .args(["--kernel", kernel_path.to_str().unwrap()]) 3049 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3050 .default_disks() 3051 .default_net() 3052 .capture_output() 3053 .spawn() 3054 .unwrap(); 3055 3056 let r = std::panic::catch_unwind(|| { 3057 guest.wait_vm_boot(None).unwrap(); 3058 3059 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3060 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3061 3062 let grep_cmd = if cfg!(target_arch = "x86_64") { 3063 "grep -c PCI-MSI /proc/interrupts" 3064 } else { 3065 "grep -c ITS-MSI /proc/interrupts" 3066 }; 3067 assert_eq!( 3068 guest 3069 .ssh_command(grep_cmd) 3070 .unwrap() 3071 .trim() 3072 .parse::<u32>() 3073 .unwrap_or_default(), 3074 12 3075 ); 3076 }); 3077 3078 let _ = child.kill(); 3079 let output = child.wait_with_output().unwrap(); 3080 3081 handle_child_output(r, &output); 3082 } 3083 3084 fn _test_virtio_block(image_name: &str, disable_io_uring: bool, disable_aio: bool) { 3085 let focal = UbuntuDiskConfig::new(image_name.to_string()); 3086 let guest = Guest::new(Box::new(focal)); 3087 3088 let mut workload_path = dirs::home_dir().unwrap(); 3089 workload_path.push("workloads"); 3090 3091 let mut blk_file_path = workload_path; 3092 blk_file_path.push("blk.img"); 3093 3094 let kernel_path = direct_kernel_boot_path(); 3095 3096 let mut cloud_child = GuestCommand::new(&guest) 3097 .args(["--cpus", "boot=4"]) 3098 .args(["--memory", "size=512M,shared=on"]) 3099 .args(["--kernel", kernel_path.to_str().unwrap()]) 3100 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3101 .args([ 3102 "--disk", 3103 format!( 3104 "path={}", 3105 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3106 ) 3107 .as_str(), 3108 format!( 3109 "path={}", 3110 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3111 ) 3112 .as_str(), 3113 format!( 3114 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", 3115 blk_file_path.to_str().unwrap(), 3116 disable_io_uring, 3117 disable_aio, 3118 ) 3119 .as_str(), 3120 ]) 3121 .default_net() 3122 .capture_output() 3123 .spawn() 3124 .unwrap(); 3125 3126 let r = std::panic::catch_unwind(|| { 3127 guest.wait_vm_boot(None).unwrap(); 3128 3129 // Check both if /dev/vdc exists and if the block size is 16M. 3130 assert_eq!( 3131 guest 3132 .ssh_command("lsblk | grep vdc | grep -c 16M") 3133 .unwrap() 3134 .trim() 3135 .parse::<u32>() 3136 .unwrap_or_default(), 3137 1 3138 ); 3139 3140 // Check both if /dev/vdc exists and if this block is RO. 3141 assert_eq!( 3142 guest 3143 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 3144 .unwrap() 3145 .trim() 3146 .parse::<u32>() 3147 .unwrap_or_default(), 3148 1 3149 ); 3150 3151 // Check if the number of queues is 4. 3152 assert_eq!( 3153 guest 3154 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 3155 .unwrap() 3156 .trim() 3157 .parse::<u32>() 3158 .unwrap_or_default(), 3159 4 3160 ); 3161 }); 3162 3163 let _ = cloud_child.kill(); 3164 let output = cloud_child.wait_with_output().unwrap(); 3165 3166 handle_child_output(r, &output); 3167 } 3168 3169 #[test] 3170 fn test_virtio_block_io_uring() { 3171 _test_virtio_block(FOCAL_IMAGE_NAME, false, true) 3172 } 3173 3174 #[test] 3175 fn test_virtio_block_aio() { 3176 _test_virtio_block(FOCAL_IMAGE_NAME, true, false) 3177 } 3178 3179 #[test] 3180 fn test_virtio_block_sync() { 3181 _test_virtio_block(FOCAL_IMAGE_NAME, true, true) 3182 } 3183 3184 #[test] 3185 fn test_virtio_block_qcow2() { 3186 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false, false) 3187 } 3188 3189 #[test] 3190 fn test_virtio_block_qcow2_backing_file() { 3191 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE, false, false) 3192 } 3193 3194 #[test] 3195 fn test_virtio_block_vhd() { 3196 let mut workload_path = dirs::home_dir().unwrap(); 3197 workload_path.push("workloads"); 3198 3199 let mut raw_file_path = workload_path.clone(); 3200 let mut vhd_file_path = workload_path; 3201 raw_file_path.push(FOCAL_IMAGE_NAME); 3202 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 3203 3204 // Generate VHD file from RAW file 3205 std::process::Command::new("qemu-img") 3206 .arg("convert") 3207 .arg("-p") 3208 .args(["-f", "raw"]) 3209 .args(["-O", "vpc"]) 3210 .args(["-o", "subformat=fixed"]) 3211 .arg(raw_file_path.to_str().unwrap()) 3212 .arg(vhd_file_path.to_str().unwrap()) 3213 .output() 3214 .expect("Expect generating VHD image from RAW image"); 3215 3216 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false, false) 3217 } 3218 3219 #[test] 3220 fn test_virtio_block_vhdx() { 3221 let mut workload_path = dirs::home_dir().unwrap(); 3222 workload_path.push("workloads"); 3223 3224 let mut raw_file_path = workload_path.clone(); 3225 let mut vhdx_file_path = workload_path; 3226 raw_file_path.push(FOCAL_IMAGE_NAME); 3227 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 3228 3229 // Generate dynamic VHDX file from RAW file 3230 std::process::Command::new("qemu-img") 3231 .arg("convert") 3232 .arg("-p") 3233 .args(["-f", "raw"]) 3234 .args(["-O", "vhdx"]) 3235 .arg(raw_file_path.to_str().unwrap()) 3236 .arg(vhdx_file_path.to_str().unwrap()) 3237 .output() 3238 .expect("Expect generating dynamic VHDx image from RAW image"); 3239 3240 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false, false) 3241 } 3242 3243 #[test] 3244 fn test_virtio_block_dynamic_vhdx_expand() { 3245 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 3246 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 3247 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 3248 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 3249 3250 let mut workload_path = dirs::home_dir().unwrap(); 3251 workload_path.push("workloads"); 3252 3253 let mut vhdx_file_path = workload_path; 3254 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 3255 let vhdx_path = vhdx_file_path.to_str().unwrap(); 3256 3257 // Generate a 100 MiB dynamic VHDX file 3258 std::process::Command::new("qemu-img") 3259 .arg("create") 3260 .args(["-f", "vhdx"]) 3261 .arg(vhdx_path) 3262 .arg(VIRTUAL_DISK_SIZE.to_string()) 3263 .output() 3264 .expect("Expect generating dynamic VHDx image from RAW image"); 3265 3266 // Check if the size matches with empty VHDx file size 3267 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 3268 3269 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3270 let guest = Guest::new(Box::new(focal)); 3271 let kernel_path = direct_kernel_boot_path(); 3272 3273 let mut cloud_child = GuestCommand::new(&guest) 3274 .args(["--cpus", "boot=1"]) 3275 .args(["--memory", "size=512M"]) 3276 .args(["--kernel", kernel_path.to_str().unwrap()]) 3277 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3278 .args([ 3279 "--disk", 3280 format!( 3281 "path={}", 3282 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3283 ) 3284 .as_str(), 3285 format!( 3286 "path={}", 3287 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3288 ) 3289 .as_str(), 3290 format!("path={vhdx_path}").as_str(), 3291 ]) 3292 .default_net() 3293 .capture_output() 3294 .spawn() 3295 .unwrap(); 3296 3297 let r = std::panic::catch_unwind(|| { 3298 guest.wait_vm_boot(None).unwrap(); 3299 3300 // Check both if /dev/vdc exists and if the block size is 100 MiB. 3301 assert_eq!( 3302 guest 3303 .ssh_command("lsblk | grep vdc | grep -c 100M") 3304 .unwrap() 3305 .trim() 3306 .parse::<u32>() 3307 .unwrap_or_default(), 3308 1 3309 ); 3310 3311 // Write 100 MB of data to the VHDx disk 3312 guest 3313 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 3314 .unwrap(); 3315 }); 3316 3317 // Check if the size matches with expected expanded VHDx file size 3318 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 3319 3320 let _ = cloud_child.kill(); 3321 let output = cloud_child.wait_with_output().unwrap(); 3322 3323 handle_child_output(r, &output); 3324 } 3325 3326 fn vhdx_image_size(disk_name: &str) -> u64 { 3327 std::fs::File::open(disk_name) 3328 .unwrap() 3329 .seek(SeekFrom::End(0)) 3330 .unwrap() 3331 } 3332 3333 #[test] 3334 fn test_virtio_block_direct_and_firmware() { 3335 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3336 let guest = Guest::new(Box::new(focal)); 3337 3338 // The OS disk must be copied to a location that is not backed by 3339 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 3340 // with EINVAL because tmpfs doesn't support this flag. 3341 let mut workloads_path = dirs::home_dir().unwrap(); 3342 workloads_path.push("workloads"); 3343 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 3344 let mut os_path = os_dir.as_path().to_path_buf(); 3345 os_path.push("osdisk.img"); 3346 rate_limited_copy( 3347 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3348 os_path.as_path(), 3349 ) 3350 .expect("copying of OS disk failed"); 3351 3352 let mut child = GuestCommand::new(&guest) 3353 .args(["--cpus", "boot=1"]) 3354 .args(["--memory", "size=512M"]) 3355 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 3356 .args([ 3357 "--disk", 3358 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 3359 format!( 3360 "path={}", 3361 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3362 ) 3363 .as_str(), 3364 ]) 3365 .default_net() 3366 .capture_output() 3367 .spawn() 3368 .unwrap(); 3369 3370 let r = std::panic::catch_unwind(|| { 3371 guest.wait_vm_boot(Some(120)).unwrap(); 3372 }); 3373 3374 let _ = child.kill(); 3375 let output = child.wait_with_output().unwrap(); 3376 3377 handle_child_output(r, &output); 3378 } 3379 3380 #[test] 3381 fn test_vhost_user_net_default() { 3382 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 3383 } 3384 3385 #[test] 3386 fn test_vhost_user_net_named_tap() { 3387 test_vhost_user_net( 3388 Some("mytap0"), 3389 2, 3390 &prepare_vhost_user_net_daemon, 3391 false, 3392 false, 3393 ) 3394 } 3395 3396 #[test] 3397 fn test_vhost_user_net_existing_tap() { 3398 test_vhost_user_net( 3399 Some("vunet-tap0"), 3400 2, 3401 &prepare_vhost_user_net_daemon, 3402 false, 3403 false, 3404 ) 3405 } 3406 3407 #[test] 3408 fn test_vhost_user_net_multiple_queues() { 3409 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 3410 } 3411 3412 #[test] 3413 fn test_vhost_user_net_tap_multiple_queues() { 3414 test_vhost_user_net( 3415 Some("vunet-tap1"), 3416 4, 3417 &prepare_vhost_user_net_daemon, 3418 false, 3419 false, 3420 ) 3421 } 3422 3423 #[test] 3424 fn test_vhost_user_net_host_mac() { 3425 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 3426 } 3427 3428 #[test] 3429 fn test_vhost_user_net_client_mode() { 3430 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 3431 } 3432 3433 #[test] 3434 #[cfg(not(target_arch = "aarch64"))] 3435 fn test_vhost_user_blk_default() { 3436 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 3437 } 3438 3439 #[test] 3440 #[cfg(not(target_arch = "aarch64"))] 3441 fn test_vhost_user_blk_readonly() { 3442 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3443 } 3444 3445 #[test] 3446 #[cfg(not(target_arch = "aarch64"))] 3447 fn test_vhost_user_blk_direct() { 3448 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3449 } 3450 3451 #[test] 3452 fn test_boot_from_vhost_user_blk_default() { 3453 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3454 } 3455 3456 #[test] 3457 #[cfg(target_arch = "x86_64")] 3458 fn test_split_irqchip() { 3459 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3460 let guest = Guest::new(Box::new(focal)); 3461 3462 let mut child = GuestCommand::new(&guest) 3463 .args(["--cpus", "boot=1"]) 3464 .args(["--memory", "size=512M"]) 3465 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3466 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3467 .default_disks() 3468 .default_net() 3469 .capture_output() 3470 .spawn() 3471 .unwrap(); 3472 3473 let r = std::panic::catch_unwind(|| { 3474 guest.wait_vm_boot(None).unwrap(); 3475 3476 assert_eq!( 3477 guest 3478 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3479 .unwrap() 3480 .trim() 3481 .parse::<u32>() 3482 .unwrap_or(1), 3483 0 3484 ); 3485 assert_eq!( 3486 guest 3487 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3488 .unwrap() 3489 .trim() 3490 .parse::<u32>() 3491 .unwrap_or(1), 3492 0 3493 ); 3494 }); 3495 3496 let _ = child.kill(); 3497 let output = child.wait_with_output().unwrap(); 3498 3499 handle_child_output(r, &output); 3500 } 3501 3502 #[test] 3503 #[cfg(target_arch = "x86_64")] 3504 fn test_dmi_serial_number() { 3505 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3506 let guest = Guest::new(Box::new(focal)); 3507 3508 let mut child = GuestCommand::new(&guest) 3509 .args(["--cpus", "boot=1"]) 3510 .args(["--memory", "size=512M"]) 3511 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3512 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3513 .args(["--platform", "serial_number=a=b;c=d"]) 3514 .default_disks() 3515 .default_net() 3516 .capture_output() 3517 .spawn() 3518 .unwrap(); 3519 3520 let r = std::panic::catch_unwind(|| { 3521 guest.wait_vm_boot(None).unwrap(); 3522 3523 assert_eq!( 3524 guest 3525 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3526 .unwrap() 3527 .trim(), 3528 "a=b;c=d" 3529 ); 3530 }); 3531 3532 let _ = child.kill(); 3533 let output = child.wait_with_output().unwrap(); 3534 3535 handle_child_output(r, &output); 3536 } 3537 3538 #[test] 3539 #[cfg(target_arch = "x86_64")] 3540 fn test_dmi_uuid() { 3541 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3542 let guest = Guest::new(Box::new(focal)); 3543 3544 let mut child = GuestCommand::new(&guest) 3545 .args(["--cpus", "boot=1"]) 3546 .args(["--memory", "size=512M"]) 3547 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3548 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3549 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3550 .default_disks() 3551 .default_net() 3552 .capture_output() 3553 .spawn() 3554 .unwrap(); 3555 3556 let r = std::panic::catch_unwind(|| { 3557 guest.wait_vm_boot(None).unwrap(); 3558 3559 assert_eq!( 3560 guest 3561 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3562 .unwrap() 3563 .trim(), 3564 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3565 ); 3566 }); 3567 3568 let _ = child.kill(); 3569 let output = child.wait_with_output().unwrap(); 3570 3571 handle_child_output(r, &output); 3572 } 3573 3574 #[test] 3575 #[cfg(target_arch = "x86_64")] 3576 fn test_dmi_oem_strings() { 3577 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3578 let guest = Guest::new(Box::new(focal)); 3579 3580 let s1 = "io.systemd.credential:xx=yy"; 3581 let s2 = "This is a test string"; 3582 3583 let oem_strings = format!("oem_strings=[{s1},{s2}]"); 3584 3585 let mut child = GuestCommand::new(&guest) 3586 .args(["--cpus", "boot=1"]) 3587 .args(["--memory", "size=512M"]) 3588 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3589 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3590 .args(["--platform", &oem_strings]) 3591 .default_disks() 3592 .default_net() 3593 .capture_output() 3594 .spawn() 3595 .unwrap(); 3596 3597 let r = std::panic::catch_unwind(|| { 3598 guest.wait_vm_boot(None).unwrap(); 3599 3600 assert_eq!( 3601 guest 3602 .ssh_command("sudo dmidecode --oem-string count") 3603 .unwrap() 3604 .trim(), 3605 "2" 3606 ); 3607 3608 assert_eq!( 3609 guest 3610 .ssh_command("sudo dmidecode --oem-string 1") 3611 .unwrap() 3612 .trim(), 3613 s1 3614 ); 3615 3616 assert_eq!( 3617 guest 3618 .ssh_command("sudo dmidecode --oem-string 2") 3619 .unwrap() 3620 .trim(), 3621 s2 3622 ); 3623 }); 3624 3625 let _ = child.kill(); 3626 let output = child.wait_with_output().unwrap(); 3627 3628 handle_child_output(r, &output); 3629 } 3630 3631 #[test] 3632 fn test_virtio_fs() { 3633 _test_virtio_fs(&prepare_virtiofsd, false, None) 3634 } 3635 3636 #[test] 3637 fn test_virtio_fs_hotplug() { 3638 _test_virtio_fs(&prepare_virtiofsd, true, None) 3639 } 3640 3641 #[test] 3642 #[cfg(not(feature = "mshv"))] 3643 fn test_virtio_fs_multi_segment_hotplug() { 3644 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3645 } 3646 3647 #[test] 3648 #[cfg(not(feature = "mshv"))] 3649 fn test_virtio_fs_multi_segment() { 3650 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3651 } 3652 3653 #[test] 3654 fn test_virtio_pmem_persist_writes() { 3655 test_virtio_pmem(false, false) 3656 } 3657 3658 #[test] 3659 fn test_virtio_pmem_discard_writes() { 3660 test_virtio_pmem(true, false) 3661 } 3662 3663 #[test] 3664 fn test_virtio_pmem_with_size() { 3665 test_virtio_pmem(true, true) 3666 } 3667 3668 #[test] 3669 fn test_boot_from_virtio_pmem() { 3670 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3671 let guest = Guest::new(Box::new(focal)); 3672 3673 let kernel_path = direct_kernel_boot_path(); 3674 3675 let mut child = GuestCommand::new(&guest) 3676 .args(["--cpus", "boot=1"]) 3677 .args(["--memory", "size=512M"]) 3678 .args(["--kernel", kernel_path.to_str().unwrap()]) 3679 .args([ 3680 "--disk", 3681 format!( 3682 "path={}", 3683 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3684 ) 3685 .as_str(), 3686 ]) 3687 .default_net() 3688 .args([ 3689 "--pmem", 3690 format!( 3691 "file={},size={}", 3692 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3693 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3694 .unwrap() 3695 .len() 3696 ) 3697 .as_str(), 3698 ]) 3699 .args([ 3700 "--cmdline", 3701 DIRECT_KERNEL_BOOT_CMDLINE 3702 .replace("vda1", "pmem0p1") 3703 .as_str(), 3704 ]) 3705 .capture_output() 3706 .spawn() 3707 .unwrap(); 3708 3709 let r = std::panic::catch_unwind(|| { 3710 guest.wait_vm_boot(None).unwrap(); 3711 3712 // Simple checks to validate the VM booted properly 3713 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3714 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3715 }); 3716 3717 let _ = child.kill(); 3718 let output = child.wait_with_output().unwrap(); 3719 3720 handle_child_output(r, &output); 3721 } 3722 3723 #[test] 3724 fn test_multiple_network_interfaces() { 3725 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3726 let guest = Guest::new(Box::new(focal)); 3727 3728 let kernel_path = direct_kernel_boot_path(); 3729 3730 let mut child = GuestCommand::new(&guest) 3731 .args(["--cpus", "boot=1"]) 3732 .args(["--memory", "size=512M"]) 3733 .args(["--kernel", kernel_path.to_str().unwrap()]) 3734 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3735 .default_disks() 3736 .args([ 3737 "--net", 3738 guest.default_net_string().as_str(), 3739 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3740 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3741 ]) 3742 .capture_output() 3743 .spawn() 3744 .unwrap(); 3745 3746 let r = std::panic::catch_unwind(|| { 3747 guest.wait_vm_boot(None).unwrap(); 3748 3749 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3750 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3751 3752 // 3 network interfaces + default localhost ==> 4 interfaces 3753 assert_eq!( 3754 guest 3755 .ssh_command("ip -o link | wc -l") 3756 .unwrap() 3757 .trim() 3758 .parse::<u32>() 3759 .unwrap_or_default(), 3760 4 3761 ); 3762 }); 3763 3764 let _ = child.kill(); 3765 let output = child.wait_with_output().unwrap(); 3766 3767 handle_child_output(r, &output); 3768 } 3769 3770 #[test] 3771 #[cfg(target_arch = "aarch64")] 3772 fn test_pmu_on() { 3773 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3774 let guest = Guest::new(Box::new(focal)); 3775 let mut child = GuestCommand::new(&guest) 3776 .args(["--cpus", "boot=1"]) 3777 .args(["--memory", "size=512M"]) 3778 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3779 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3780 .default_disks() 3781 .default_net() 3782 .capture_output() 3783 .spawn() 3784 .unwrap(); 3785 3786 let r = std::panic::catch_unwind(|| { 3787 guest.wait_vm_boot(None).unwrap(); 3788 3789 // Test that PMU exists. 3790 assert_eq!( 3791 guest 3792 .ssh_command(GREP_PMU_IRQ_CMD) 3793 .unwrap() 3794 .trim() 3795 .parse::<u32>() 3796 .unwrap_or_default(), 3797 1 3798 ); 3799 }); 3800 3801 let _ = child.kill(); 3802 let output = child.wait_with_output().unwrap(); 3803 3804 handle_child_output(r, &output); 3805 } 3806 3807 #[test] 3808 fn test_serial_off() { 3809 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3810 let guest = Guest::new(Box::new(focal)); 3811 let mut child = GuestCommand::new(&guest) 3812 .args(["--cpus", "boot=1"]) 3813 .args(["--memory", "size=512M"]) 3814 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3815 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3816 .default_disks() 3817 .default_net() 3818 .args(["--serial", "off"]) 3819 .capture_output() 3820 .spawn() 3821 .unwrap(); 3822 3823 let r = std::panic::catch_unwind(|| { 3824 guest.wait_vm_boot(None).unwrap(); 3825 3826 // Test that there is no ttyS0 3827 assert_eq!( 3828 guest 3829 .ssh_command(GREP_SERIAL_IRQ_CMD) 3830 .unwrap() 3831 .trim() 3832 .parse::<u32>() 3833 .unwrap_or(1), 3834 0 3835 ); 3836 }); 3837 3838 let _ = child.kill(); 3839 let output = child.wait_with_output().unwrap(); 3840 3841 handle_child_output(r, &output); 3842 } 3843 3844 #[test] 3845 fn test_serial_null() { 3846 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3847 let guest = Guest::new(Box::new(focal)); 3848 let mut cmd = GuestCommand::new(&guest); 3849 #[cfg(target_arch = "x86_64")] 3850 let console_str: &str = "console=ttyS0"; 3851 #[cfg(target_arch = "aarch64")] 3852 let console_str: &str = "console=ttyAMA0"; 3853 3854 cmd.args(["--cpus", "boot=1"]) 3855 .args(["--memory", "size=512M"]) 3856 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3857 .args([ 3858 "--cmdline", 3859 DIRECT_KERNEL_BOOT_CMDLINE 3860 .replace("console=hvc0 ", console_str) 3861 .as_str(), 3862 ]) 3863 .default_disks() 3864 .default_net() 3865 .args(["--serial", "null"]) 3866 .args(["--console", "off"]) 3867 .capture_output(); 3868 3869 let mut child = cmd.spawn().unwrap(); 3870 3871 let r = std::panic::catch_unwind(|| { 3872 guest.wait_vm_boot(None).unwrap(); 3873 3874 // Test that there is a ttyS0 3875 assert_eq!( 3876 guest 3877 .ssh_command(GREP_SERIAL_IRQ_CMD) 3878 .unwrap() 3879 .trim() 3880 .parse::<u32>() 3881 .unwrap_or_default(), 3882 1 3883 ); 3884 }); 3885 3886 let _ = child.kill(); 3887 let output = child.wait_with_output().unwrap(); 3888 handle_child_output(r, &output); 3889 3890 let r = std::panic::catch_unwind(|| { 3891 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3892 }); 3893 3894 handle_child_output(r, &output); 3895 } 3896 3897 #[test] 3898 fn test_serial_tty() { 3899 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3900 let guest = Guest::new(Box::new(focal)); 3901 3902 let kernel_path = direct_kernel_boot_path(); 3903 3904 #[cfg(target_arch = "x86_64")] 3905 let console_str: &str = "console=ttyS0"; 3906 #[cfg(target_arch = "aarch64")] 3907 let console_str: &str = "console=ttyAMA0"; 3908 3909 let mut child = GuestCommand::new(&guest) 3910 .args(["--cpus", "boot=1"]) 3911 .args(["--memory", "size=512M"]) 3912 .args(["--kernel", kernel_path.to_str().unwrap()]) 3913 .args([ 3914 "--cmdline", 3915 DIRECT_KERNEL_BOOT_CMDLINE 3916 .replace("console=hvc0 ", console_str) 3917 .as_str(), 3918 ]) 3919 .default_disks() 3920 .default_net() 3921 .args(["--serial", "tty"]) 3922 .args(["--console", "off"]) 3923 .capture_output() 3924 .spawn() 3925 .unwrap(); 3926 3927 let r = std::panic::catch_unwind(|| { 3928 guest.wait_vm_boot(None).unwrap(); 3929 3930 // Test that there is a ttyS0 3931 assert_eq!( 3932 guest 3933 .ssh_command(GREP_SERIAL_IRQ_CMD) 3934 .unwrap() 3935 .trim() 3936 .parse::<u32>() 3937 .unwrap_or_default(), 3938 1 3939 ); 3940 }); 3941 3942 // This sleep is needed to wait for the login prompt 3943 thread::sleep(std::time::Duration::new(2, 0)); 3944 3945 let _ = child.kill(); 3946 let output = child.wait_with_output().unwrap(); 3947 handle_child_output(r, &output); 3948 3949 let r = std::panic::catch_unwind(|| { 3950 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 3951 }); 3952 3953 handle_child_output(r, &output); 3954 } 3955 3956 #[test] 3957 fn test_serial_file() { 3958 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3959 let guest = Guest::new(Box::new(focal)); 3960 3961 let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output"); 3962 #[cfg(target_arch = "x86_64")] 3963 let console_str: &str = "console=ttyS0"; 3964 #[cfg(target_arch = "aarch64")] 3965 let console_str: &str = "console=ttyAMA0"; 3966 3967 let mut child = GuestCommand::new(&guest) 3968 .args(["--cpus", "boot=1"]) 3969 .args(["--memory", "size=512M"]) 3970 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3971 .args([ 3972 "--cmdline", 3973 DIRECT_KERNEL_BOOT_CMDLINE 3974 .replace("console=hvc0 ", console_str) 3975 .as_str(), 3976 ]) 3977 .default_disks() 3978 .default_net() 3979 .args([ 3980 "--serial", 3981 format!("file={}", serial_path.to_str().unwrap()).as_str(), 3982 ]) 3983 .capture_output() 3984 .spawn() 3985 .unwrap(); 3986 3987 let r = std::panic::catch_unwind(|| { 3988 guest.wait_vm_boot(None).unwrap(); 3989 3990 // Test that there is a ttyS0 3991 assert_eq!( 3992 guest 3993 .ssh_command(GREP_SERIAL_IRQ_CMD) 3994 .unwrap() 3995 .trim() 3996 .parse::<u32>() 3997 .unwrap_or_default(), 3998 1 3999 ); 4000 4001 guest.ssh_command("sudo shutdown -h now").unwrap(); 4002 }); 4003 4004 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4005 let _ = child.kill(); 4006 let output = child.wait_with_output().unwrap(); 4007 handle_child_output(r, &output); 4008 4009 let r = std::panic::catch_unwind(|| { 4010 // Check that the cloud-hypervisor binary actually terminated 4011 assert!(output.status.success()); 4012 4013 // Do this check after shutdown of the VM as an easy way to ensure 4014 // all writes are flushed to disk 4015 let mut f = std::fs::File::open(serial_path).unwrap(); 4016 let mut buf = String::new(); 4017 f.read_to_string(&mut buf).unwrap(); 4018 assert!(buf.contains(CONSOLE_TEST_STRING)); 4019 }); 4020 4021 handle_child_output(r, &output); 4022 } 4023 4024 #[test] 4025 fn test_pty_interaction() { 4026 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4027 let guest = Guest::new(Box::new(focal)); 4028 let api_socket = temp_api_path(&guest.tmp_dir); 4029 let serial_option = if cfg!(target_arch = "x86_64") { 4030 " console=ttyS0" 4031 } else { 4032 " console=ttyAMA0" 4033 }; 4034 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4035 4036 let mut child = GuestCommand::new(&guest) 4037 .args(["--cpus", "boot=1"]) 4038 .args(["--memory", "size=512M"]) 4039 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4040 .args(["--cmdline", &cmdline]) 4041 .default_disks() 4042 .default_net() 4043 .args(["--serial", "null"]) 4044 .args(["--console", "pty"]) 4045 .args(["--api-socket", &api_socket]) 4046 .spawn() 4047 .unwrap(); 4048 4049 let r = std::panic::catch_unwind(|| { 4050 guest.wait_vm_boot(None).unwrap(); 4051 // Get pty fd for console 4052 let console_path = get_pty_path(&api_socket, "console"); 4053 _test_pty_interaction(console_path); 4054 4055 guest.ssh_command("sudo shutdown -h now").unwrap(); 4056 }); 4057 4058 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4059 let _ = child.kill(); 4060 let output = child.wait_with_output().unwrap(); 4061 handle_child_output(r, &output); 4062 4063 let r = std::panic::catch_unwind(|| { 4064 // Check that the cloud-hypervisor binary actually terminated 4065 assert!(output.status.success()) 4066 }); 4067 handle_child_output(r, &output); 4068 } 4069 4070 #[test] 4071 fn test_serial_socket_interaction() { 4072 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4073 let guest = Guest::new(Box::new(focal)); 4074 let serial_socket = guest.tmp_dir.as_path().join("/tmp/serial.socket"); 4075 let serial_socket_pty = guest.tmp_dir.as_path().join("/tmp/serial.pty"); 4076 let serial_option = if cfg!(target_arch = "x86_64") { 4077 " console=ttyS0" 4078 } else { 4079 " console=ttyAMA0" 4080 }; 4081 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4082 4083 let mut child = GuestCommand::new(&guest) 4084 .args(["--cpus", "boot=1"]) 4085 .args(["--memory", "size=512M"]) 4086 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4087 .args(["--cmdline", &cmdline]) 4088 .default_disks() 4089 .default_net() 4090 .args(["--console", "null"]) 4091 .args([ 4092 "--serial", 4093 format!("socket={}", serial_socket.to_str().unwrap()).as_str(), 4094 ]) 4095 .spawn() 4096 .unwrap(); 4097 4098 let _ = std::panic::catch_unwind(|| { 4099 guest.wait_vm_boot(None).unwrap(); 4100 }); 4101 4102 let mut socat_command = Command::new("socat"); 4103 let socat_args = [ 4104 &format!("pty,link={},raw", serial_socket_pty.display()), 4105 &format!("UNIX-CONNECT:{}", serial_socket.display()), 4106 ]; 4107 socat_command.args(socat_args); 4108 4109 let mut socat_child = socat_command.spawn().unwrap(); 4110 thread::sleep(std::time::Duration::new(1, 0)); 4111 4112 let _ = std::panic::catch_unwind(|| { 4113 _test_pty_interaction(serial_socket_pty); 4114 }); 4115 4116 let _ = socat_child.kill(); 4117 4118 let r = std::panic::catch_unwind(|| { 4119 guest.ssh_command("sudo shutdown -h now").unwrap(); 4120 }); 4121 4122 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4123 let _ = child.kill(); 4124 let output = child.wait_with_output().unwrap(); 4125 handle_child_output(r, &output); 4126 4127 let r = std::panic::catch_unwind(|| { 4128 // Check that the cloud-hypervisor binary actually terminated 4129 assert!(output.status.success()) 4130 }); 4131 handle_child_output(r, &output); 4132 } 4133 4134 #[test] 4135 fn test_virtio_console() { 4136 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4137 let guest = Guest::new(Box::new(focal)); 4138 4139 let kernel_path = direct_kernel_boot_path(); 4140 4141 let mut child = GuestCommand::new(&guest) 4142 .args(["--cpus", "boot=1"]) 4143 .args(["--memory", "size=512M"]) 4144 .args(["--kernel", kernel_path.to_str().unwrap()]) 4145 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4146 .default_disks() 4147 .default_net() 4148 .args(["--console", "tty"]) 4149 .args(["--serial", "null"]) 4150 .capture_output() 4151 .spawn() 4152 .unwrap(); 4153 4154 let text = String::from("On a branch floating down river a cricket, singing."); 4155 let cmd = format!("echo {text} | sudo tee /dev/hvc0"); 4156 4157 let r = std::panic::catch_unwind(|| { 4158 guest.wait_vm_boot(None).unwrap(); 4159 4160 assert!(guest 4161 .does_device_vendor_pair_match("0x1043", "0x1af4") 4162 .unwrap_or_default()); 4163 4164 guest.ssh_command(&cmd).unwrap(); 4165 }); 4166 4167 let _ = child.kill(); 4168 let output = child.wait_with_output().unwrap(); 4169 handle_child_output(r, &output); 4170 4171 let r = std::panic::catch_unwind(|| { 4172 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 4173 }); 4174 4175 handle_child_output(r, &output); 4176 } 4177 4178 #[test] 4179 fn test_console_file() { 4180 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4181 let guest = Guest::new(Box::new(focal)); 4182 4183 let console_path = guest.tmp_dir.as_path().join("/tmp/console-output"); 4184 let mut child = GuestCommand::new(&guest) 4185 .args(["--cpus", "boot=1"]) 4186 .args(["--memory", "size=512M"]) 4187 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4188 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4189 .default_disks() 4190 .default_net() 4191 .args([ 4192 "--console", 4193 format!("file={}", console_path.to_str().unwrap()).as_str(), 4194 ]) 4195 .capture_output() 4196 .spawn() 4197 .unwrap(); 4198 4199 guest.wait_vm_boot(None).unwrap(); 4200 4201 guest.ssh_command("sudo shutdown -h now").unwrap(); 4202 4203 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4204 let _ = child.kill(); 4205 let output = child.wait_with_output().unwrap(); 4206 4207 let r = std::panic::catch_unwind(|| { 4208 // Check that the cloud-hypervisor binary actually terminated 4209 assert!(output.status.success()); 4210 4211 // Do this check after shutdown of the VM as an easy way to ensure 4212 // all writes are flushed to disk 4213 let mut f = std::fs::File::open(console_path).unwrap(); 4214 let mut buf = String::new(); 4215 f.read_to_string(&mut buf).unwrap(); 4216 4217 if !buf.contains(CONSOLE_TEST_STRING) { 4218 eprintln!( 4219 "\n\n==== Console file output ====\n\n{buf}\n\n==== End console file output ====" 4220 ); 4221 } 4222 assert!(buf.contains(CONSOLE_TEST_STRING)); 4223 }); 4224 4225 handle_child_output(r, &output); 4226 } 4227 4228 #[test] 4229 #[cfg(target_arch = "x86_64")] 4230 #[cfg(not(feature = "mshv"))] 4231 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 4232 // backed networking interfaces, bound through a simple bridge on the host. 4233 // So if the nested cloud-hypervisor succeeds in getting a directly 4234 // assigned interface from its cloud-hypervisor host, we should be able to 4235 // ssh into it, and verify that it's running with the right kernel command 4236 // line (We tag the command line from cloud-hypervisor for that purpose). 4237 // The third device is added to validate that hotplug works correctly since 4238 // it is being added to the L2 VM through hotplugging mechanism. 4239 // Also, we pass-through a virtio-blk device to the L2 VM to test the 32-bit 4240 // vfio device support 4241 fn test_vfio() { 4242 setup_vfio_network_interfaces(); 4243 4244 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4245 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 4246 4247 let mut workload_path = dirs::home_dir().unwrap(); 4248 workload_path.push("workloads"); 4249 4250 let kernel_path = direct_kernel_boot_path(); 4251 4252 let mut vfio_path = workload_path.clone(); 4253 vfio_path.push("vfio"); 4254 4255 let mut cloud_init_vfio_base_path = vfio_path.clone(); 4256 cloud_init_vfio_base_path.push("cloudinit.img"); 4257 4258 // We copy our cloudinit into the vfio mount point, for the nested 4259 // cloud-hypervisor guest to use. 4260 rate_limited_copy( 4261 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 4262 &cloud_init_vfio_base_path, 4263 ) 4264 .expect("copying of cloud-init disk failed"); 4265 4266 let mut vfio_disk_path = workload_path.clone(); 4267 vfio_disk_path.push("vfio.img"); 4268 4269 // Create the vfio disk image 4270 let output = Command::new("mkfs.ext4") 4271 .arg("-d") 4272 .arg(vfio_path.to_str().unwrap()) 4273 .arg(vfio_disk_path.to_str().unwrap()) 4274 .arg("2g") 4275 .output() 4276 .unwrap(); 4277 if !output.status.success() { 4278 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 4279 panic!("mkfs.ext4 command generated an error"); 4280 } 4281 4282 let mut blk_file_path = workload_path; 4283 blk_file_path.push("blk.img"); 4284 4285 let vfio_tap0 = "vfio-tap0"; 4286 let vfio_tap1 = "vfio-tap1"; 4287 let vfio_tap2 = "vfio-tap2"; 4288 let vfio_tap3 = "vfio-tap3"; 4289 4290 let mut child = GuestCommand::new(&guest) 4291 .args(["--cpus", "boot=4"]) 4292 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 4293 .args(["--kernel", kernel_path.to_str().unwrap()]) 4294 .args([ 4295 "--disk", 4296 format!( 4297 "path={}", 4298 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4299 ) 4300 .as_str(), 4301 format!( 4302 "path={}", 4303 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4304 ) 4305 .as_str(), 4306 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 4307 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 4308 ]) 4309 .args([ 4310 "--cmdline", 4311 format!( 4312 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 4313 ) 4314 .as_str(), 4315 ]) 4316 .args([ 4317 "--net", 4318 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 4319 format!( 4320 "tap={},mac={},iommu=on", 4321 vfio_tap1, guest.network.l2_guest_mac1 4322 ) 4323 .as_str(), 4324 format!( 4325 "tap={},mac={},iommu=on", 4326 vfio_tap2, guest.network.l2_guest_mac2 4327 ) 4328 .as_str(), 4329 format!( 4330 "tap={},mac={},iommu=on", 4331 vfio_tap3, guest.network.l2_guest_mac3 4332 ) 4333 .as_str(), 4334 ]) 4335 .capture_output() 4336 .spawn() 4337 .unwrap(); 4338 4339 thread::sleep(std::time::Duration::new(30, 0)); 4340 4341 let r = std::panic::catch_unwind(|| { 4342 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 4343 thread::sleep(std::time::Duration::new(120, 0)); 4344 4345 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 4346 // added to its kernel command line. 4347 // Let's ssh into it and verify that it's there. If it is it means 4348 // we're in the right guest (The L2 one) because the QEMU L1 guest 4349 // does not have this command line tag. 4350 assert!(check_matched_lines_count( 4351 guest.ssh_command_l2_1("cat /proc/cmdline").unwrap().trim(), 4352 vec!["VFIOTAG"], 4353 1 4354 )); 4355 4356 // Let's also verify from the second virtio-net device passed to 4357 // the L2 VM. 4358 assert!(check_matched_lines_count( 4359 guest.ssh_command_l2_2("cat /proc/cmdline").unwrap().trim(), 4360 vec!["VFIOTAG"], 4361 1 4362 )); 4363 4364 // Check the amount of PCI devices appearing in L2 VM. 4365 assert!(check_lines_count( 4366 guest 4367 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4368 .unwrap() 4369 .trim(), 4370 8 4371 )); 4372 4373 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 4374 assert!(check_matched_lines_count( 4375 guest.ssh_command_l2_1("lsblk").unwrap().trim(), 4376 vec!["vdc", "16M"], 4377 1 4378 )); 4379 4380 // Hotplug an extra virtio-net device through L2 VM. 4381 guest 4382 .ssh_command_l1( 4383 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 4384 ) 4385 .unwrap(); 4386 guest 4387 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 4388 .unwrap(); 4389 let vfio_hotplug_output = guest 4390 .ssh_command_l1( 4391 "sudo /mnt/ch-remote \ 4392 --api-socket=/tmp/ch_api.sock \ 4393 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 4394 ) 4395 .unwrap(); 4396 assert!(check_matched_lines_count( 4397 vfio_hotplug_output.trim(), 4398 vec!["{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}"], 4399 1 4400 )); 4401 4402 thread::sleep(std::time::Duration::new(10, 0)); 4403 4404 // Let's also verify from the third virtio-net device passed to 4405 // the L2 VM. This third device has been hotplugged through the L2 4406 // VM, so this is our way to validate hotplug works for VFIO PCI. 4407 assert!(check_matched_lines_count( 4408 guest.ssh_command_l2_3("cat /proc/cmdline").unwrap().trim(), 4409 vec!["VFIOTAG"], 4410 1 4411 )); 4412 4413 // Check the amount of PCI devices appearing in L2 VM. 4414 // There should be one more device than before, raising the count 4415 // up to 9 PCI devices. 4416 assert!(check_lines_count( 4417 guest 4418 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4419 .unwrap() 4420 .trim(), 4421 9 4422 )); 4423 4424 // Let's now verify that we can correctly remove the virtio-net 4425 // device through the "remove-device" command responsible for 4426 // unplugging VFIO devices. 4427 guest 4428 .ssh_command_l1( 4429 "sudo /mnt/ch-remote \ 4430 --api-socket=/tmp/ch_api.sock \ 4431 remove-device vfio123", 4432 ) 4433 .unwrap(); 4434 thread::sleep(std::time::Duration::new(10, 0)); 4435 4436 // Check the amount of PCI devices appearing in L2 VM is back down 4437 // to 8 devices. 4438 assert!(check_lines_count( 4439 guest 4440 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4441 .unwrap() 4442 .trim(), 4443 8 4444 )); 4445 4446 // Perform memory hotplug in L2 and validate the memory is showing 4447 // up as expected. In order to check, we will use the virtio-net 4448 // device already passed through L2 as a VFIO device, this will 4449 // verify that VFIO devices are functional with memory hotplug. 4450 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 4451 guest 4452 .ssh_command_l2_1( 4453 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 4454 ) 4455 .unwrap(); 4456 guest 4457 .ssh_command_l1( 4458 "sudo /mnt/ch-remote \ 4459 --api-socket=/tmp/ch_api.sock \ 4460 resize --memory=1073741824", 4461 ) 4462 .unwrap(); 4463 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 4464 }); 4465 4466 let _ = child.kill(); 4467 let output = child.wait_with_output().unwrap(); 4468 4469 cleanup_vfio_network_interfaces(); 4470 4471 handle_child_output(r, &output); 4472 } 4473 4474 #[test] 4475 fn test_direct_kernel_boot_noacpi() { 4476 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4477 let guest = Guest::new(Box::new(focal)); 4478 4479 let kernel_path = direct_kernel_boot_path(); 4480 4481 let mut child = GuestCommand::new(&guest) 4482 .args(["--cpus", "boot=1"]) 4483 .args(["--memory", "size=512M"]) 4484 .args(["--kernel", kernel_path.to_str().unwrap()]) 4485 .args([ 4486 "--cmdline", 4487 format!("{DIRECT_KERNEL_BOOT_CMDLINE} acpi=off").as_str(), 4488 ]) 4489 .default_disks() 4490 .default_net() 4491 .capture_output() 4492 .spawn() 4493 .unwrap(); 4494 4495 let r = std::panic::catch_unwind(|| { 4496 guest.wait_vm_boot(None).unwrap(); 4497 4498 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4499 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4500 }); 4501 4502 let _ = child.kill(); 4503 let output = child.wait_with_output().unwrap(); 4504 4505 handle_child_output(r, &output); 4506 } 4507 4508 #[test] 4509 fn test_virtio_vsock() { 4510 _test_virtio_vsock(false) 4511 } 4512 4513 #[test] 4514 fn test_virtio_vsock_hotplug() { 4515 _test_virtio_vsock(true); 4516 } 4517 4518 #[test] 4519 fn test_api_http_shutdown() { 4520 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4521 let guest = Guest::new(Box::new(focal)); 4522 4523 _test_api_shutdown(TargetApi::new_http_api(&guest.tmp_dir), guest) 4524 } 4525 4526 #[test] 4527 fn test_api_http_delete() { 4528 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4529 let guest = Guest::new(Box::new(focal)); 4530 4531 _test_api_delete(TargetApi::new_http_api(&guest.tmp_dir), guest); 4532 } 4533 4534 #[test] 4535 fn test_api_http_pause_resume() { 4536 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4537 let guest = Guest::new(Box::new(focal)); 4538 4539 _test_api_pause_resume(TargetApi::new_http_api(&guest.tmp_dir), guest) 4540 } 4541 4542 #[test] 4543 fn test_api_http_create_boot() { 4544 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4545 let guest = Guest::new(Box::new(focal)); 4546 4547 _test_api_create_boot(TargetApi::new_http_api(&guest.tmp_dir), guest) 4548 } 4549 4550 #[test] 4551 fn test_virtio_iommu() { 4552 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4553 } 4554 4555 #[test] 4556 // We cannot force the software running in the guest to reprogram the BAR 4557 // with some different addresses, but we have a reliable way of testing it 4558 // with a standard Linux kernel. 4559 // By removing a device from the PCI tree, and then rescanning the tree, 4560 // Linux consistently chooses to reorganize the PCI device BARs to other 4561 // locations in the guest address space. 4562 // This test creates a dedicated PCI network device to be checked as being 4563 // properly probed first, then removing it, and adding it again by doing a 4564 // rescan. 4565 fn test_pci_bar_reprogramming() { 4566 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4567 let guest = Guest::new(Box::new(focal)); 4568 4569 #[cfg(target_arch = "x86_64")] 4570 let kernel_path = direct_kernel_boot_path(); 4571 #[cfg(target_arch = "aarch64")] 4572 let kernel_path = edk2_path(); 4573 4574 let mut child = GuestCommand::new(&guest) 4575 .args(["--cpus", "boot=1"]) 4576 .args(["--memory", "size=512M"]) 4577 .args(["--kernel", kernel_path.to_str().unwrap()]) 4578 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4579 .default_disks() 4580 .args([ 4581 "--net", 4582 guest.default_net_string().as_str(), 4583 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4584 ]) 4585 .capture_output() 4586 .spawn() 4587 .unwrap(); 4588 4589 let r = std::panic::catch_unwind(|| { 4590 guest.wait_vm_boot(None).unwrap(); 4591 4592 // 2 network interfaces + default localhost ==> 3 interfaces 4593 assert_eq!( 4594 guest 4595 .ssh_command("ip -o link | wc -l") 4596 .unwrap() 4597 .trim() 4598 .parse::<u32>() 4599 .unwrap_or_default(), 4600 3 4601 ); 4602 4603 let init_bar_addr = guest 4604 .ssh_command( 4605 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4606 ) 4607 .unwrap(); 4608 4609 // Remove the PCI device 4610 guest 4611 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4612 .unwrap(); 4613 4614 // Only 1 network interface left + default localhost ==> 2 interfaces 4615 assert_eq!( 4616 guest 4617 .ssh_command("ip -o link | wc -l") 4618 .unwrap() 4619 .trim() 4620 .parse::<u32>() 4621 .unwrap_or_default(), 4622 2 4623 ); 4624 4625 // Remove the PCI device 4626 guest 4627 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4628 .unwrap(); 4629 4630 // Back to 2 network interface + default localhost ==> 3 interfaces 4631 assert_eq!( 4632 guest 4633 .ssh_command("ip -o link | wc -l") 4634 .unwrap() 4635 .trim() 4636 .parse::<u32>() 4637 .unwrap_or_default(), 4638 3 4639 ); 4640 4641 let new_bar_addr = guest 4642 .ssh_command( 4643 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4644 ) 4645 .unwrap(); 4646 4647 // Let's compare the BAR addresses for our virtio-net device. 4648 // They should be different as we expect the BAR reprogramming 4649 // to have happened. 4650 assert_ne!(init_bar_addr, new_bar_addr); 4651 }); 4652 4653 let _ = child.kill(); 4654 let output = child.wait_with_output().unwrap(); 4655 4656 handle_child_output(r, &output); 4657 } 4658 4659 #[test] 4660 fn test_memory_mergeable_off() { 4661 test_memory_mergeable(false) 4662 } 4663 4664 #[test] 4665 #[cfg(target_arch = "x86_64")] 4666 fn test_cpu_hotplug() { 4667 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4668 let guest = Guest::new(Box::new(focal)); 4669 let api_socket = temp_api_path(&guest.tmp_dir); 4670 4671 let kernel_path = direct_kernel_boot_path(); 4672 4673 let mut child = GuestCommand::new(&guest) 4674 .args(["--cpus", "boot=2,max=4"]) 4675 .args(["--memory", "size=512M"]) 4676 .args(["--kernel", kernel_path.to_str().unwrap()]) 4677 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4678 .default_disks() 4679 .default_net() 4680 .args(["--api-socket", &api_socket]) 4681 .capture_output() 4682 .spawn() 4683 .unwrap(); 4684 4685 let r = std::panic::catch_unwind(|| { 4686 guest.wait_vm_boot(None).unwrap(); 4687 4688 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4689 4690 // Resize the VM 4691 let desired_vcpus = 4; 4692 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4693 4694 guest 4695 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4696 .unwrap(); 4697 guest 4698 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4699 .unwrap(); 4700 thread::sleep(std::time::Duration::new(10, 0)); 4701 assert_eq!( 4702 guest.get_cpu_count().unwrap_or_default(), 4703 u32::from(desired_vcpus) 4704 ); 4705 4706 guest.reboot_linux(0, None); 4707 4708 assert_eq!( 4709 guest.get_cpu_count().unwrap_or_default(), 4710 u32::from(desired_vcpus) 4711 ); 4712 4713 // Resize the VM 4714 let desired_vcpus = 2; 4715 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4716 4717 thread::sleep(std::time::Duration::new(10, 0)); 4718 assert_eq!( 4719 guest.get_cpu_count().unwrap_or_default(), 4720 u32::from(desired_vcpus) 4721 ); 4722 4723 // Resize the VM back up to 4 4724 let desired_vcpus = 4; 4725 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4726 4727 guest 4728 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4729 .unwrap(); 4730 guest 4731 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4732 .unwrap(); 4733 thread::sleep(std::time::Duration::new(10, 0)); 4734 assert_eq!( 4735 guest.get_cpu_count().unwrap_or_default(), 4736 u32::from(desired_vcpus) 4737 ); 4738 }); 4739 4740 let _ = child.kill(); 4741 let output = child.wait_with_output().unwrap(); 4742 4743 handle_child_output(r, &output); 4744 } 4745 4746 #[test] 4747 fn test_memory_hotplug() { 4748 #[cfg(target_arch = "aarch64")] 4749 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4750 #[cfg(target_arch = "x86_64")] 4751 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4752 let focal = UbuntuDiskConfig::new(focal_image); 4753 let guest = Guest::new(Box::new(focal)); 4754 let api_socket = temp_api_path(&guest.tmp_dir); 4755 4756 #[cfg(target_arch = "aarch64")] 4757 let kernel_path = edk2_path(); 4758 #[cfg(target_arch = "x86_64")] 4759 let kernel_path = direct_kernel_boot_path(); 4760 4761 let mut child = GuestCommand::new(&guest) 4762 .args(["--cpus", "boot=2,max=4"]) 4763 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4764 .args(["--kernel", kernel_path.to_str().unwrap()]) 4765 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4766 .default_disks() 4767 .default_net() 4768 .args(["--balloon", "size=0"]) 4769 .args(["--api-socket", &api_socket]) 4770 .capture_output() 4771 .spawn() 4772 .unwrap(); 4773 4774 let r = std::panic::catch_unwind(|| { 4775 guest.wait_vm_boot(None).unwrap(); 4776 4777 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4778 4779 guest.enable_memory_hotplug(); 4780 4781 // Add RAM to the VM 4782 let desired_ram = 1024 << 20; 4783 resize_command(&api_socket, None, Some(desired_ram), None, None); 4784 4785 thread::sleep(std::time::Duration::new(10, 0)); 4786 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4787 4788 // Use balloon to remove RAM from the VM 4789 let desired_balloon = 512 << 20; 4790 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4791 4792 thread::sleep(std::time::Duration::new(10, 0)); 4793 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4794 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4795 4796 guest.reboot_linux(0, None); 4797 4798 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4799 4800 // Use balloon add RAM to the VM 4801 let desired_balloon = 0; 4802 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4803 4804 thread::sleep(std::time::Duration::new(10, 0)); 4805 4806 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4807 4808 guest.enable_memory_hotplug(); 4809 4810 // Add RAM to the VM 4811 let desired_ram = 2048 << 20; 4812 resize_command(&api_socket, None, Some(desired_ram), None, None); 4813 4814 thread::sleep(std::time::Duration::new(10, 0)); 4815 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4816 4817 // Remove RAM to the VM (only applies after reboot) 4818 let desired_ram = 1024 << 20; 4819 resize_command(&api_socket, None, Some(desired_ram), None, None); 4820 4821 guest.reboot_linux(1, None); 4822 4823 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4824 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4825 }); 4826 4827 let _ = child.kill(); 4828 let output = child.wait_with_output().unwrap(); 4829 4830 handle_child_output(r, &output); 4831 } 4832 4833 #[test] 4834 #[cfg(not(feature = "mshv"))] 4835 fn test_virtio_mem() { 4836 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4837 let guest = Guest::new(Box::new(focal)); 4838 let api_socket = temp_api_path(&guest.tmp_dir); 4839 4840 let kernel_path = direct_kernel_boot_path(); 4841 4842 let mut child = GuestCommand::new(&guest) 4843 .args(["--cpus", "boot=2,max=4"]) 4844 .args([ 4845 "--memory", 4846 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4847 ]) 4848 .args(["--kernel", kernel_path.to_str().unwrap()]) 4849 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4850 .default_disks() 4851 .default_net() 4852 .args(["--api-socket", &api_socket]) 4853 .capture_output() 4854 .spawn() 4855 .unwrap(); 4856 4857 let r = std::panic::catch_unwind(|| { 4858 guest.wait_vm_boot(None).unwrap(); 4859 4860 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4861 4862 guest.enable_memory_hotplug(); 4863 4864 // Add RAM to the VM 4865 let desired_ram = 1024 << 20; 4866 resize_command(&api_socket, None, Some(desired_ram), None, None); 4867 4868 thread::sleep(std::time::Duration::new(10, 0)); 4869 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4870 4871 // Add RAM to the VM 4872 let desired_ram = 2048 << 20; 4873 resize_command(&api_socket, None, Some(desired_ram), None, None); 4874 4875 thread::sleep(std::time::Duration::new(10, 0)); 4876 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4877 4878 // Remove RAM from the VM 4879 let desired_ram = 1024 << 20; 4880 resize_command(&api_socket, None, Some(desired_ram), None, None); 4881 4882 thread::sleep(std::time::Duration::new(10, 0)); 4883 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4884 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4885 4886 guest.reboot_linux(0, None); 4887 4888 // Check the amount of memory after reboot is 1GiB 4889 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4890 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4891 4892 // Check we can still resize to 512MiB 4893 let desired_ram = 512 << 20; 4894 resize_command(&api_socket, None, Some(desired_ram), None, None); 4895 thread::sleep(std::time::Duration::new(10, 0)); 4896 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4897 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4898 }); 4899 4900 let _ = child.kill(); 4901 let output = child.wait_with_output().unwrap(); 4902 4903 handle_child_output(r, &output); 4904 } 4905 4906 #[test] 4907 #[cfg(target_arch = "x86_64")] 4908 #[cfg(not(feature = "mshv"))] 4909 // Test both vCPU and memory resizing together 4910 fn test_resize() { 4911 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4912 let guest = Guest::new(Box::new(focal)); 4913 let api_socket = temp_api_path(&guest.tmp_dir); 4914 4915 let kernel_path = direct_kernel_boot_path(); 4916 4917 let mut child = GuestCommand::new(&guest) 4918 .args(["--cpus", "boot=2,max=4"]) 4919 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4920 .args(["--kernel", kernel_path.to_str().unwrap()]) 4921 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4922 .default_disks() 4923 .default_net() 4924 .args(["--api-socket", &api_socket]) 4925 .capture_output() 4926 .spawn() 4927 .unwrap(); 4928 4929 let r = std::panic::catch_unwind(|| { 4930 guest.wait_vm_boot(None).unwrap(); 4931 4932 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4933 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4934 4935 guest.enable_memory_hotplug(); 4936 4937 // Resize the VM 4938 let desired_vcpus = 4; 4939 let desired_ram = 1024 << 20; 4940 resize_command( 4941 &api_socket, 4942 Some(desired_vcpus), 4943 Some(desired_ram), 4944 None, 4945 None, 4946 ); 4947 4948 guest 4949 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4950 .unwrap(); 4951 guest 4952 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4953 .unwrap(); 4954 thread::sleep(std::time::Duration::new(10, 0)); 4955 assert_eq!( 4956 guest.get_cpu_count().unwrap_or_default(), 4957 u32::from(desired_vcpus) 4958 ); 4959 4960 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4961 }); 4962 4963 let _ = child.kill(); 4964 let output = child.wait_with_output().unwrap(); 4965 4966 handle_child_output(r, &output); 4967 } 4968 4969 #[test] 4970 fn test_memory_overhead() { 4971 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4972 let guest = Guest::new(Box::new(focal)); 4973 4974 let kernel_path = direct_kernel_boot_path(); 4975 4976 let guest_memory_size_kb = 512 * 1024; 4977 4978 let mut child = GuestCommand::new(&guest) 4979 .args(["--cpus", "boot=1"]) 4980 .args(["--memory", format!("size={guest_memory_size_kb}K").as_str()]) 4981 .args(["--kernel", kernel_path.to_str().unwrap()]) 4982 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4983 .default_net() 4984 .default_disks() 4985 .capture_output() 4986 .spawn() 4987 .unwrap(); 4988 4989 guest.wait_vm_boot(None).unwrap(); 4990 4991 let r = std::panic::catch_unwind(|| { 4992 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 4993 eprintln!("Guest memory overhead: {overhead} vs {MAXIMUM_VMM_OVERHEAD_KB}"); 4994 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 4995 }); 4996 4997 let _ = child.kill(); 4998 let output = child.wait_with_output().unwrap(); 4999 5000 handle_child_output(r, &output); 5001 } 5002 5003 #[test] 5004 fn test_disk_hotplug() { 5005 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5006 let guest = Guest::new(Box::new(focal)); 5007 5008 #[cfg(target_arch = "x86_64")] 5009 let kernel_path = direct_kernel_boot_path(); 5010 #[cfg(target_arch = "aarch64")] 5011 let kernel_path = edk2_path(); 5012 5013 let api_socket = temp_api_path(&guest.tmp_dir); 5014 5015 let mut child = GuestCommand::new(&guest) 5016 .args(["--api-socket", &api_socket]) 5017 .args(["--cpus", "boot=1"]) 5018 .args(["--memory", "size=512M"]) 5019 .args(["--kernel", kernel_path.to_str().unwrap()]) 5020 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5021 .default_disks() 5022 .default_net() 5023 .capture_output() 5024 .spawn() 5025 .unwrap(); 5026 5027 let r = std::panic::catch_unwind(|| { 5028 guest.wait_vm_boot(None).unwrap(); 5029 5030 // Check /dev/vdc is not there 5031 assert_eq!( 5032 guest 5033 .ssh_command("lsblk | grep -c vdc.*16M || true") 5034 .unwrap() 5035 .trim() 5036 .parse::<u32>() 5037 .unwrap_or(1), 5038 0 5039 ); 5040 5041 // Now let's add the extra disk. 5042 let mut blk_file_path = dirs::home_dir().unwrap(); 5043 blk_file_path.push("workloads"); 5044 blk_file_path.push("blk.img"); 5045 let (cmd_success, cmd_output) = remote_command_w_output( 5046 &api_socket, 5047 "add-disk", 5048 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5049 ); 5050 assert!(cmd_success); 5051 assert!(String::from_utf8_lossy(&cmd_output) 5052 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5053 5054 thread::sleep(std::time::Duration::new(10, 0)); 5055 5056 // Check that /dev/vdc exists and the block size is 16M. 5057 assert_eq!( 5058 guest 5059 .ssh_command("lsblk | grep vdc | grep -c 16M") 5060 .unwrap() 5061 .trim() 5062 .parse::<u32>() 5063 .unwrap_or_default(), 5064 1 5065 ); 5066 // And check the block device can be read. 5067 guest 5068 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5069 .unwrap(); 5070 5071 // Let's remove it the extra disk. 5072 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5073 thread::sleep(std::time::Duration::new(5, 0)); 5074 // And check /dev/vdc is not there 5075 assert_eq!( 5076 guest 5077 .ssh_command("lsblk | grep -c vdc.*16M || true") 5078 .unwrap() 5079 .trim() 5080 .parse::<u32>() 5081 .unwrap_or(1), 5082 0 5083 ); 5084 5085 // And add it back to validate unplug did work correctly. 5086 let (cmd_success, cmd_output) = remote_command_w_output( 5087 &api_socket, 5088 "add-disk", 5089 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5090 ); 5091 assert!(cmd_success); 5092 assert!(String::from_utf8_lossy(&cmd_output) 5093 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5094 5095 thread::sleep(std::time::Duration::new(10, 0)); 5096 5097 // Check that /dev/vdc exists and the block size is 16M. 5098 assert_eq!( 5099 guest 5100 .ssh_command("lsblk | grep vdc | grep -c 16M") 5101 .unwrap() 5102 .trim() 5103 .parse::<u32>() 5104 .unwrap_or_default(), 5105 1 5106 ); 5107 // And check the block device can be read. 5108 guest 5109 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5110 .unwrap(); 5111 5112 // Reboot the VM. 5113 guest.reboot_linux(0, None); 5114 5115 // Check still there after reboot 5116 assert_eq!( 5117 guest 5118 .ssh_command("lsblk | grep vdc | grep -c 16M") 5119 .unwrap() 5120 .trim() 5121 .parse::<u32>() 5122 .unwrap_or_default(), 5123 1 5124 ); 5125 5126 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5127 5128 thread::sleep(std::time::Duration::new(20, 0)); 5129 5130 // Check device has gone away 5131 assert_eq!( 5132 guest 5133 .ssh_command("lsblk | grep -c vdc.*16M || true") 5134 .unwrap() 5135 .trim() 5136 .parse::<u32>() 5137 .unwrap_or(1), 5138 0 5139 ); 5140 5141 guest.reboot_linux(1, None); 5142 5143 // Check device still absent 5144 assert_eq!( 5145 guest 5146 .ssh_command("lsblk | grep -c vdc.*16M || true") 5147 .unwrap() 5148 .trim() 5149 .parse::<u32>() 5150 .unwrap_or(1), 5151 0 5152 ); 5153 }); 5154 5155 let _ = child.kill(); 5156 let output = child.wait_with_output().unwrap(); 5157 5158 handle_child_output(r, &output); 5159 } 5160 5161 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 5162 const LOOP_CONFIGURE: u64 = 0x4c0a; 5163 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 5164 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 5165 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 5166 5167 #[repr(C)] 5168 struct LoopInfo64 { 5169 lo_device: u64, 5170 lo_inode: u64, 5171 lo_rdevice: u64, 5172 lo_offset: u64, 5173 lo_sizelimit: u64, 5174 lo_number: u32, 5175 lo_encrypt_type: u32, 5176 lo_encrypt_key_size: u32, 5177 lo_flags: u32, 5178 lo_file_name: [u8; 64], 5179 lo_crypt_name: [u8; 64], 5180 lo_encrypt_key: [u8; 32], 5181 lo_init: [u64; 2], 5182 } 5183 5184 impl Default for LoopInfo64 { 5185 fn default() -> Self { 5186 LoopInfo64 { 5187 lo_device: 0, 5188 lo_inode: 0, 5189 lo_rdevice: 0, 5190 lo_offset: 0, 5191 lo_sizelimit: 0, 5192 lo_number: 0, 5193 lo_encrypt_type: 0, 5194 lo_encrypt_key_size: 0, 5195 lo_flags: 0, 5196 lo_file_name: [0; 64], 5197 lo_crypt_name: [0; 64], 5198 lo_encrypt_key: [0; 32], 5199 lo_init: [0; 2], 5200 } 5201 } 5202 } 5203 5204 #[derive(Default)] 5205 #[repr(C)] 5206 struct LoopConfig { 5207 fd: u32, 5208 block_size: u32, 5209 info: LoopInfo64, 5210 _reserved: [u64; 8], 5211 } 5212 5213 // Open loop-control device 5214 let loop_ctl_file = OpenOptions::new() 5215 .read(true) 5216 .write(true) 5217 .open(LOOP_CTL_PATH) 5218 .unwrap(); 5219 5220 // Request a free loop device 5221 let loop_device_number = 5222 unsafe { libc::ioctl(loop_ctl_file.as_raw_fd(), LOOP_CTL_GET_FREE as _) }; 5223 5224 if loop_device_number < 0 { 5225 panic!("Couldn't find a free loop device"); 5226 } 5227 5228 // Create loop device path 5229 let loop_device_path = format!("{LOOP_DEVICE_PREFIX}{loop_device_number}"); 5230 5231 // Open loop device 5232 let loop_device_file = OpenOptions::new() 5233 .read(true) 5234 .write(true) 5235 .open(&loop_device_path) 5236 .unwrap(); 5237 5238 // Open backing file 5239 let backing_file = OpenOptions::new() 5240 .read(true) 5241 .write(true) 5242 .open(backing_file_path) 5243 .unwrap(); 5244 5245 let loop_config = LoopConfig { 5246 fd: backing_file.as_raw_fd() as u32, 5247 block_size, 5248 ..Default::default() 5249 }; 5250 5251 for i in 0..num_retries { 5252 let ret = unsafe { 5253 libc::ioctl( 5254 loop_device_file.as_raw_fd(), 5255 LOOP_CONFIGURE as _, 5256 &loop_config, 5257 ) 5258 }; 5259 if ret != 0 { 5260 if i < num_retries - 1 { 5261 println!( 5262 "Iteration {}: Failed to configure the loop device {}: {}", 5263 i, 5264 loop_device_path, 5265 std::io::Error::last_os_error() 5266 ); 5267 } else { 5268 panic!( 5269 "Failed {} times trying to configure the loop device {}: {}", 5270 num_retries, 5271 loop_device_path, 5272 std::io::Error::last_os_error() 5273 ); 5274 } 5275 } else { 5276 break; 5277 } 5278 5279 // Wait for a bit before retrying 5280 thread::sleep(std::time::Duration::new(5, 0)); 5281 } 5282 5283 loop_device_path 5284 } 5285 5286 #[test] 5287 fn test_virtio_block_topology() { 5288 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5289 let guest = Guest::new(Box::new(focal)); 5290 5291 let kernel_path = direct_kernel_boot_path(); 5292 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5293 5294 let output = exec_host_command_output( 5295 format!( 5296 "qemu-img create -f raw {} 16M", 5297 test_disk_path.to_str().unwrap() 5298 ) 5299 .as_str(), 5300 ); 5301 if !output.status.success() { 5302 let stdout = String::from_utf8_lossy(&output.stdout); 5303 let stderr = String::from_utf8_lossy(&output.stderr); 5304 panic!("qemu-img command failed\nstdout\n{stdout}\nstderr\n{stderr}"); 5305 } 5306 5307 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5308 5309 let mut child = GuestCommand::new(&guest) 5310 .args(["--cpus", "boot=1"]) 5311 .args(["--memory", "size=512M"]) 5312 .args(["--kernel", kernel_path.to_str().unwrap()]) 5313 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5314 .args([ 5315 "--disk", 5316 format!( 5317 "path={}", 5318 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5319 ) 5320 .as_str(), 5321 format!( 5322 "path={}", 5323 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5324 ) 5325 .as_str(), 5326 format!("path={}", &loop_dev).as_str(), 5327 ]) 5328 .default_net() 5329 .capture_output() 5330 .spawn() 5331 .unwrap(); 5332 5333 let r = std::panic::catch_unwind(|| { 5334 guest.wait_vm_boot(None).unwrap(); 5335 5336 // MIN-IO column 5337 assert_eq!( 5338 guest 5339 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5340 .unwrap() 5341 .trim() 5342 .parse::<u32>() 5343 .unwrap_or_default(), 5344 4096 5345 ); 5346 // PHY-SEC column 5347 assert_eq!( 5348 guest 5349 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5350 .unwrap() 5351 .trim() 5352 .parse::<u32>() 5353 .unwrap_or_default(), 5354 4096 5355 ); 5356 // LOG-SEC column 5357 assert_eq!( 5358 guest 5359 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5360 .unwrap() 5361 .trim() 5362 .parse::<u32>() 5363 .unwrap_or_default(), 5364 4096 5365 ); 5366 }); 5367 5368 let _ = child.kill(); 5369 let output = child.wait_with_output().unwrap(); 5370 5371 handle_child_output(r, &output); 5372 5373 Command::new("losetup") 5374 .args(["-d", &loop_dev]) 5375 .output() 5376 .expect("loop device not found"); 5377 } 5378 5379 #[test] 5380 fn test_virtio_balloon_deflate_on_oom() { 5381 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5382 let guest = Guest::new(Box::new(focal)); 5383 5384 let kernel_path = direct_kernel_boot_path(); 5385 5386 let api_socket = temp_api_path(&guest.tmp_dir); 5387 5388 //Let's start a 4G guest with balloon occupied 2G memory 5389 let mut child = GuestCommand::new(&guest) 5390 .args(["--api-socket", &api_socket]) 5391 .args(["--cpus", "boot=1"]) 5392 .args(["--memory", "size=4G"]) 5393 .args(["--kernel", kernel_path.to_str().unwrap()]) 5394 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5395 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5396 .default_disks() 5397 .default_net() 5398 .capture_output() 5399 .spawn() 5400 .unwrap(); 5401 5402 let r = std::panic::catch_unwind(|| { 5403 guest.wait_vm_boot(None).unwrap(); 5404 5405 // Wait for balloon memory's initialization and check its size. 5406 // The virtio-balloon driver might take a few seconds to report the 5407 // balloon effective size back to the VMM. 5408 thread::sleep(std::time::Duration::new(20, 0)); 5409 5410 let orig_balloon = balloon_size(&api_socket); 5411 println!("The original balloon memory size is {orig_balloon} bytes"); 5412 assert!(orig_balloon == 2147483648); 5413 5414 // Two steps to verify if the 'deflate_on_oom' parameter works. 5415 // 1st: run a command to trigger an OOM in the guest. 5416 guest 5417 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5418 .unwrap(); 5419 5420 // Give some time for the OOM to happen in the guest and be reported 5421 // back to the host. 5422 thread::sleep(std::time::Duration::new(20, 0)); 5423 5424 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5425 let deflated_balloon = balloon_size(&api_socket); 5426 println!("After deflating, balloon memory size is {deflated_balloon} bytes"); 5427 // Verify the balloon size deflated 5428 assert!(deflated_balloon < 2147483648); 5429 }); 5430 5431 let _ = child.kill(); 5432 let output = child.wait_with_output().unwrap(); 5433 5434 handle_child_output(r, &output); 5435 } 5436 5437 #[test] 5438 #[cfg(not(feature = "mshv"))] 5439 fn test_virtio_balloon_free_page_reporting() { 5440 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5441 let guest = Guest::new(Box::new(focal)); 5442 5443 //Let's start a 4G guest with balloon occupied 2G memory 5444 let mut child = GuestCommand::new(&guest) 5445 .args(["--cpus", "boot=1"]) 5446 .args(["--memory", "size=4G"]) 5447 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5448 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5449 .args(["--balloon", "size=0,free_page_reporting=on"]) 5450 .default_disks() 5451 .default_net() 5452 .capture_output() 5453 .spawn() 5454 .unwrap(); 5455 5456 let pid = child.id(); 5457 let r = std::panic::catch_unwind(|| { 5458 guest.wait_vm_boot(None).unwrap(); 5459 5460 // Check the initial RSS is less than 1GiB 5461 let rss = process_rss_kib(pid); 5462 println!("RSS {rss} < 1048576"); 5463 assert!(rss < 1048576); 5464 5465 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5466 // seconds 5467 let guest_ip = guest.network.guest_ip.clone(); 5468 thread::spawn(move || { 5469 ssh_command_ip( 5470 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5471 &guest_ip, 5472 DEFAULT_SSH_RETRIES, 5473 DEFAULT_SSH_TIMEOUT, 5474 ) 5475 .unwrap(); 5476 }); 5477 5478 // Wait for 50 seconds to make sure the stress command is consuming 5479 // the expected amount of memory. 5480 thread::sleep(std::time::Duration::new(50, 0)); 5481 let rss = process_rss_kib(pid); 5482 println!("RSS {rss} >= 2097152"); 5483 assert!(rss >= 2097152); 5484 5485 // Wait for an extra minute to make sure the stress command has 5486 // completed and that the guest reported the free pages to the VMM 5487 // through the virtio-balloon device. We expect the RSS to be under 5488 // 2GiB. 5489 thread::sleep(std::time::Duration::new(60, 0)); 5490 let rss = process_rss_kib(pid); 5491 println!("RSS {rss} < 2097152"); 5492 assert!(rss < 2097152); 5493 }); 5494 5495 let _ = child.kill(); 5496 let output = child.wait_with_output().unwrap(); 5497 5498 handle_child_output(r, &output); 5499 } 5500 5501 #[test] 5502 fn test_pmem_hotplug() { 5503 _test_pmem_hotplug(None) 5504 } 5505 5506 #[test] 5507 fn test_pmem_multi_segment_hotplug() { 5508 _test_pmem_hotplug(Some(15)) 5509 } 5510 5511 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5512 #[cfg(target_arch = "aarch64")] 5513 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5514 #[cfg(target_arch = "x86_64")] 5515 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5516 let focal = UbuntuDiskConfig::new(focal_image); 5517 let guest = Guest::new(Box::new(focal)); 5518 5519 #[cfg(target_arch = "x86_64")] 5520 let kernel_path = direct_kernel_boot_path(); 5521 #[cfg(target_arch = "aarch64")] 5522 let kernel_path = edk2_path(); 5523 5524 let api_socket = temp_api_path(&guest.tmp_dir); 5525 5526 let mut cmd = GuestCommand::new(&guest); 5527 5528 cmd.args(["--api-socket", &api_socket]) 5529 .args(["--cpus", "boot=1"]) 5530 .args(["--memory", "size=512M"]) 5531 .args(["--kernel", kernel_path.to_str().unwrap()]) 5532 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5533 .default_disks() 5534 .default_net() 5535 .capture_output(); 5536 5537 if pci_segment.is_some() { 5538 cmd.args([ 5539 "--platform", 5540 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5541 ]); 5542 } 5543 5544 let mut child = cmd.spawn().unwrap(); 5545 5546 let r = std::panic::catch_unwind(|| { 5547 guest.wait_vm_boot(None).unwrap(); 5548 5549 // Check /dev/pmem0 is not there 5550 assert_eq!( 5551 guest 5552 .ssh_command("lsblk | grep -c pmem0 || true") 5553 .unwrap() 5554 .trim() 5555 .parse::<u32>() 5556 .unwrap_or(1), 5557 0 5558 ); 5559 5560 let pmem_temp_file = TempFile::new().unwrap(); 5561 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5562 let (cmd_success, cmd_output) = remote_command_w_output( 5563 &api_socket, 5564 "add-pmem", 5565 Some(&format!( 5566 "file={},id=test0{}", 5567 pmem_temp_file.as_path().to_str().unwrap(), 5568 if let Some(pci_segment) = pci_segment { 5569 format!(",pci_segment={pci_segment}") 5570 } else { 5571 "".to_owned() 5572 } 5573 )), 5574 ); 5575 assert!(cmd_success); 5576 if let Some(pci_segment) = pci_segment { 5577 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5578 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5579 ))); 5580 } else { 5581 assert!(String::from_utf8_lossy(&cmd_output) 5582 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5583 } 5584 5585 // Check that /dev/pmem0 exists and the block size is 128M 5586 assert_eq!( 5587 guest 5588 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5589 .unwrap() 5590 .trim() 5591 .parse::<u32>() 5592 .unwrap_or_default(), 5593 1 5594 ); 5595 5596 guest.reboot_linux(0, None); 5597 5598 // Check still there after reboot 5599 assert_eq!( 5600 guest 5601 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5602 .unwrap() 5603 .trim() 5604 .parse::<u32>() 5605 .unwrap_or_default(), 5606 1 5607 ); 5608 5609 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5610 5611 thread::sleep(std::time::Duration::new(20, 0)); 5612 5613 // Check device has gone away 5614 assert_eq!( 5615 guest 5616 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5617 .unwrap() 5618 .trim() 5619 .parse::<u32>() 5620 .unwrap_or(1), 5621 0 5622 ); 5623 5624 guest.reboot_linux(1, None); 5625 5626 // Check still absent after reboot 5627 assert_eq!( 5628 guest 5629 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5630 .unwrap() 5631 .trim() 5632 .parse::<u32>() 5633 .unwrap_or(1), 5634 0 5635 ); 5636 }); 5637 5638 let _ = child.kill(); 5639 let output = child.wait_with_output().unwrap(); 5640 5641 handle_child_output(r, &output); 5642 } 5643 5644 #[test] 5645 fn test_net_hotplug() { 5646 _test_net_hotplug(None) 5647 } 5648 5649 #[test] 5650 fn test_net_multi_segment_hotplug() { 5651 _test_net_hotplug(Some(15)) 5652 } 5653 5654 fn _test_net_hotplug(pci_segment: Option<u16>) { 5655 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5656 let guest = Guest::new(Box::new(focal)); 5657 5658 #[cfg(target_arch = "x86_64")] 5659 let kernel_path = direct_kernel_boot_path(); 5660 #[cfg(target_arch = "aarch64")] 5661 let kernel_path = edk2_path(); 5662 5663 let api_socket = temp_api_path(&guest.tmp_dir); 5664 5665 // Boot without network 5666 let mut cmd = GuestCommand::new(&guest); 5667 5668 cmd.args(["--api-socket", &api_socket]) 5669 .args(["--cpus", "boot=1"]) 5670 .args(["--memory", "size=512M"]) 5671 .args(["--kernel", kernel_path.to_str().unwrap()]) 5672 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5673 .default_disks() 5674 .capture_output(); 5675 5676 if pci_segment.is_some() { 5677 cmd.args([ 5678 "--platform", 5679 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5680 ]); 5681 } 5682 5683 let mut child = cmd.spawn().unwrap(); 5684 5685 thread::sleep(std::time::Duration::new(20, 0)); 5686 5687 let r = std::panic::catch_unwind(|| { 5688 // Add network 5689 let (cmd_success, cmd_output) = remote_command_w_output( 5690 &api_socket, 5691 "add-net", 5692 Some( 5693 format!( 5694 "{}{},id=test0", 5695 guest.default_net_string(), 5696 if let Some(pci_segment) = pci_segment { 5697 format!(",pci_segment={pci_segment}") 5698 } else { 5699 "".to_owned() 5700 } 5701 ) 5702 .as_str(), 5703 ), 5704 ); 5705 assert!(cmd_success); 5706 5707 if let Some(pci_segment) = pci_segment { 5708 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5709 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5710 ))); 5711 } else { 5712 assert!(String::from_utf8_lossy(&cmd_output) 5713 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5714 } 5715 5716 thread::sleep(std::time::Duration::new(5, 0)); 5717 5718 // 1 network interfaces + default localhost ==> 2 interfaces 5719 assert_eq!( 5720 guest 5721 .ssh_command("ip -o link | wc -l") 5722 .unwrap() 5723 .trim() 5724 .parse::<u32>() 5725 .unwrap_or_default(), 5726 2 5727 ); 5728 5729 // Remove network 5730 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5731 thread::sleep(std::time::Duration::new(5, 0)); 5732 5733 let (cmd_success, cmd_output) = remote_command_w_output( 5734 &api_socket, 5735 "add-net", 5736 Some( 5737 format!( 5738 "{}{},id=test1", 5739 guest.default_net_string(), 5740 if let Some(pci_segment) = pci_segment { 5741 format!(",pci_segment={pci_segment}") 5742 } else { 5743 "".to_owned() 5744 } 5745 ) 5746 .as_str(), 5747 ), 5748 ); 5749 assert!(cmd_success); 5750 5751 if let Some(pci_segment) = pci_segment { 5752 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5753 "{{\"id\":\"test1\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5754 ))); 5755 } else { 5756 assert!(String::from_utf8_lossy(&cmd_output) 5757 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5758 } 5759 5760 thread::sleep(std::time::Duration::new(5, 0)); 5761 5762 // 1 network interfaces + default localhost ==> 2 interfaces 5763 assert_eq!( 5764 guest 5765 .ssh_command("ip -o link | wc -l") 5766 .unwrap() 5767 .trim() 5768 .parse::<u32>() 5769 .unwrap_or_default(), 5770 2 5771 ); 5772 5773 guest.reboot_linux(0, None); 5774 5775 // Check still there after reboot 5776 // 1 network interfaces + default localhost ==> 2 interfaces 5777 assert_eq!( 5778 guest 5779 .ssh_command("ip -o link | wc -l") 5780 .unwrap() 5781 .trim() 5782 .parse::<u32>() 5783 .unwrap_or_default(), 5784 2 5785 ); 5786 }); 5787 5788 let _ = child.kill(); 5789 let output = child.wait_with_output().unwrap(); 5790 5791 handle_child_output(r, &output); 5792 } 5793 5794 #[test] 5795 fn test_initramfs() { 5796 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5797 let guest = Guest::new(Box::new(focal)); 5798 let mut workload_path = dirs::home_dir().unwrap(); 5799 workload_path.push("workloads"); 5800 5801 #[cfg(target_arch = "x86_64")] 5802 let mut kernels = vec![direct_kernel_boot_path()]; 5803 #[cfg(target_arch = "aarch64")] 5804 let kernels = vec![direct_kernel_boot_path()]; 5805 5806 #[cfg(target_arch = "x86_64")] 5807 { 5808 let mut pvh_kernel_path = workload_path.clone(); 5809 pvh_kernel_path.push("vmlinux"); 5810 kernels.push(pvh_kernel_path); 5811 } 5812 5813 let mut initramfs_path = workload_path; 5814 initramfs_path.push("alpine_initramfs.img"); 5815 5816 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5817 let cmdline = format!("console=hvc0 quiet TEST_STRING={test_string}"); 5818 5819 kernels.iter().for_each(|k_path| { 5820 let mut child = GuestCommand::new(&guest) 5821 .args(["--kernel", k_path.to_str().unwrap()]) 5822 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 5823 .args(["--cmdline", &cmdline]) 5824 .capture_output() 5825 .spawn() 5826 .unwrap(); 5827 5828 thread::sleep(std::time::Duration::new(20, 0)); 5829 5830 let _ = child.kill(); 5831 let output = child.wait_with_output().unwrap(); 5832 5833 let r = std::panic::catch_unwind(|| { 5834 let s = String::from_utf8_lossy(&output.stdout); 5835 5836 assert_ne!(s.lines().position(|line| line == test_string), None); 5837 }); 5838 5839 handle_child_output(r, &output); 5840 }); 5841 } 5842 5843 // One thing to note about this test. The virtio-net device is heavily used 5844 // through each ssh command. There's no need to perform a dedicated test to 5845 // verify the migration went well for virtio-net. 5846 #[test] 5847 #[cfg(not(feature = "mshv"))] 5848 fn test_snapshot_restore_hotplug_virtiomem() { 5849 _test_snapshot_restore(true); 5850 } 5851 5852 #[test] 5853 fn test_snapshot_restore_basic() { 5854 _test_snapshot_restore(false); 5855 } 5856 5857 fn _test_snapshot_restore(use_hotplug: bool) { 5858 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5859 let guest = Guest::new(Box::new(focal)); 5860 let kernel_path = direct_kernel_boot_path(); 5861 5862 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 5863 5864 let net_id = "net123"; 5865 let net_params = format!( 5866 "id={},tap=,mac={},ip={},mask=255.255.255.0", 5867 net_id, guest.network.guest_mac, guest.network.host_ip 5868 ); 5869 let mut mem_params = "size=4G"; 5870 5871 if use_hotplug { 5872 mem_params = "size=4G,hotplug_method=virtio-mem,hotplug_size=32G" 5873 } 5874 5875 let cloudinit_params = format!( 5876 "path={},iommu=on", 5877 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5878 ); 5879 5880 let socket = temp_vsock_path(&guest.tmp_dir); 5881 let event_path = temp_event_monitor_path(&guest.tmp_dir); 5882 5883 let mut child = GuestCommand::new(&guest) 5884 .args(["--api-socket", &api_socket_source]) 5885 .args(["--event-monitor", format!("path={event_path}").as_str()]) 5886 .args(["--cpus", "boot=4"]) 5887 .args(["--memory", mem_params]) 5888 .args(["--balloon", "size=0"]) 5889 .args(["--kernel", kernel_path.to_str().unwrap()]) 5890 .args([ 5891 "--disk", 5892 format!( 5893 "path={}", 5894 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5895 ) 5896 .as_str(), 5897 cloudinit_params.as_str(), 5898 ]) 5899 .args(["--net", net_params.as_str()]) 5900 .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) 5901 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5902 .capture_output() 5903 .spawn() 5904 .unwrap(); 5905 5906 let console_text = String::from("On a branch floating down river a cricket, singing."); 5907 // Create the snapshot directory 5908 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 5909 5910 let r = std::panic::catch_unwind(|| { 5911 guest.wait_vm_boot(None).unwrap(); 5912 5913 // Check the number of vCPUs 5914 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 5915 // Check the guest RAM 5916 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 5917 if use_hotplug { 5918 // Increase guest RAM with virtio-mem 5919 resize_command( 5920 &api_socket_source, 5921 None, 5922 Some(6 << 30), 5923 None, 5924 Some(&event_path), 5925 ); 5926 thread::sleep(std::time::Duration::new(5, 0)); 5927 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 5928 // Use balloon to remove RAM from the VM 5929 resize_command( 5930 &api_socket_source, 5931 None, 5932 None, 5933 Some(1 << 30), 5934 Some(&event_path), 5935 ); 5936 thread::sleep(std::time::Duration::new(5, 0)); 5937 let total_memory = guest.get_total_memory().unwrap_or_default(); 5938 assert!(total_memory > 4_800_000); 5939 assert!(total_memory < 5_760_000); 5940 } 5941 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 5942 guest.check_devices_common(Some(&socket), Some(&console_text), None); 5943 5944 // x86_64: We check that removing and adding back the virtio-net device 5945 // does not break the snapshot/restore support for virtio-pci. 5946 // This is an important thing to test as the hotplug will 5947 // trigger a PCI BAR reprogramming, which is a good way of 5948 // checking if the stored resources are correctly restored. 5949 // Unplug the virtio-net device 5950 // AArch64: Device hotplug is currently not supported, skipping here. 5951 #[cfg(target_arch = "x86_64")] 5952 { 5953 assert!(remote_command( 5954 &api_socket_source, 5955 "remove-device", 5956 Some(net_id), 5957 )); 5958 thread::sleep(std::time::Duration::new(10, 0)); 5959 let latest_events = [&MetaEvent { 5960 event: "device-removed".to_string(), 5961 device_id: Some(net_id.to_string()), 5962 }]; 5963 assert!(check_latest_events_exact(&latest_events, &event_path)); 5964 5965 // Plug the virtio-net device again 5966 assert!(remote_command( 5967 &api_socket_source, 5968 "add-net", 5969 Some(net_params.as_str()), 5970 )); 5971 thread::sleep(std::time::Duration::new(10, 0)); 5972 } 5973 5974 // Pause the VM 5975 assert!(remote_command(&api_socket_source, "pause", None)); 5976 let latest_events = [ 5977 &MetaEvent { 5978 event: "pausing".to_string(), 5979 device_id: None, 5980 }, 5981 &MetaEvent { 5982 event: "paused".to_string(), 5983 device_id: None, 5984 }, 5985 ]; 5986 assert!(check_latest_events_exact(&latest_events, &event_path)); 5987 5988 // Take a snapshot from the VM 5989 assert!(remote_command( 5990 &api_socket_source, 5991 "snapshot", 5992 Some(format!("file://{snapshot_dir}").as_str()), 5993 )); 5994 5995 // Wait to make sure the snapshot is completed 5996 thread::sleep(std::time::Duration::new(10, 0)); 5997 5998 let latest_events = [ 5999 &MetaEvent { 6000 event: "snapshotting".to_string(), 6001 device_id: None, 6002 }, 6003 &MetaEvent { 6004 event: "snapshotted".to_string(), 6005 device_id: None, 6006 }, 6007 ]; 6008 assert!(check_latest_events_exact(&latest_events, &event_path)); 6009 }); 6010 6011 // Shutdown the source VM and check console output 6012 let _ = child.kill(); 6013 let output = child.wait_with_output().unwrap(); 6014 handle_child_output(r, &output); 6015 6016 let r = std::panic::catch_unwind(|| { 6017 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 6018 }); 6019 6020 handle_child_output(r, &output); 6021 6022 // Remove the vsock socket file. 6023 Command::new("rm") 6024 .arg("-f") 6025 .arg(socket.as_str()) 6026 .output() 6027 .unwrap(); 6028 6029 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 6030 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 6031 6032 // Restore the VM from the snapshot 6033 let mut child = GuestCommand::new(&guest) 6034 .args(["--api-socket", &api_socket_restored]) 6035 .args([ 6036 "--event-monitor", 6037 format!("path={event_path_restored}").as_str(), 6038 ]) 6039 .args([ 6040 "--restore", 6041 format!("source_url=file://{snapshot_dir}").as_str(), 6042 ]) 6043 .capture_output() 6044 .spawn() 6045 .unwrap(); 6046 6047 // Wait for the VM to be restored 6048 thread::sleep(std::time::Duration::new(10, 0)); 6049 let expected_events = [ 6050 &MetaEvent { 6051 event: "starting".to_string(), 6052 device_id: None, 6053 }, 6054 &MetaEvent { 6055 event: "activated".to_string(), 6056 device_id: Some("__console".to_string()), 6057 }, 6058 &MetaEvent { 6059 event: "activated".to_string(), 6060 device_id: Some("__rng".to_string()), 6061 }, 6062 &MetaEvent { 6063 event: "restoring".to_string(), 6064 device_id: None, 6065 }, 6066 ]; 6067 assert!(check_sequential_events( 6068 &expected_events, 6069 &event_path_restored 6070 )); 6071 let latest_events = [&MetaEvent { 6072 event: "restored".to_string(), 6073 device_id: None, 6074 }]; 6075 assert!(check_latest_events_exact( 6076 &latest_events, 6077 &event_path_restored 6078 )); 6079 6080 let r = std::panic::catch_unwind(|| { 6081 // Resume the VM 6082 assert!(remote_command(&api_socket_restored, "resume", None)); 6083 // There is no way that we can ensure the 'write()' to the 6084 // event file is completed when the 'resume' request is 6085 // returned successfully, because the 'write()' was done 6086 // asynchronously from a different thread of Cloud 6087 // Hypervisor (e.g. the event-monitor thread). 6088 thread::sleep(std::time::Duration::new(1, 0)); 6089 let latest_events = [ 6090 &MetaEvent { 6091 event: "resuming".to_string(), 6092 device_id: None, 6093 }, 6094 &MetaEvent { 6095 event: "resumed".to_string(), 6096 device_id: None, 6097 }, 6098 ]; 6099 assert!(check_latest_events_exact( 6100 &latest_events, 6101 &event_path_restored 6102 )); 6103 6104 // Perform same checks to validate VM has been properly restored 6105 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 6106 let total_memory = guest.get_total_memory().unwrap_or_default(); 6107 if !use_hotplug { 6108 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6109 } else { 6110 assert!(total_memory > 4_800_000); 6111 assert!(total_memory < 5_760_000); 6112 // Deflate balloon to restore entire RAM to the VM 6113 resize_command(&api_socket_restored, None, None, Some(0), None); 6114 thread::sleep(std::time::Duration::new(5, 0)); 6115 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6116 // Decrease guest RAM with virtio-mem 6117 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 6118 thread::sleep(std::time::Duration::new(5, 0)); 6119 let total_memory = guest.get_total_memory().unwrap_or_default(); 6120 assert!(total_memory > 4_800_000); 6121 assert!(total_memory < 5_760_000); 6122 } 6123 6124 guest.check_devices_common(Some(&socket), Some(&console_text), None); 6125 }); 6126 // Shutdown the target VM and check console output 6127 let _ = child.kill(); 6128 let output = child.wait_with_output().unwrap(); 6129 handle_child_output(r, &output); 6130 6131 let r = std::panic::catch_unwind(|| { 6132 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 6133 }); 6134 6135 handle_child_output(r, &output); 6136 } 6137 6138 #[test] 6139 fn test_counters() { 6140 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6141 let guest = Guest::new(Box::new(focal)); 6142 let api_socket = temp_api_path(&guest.tmp_dir); 6143 6144 let mut cmd = GuestCommand::new(&guest); 6145 cmd.args(["--cpus", "boot=1"]) 6146 .args(["--memory", "size=512M"]) 6147 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 6148 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6149 .default_disks() 6150 .args(["--net", guest.default_net_string().as_str()]) 6151 .args(["--api-socket", &api_socket]) 6152 .capture_output(); 6153 6154 let mut child = cmd.spawn().unwrap(); 6155 6156 let r = std::panic::catch_unwind(|| { 6157 guest.wait_vm_boot(None).unwrap(); 6158 6159 let orig_counters = get_counters(&api_socket); 6160 guest 6161 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 6162 .unwrap(); 6163 6164 let new_counters = get_counters(&api_socket); 6165 6166 // Check that all the counters have increased 6167 assert!(new_counters > orig_counters); 6168 }); 6169 6170 let _ = child.kill(); 6171 let output = child.wait_with_output().unwrap(); 6172 6173 handle_child_output(r, &output); 6174 } 6175 6176 #[test] 6177 #[cfg(feature = "guest_debug")] 6178 fn test_coredump() { 6179 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6180 let guest = Guest::new(Box::new(focal)); 6181 let api_socket = temp_api_path(&guest.tmp_dir); 6182 6183 let mut cmd = GuestCommand::new(&guest); 6184 cmd.args(["--cpus", "boot=4"]) 6185 .args(["--memory", "size=4G"]) 6186 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6187 .default_disks() 6188 .args(["--net", guest.default_net_string().as_str()]) 6189 .args(["--api-socket", &api_socket]) 6190 .capture_output(); 6191 6192 let mut child = cmd.spawn().unwrap(); 6193 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6194 6195 let r = std::panic::catch_unwind(|| { 6196 guest.wait_vm_boot(None).unwrap(); 6197 6198 assert!(remote_command(&api_socket, "pause", None)); 6199 6200 assert!(remote_command( 6201 &api_socket, 6202 "coredump", 6203 Some(format!("file://{vmcore_file}").as_str()), 6204 )); 6205 6206 // the num of CORE notes should equals to vcpu 6207 let readelf_core_num_cmd = 6208 format!("readelf --all {vmcore_file} |grep CORE |grep -v Type |wc -l"); 6209 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 6210 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 6211 6212 // the num of QEMU notes should equals to vcpu 6213 let readelf_vmm_num_cmd = format!("readelf --all {vmcore_file} |grep QEMU |wc -l"); 6214 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 6215 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 6216 }); 6217 6218 let _ = child.kill(); 6219 let output = child.wait_with_output().unwrap(); 6220 6221 handle_child_output(r, &output); 6222 } 6223 6224 #[test] 6225 #[cfg(feature = "guest_debug")] 6226 fn test_coredump_no_pause() { 6227 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6228 let guest = Guest::new(Box::new(focal)); 6229 let api_socket = temp_api_path(&guest.tmp_dir); 6230 6231 let mut cmd = GuestCommand::new(&guest); 6232 cmd.args(["--cpus", "boot=4"]) 6233 .args(["--memory", "size=4G"]) 6234 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6235 .default_disks() 6236 .args(["--net", guest.default_net_string().as_str()]) 6237 .args(["--api-socket", &api_socket]) 6238 .capture_output(); 6239 6240 let mut child = cmd.spawn().unwrap(); 6241 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6242 6243 let r = std::panic::catch_unwind(|| { 6244 guest.wait_vm_boot(None).unwrap(); 6245 6246 assert!(remote_command( 6247 &api_socket, 6248 "coredump", 6249 Some(format!("file://{vmcore_file}").as_str()), 6250 )); 6251 6252 assert_eq!(vm_state(&api_socket), "Running"); 6253 }); 6254 6255 let _ = child.kill(); 6256 let output = child.wait_with_output().unwrap(); 6257 6258 handle_child_output(r, &output); 6259 } 6260 6261 #[test] 6262 fn test_watchdog() { 6263 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6264 let guest = Guest::new(Box::new(focal)); 6265 let api_socket = temp_api_path(&guest.tmp_dir); 6266 6267 let kernel_path = direct_kernel_boot_path(); 6268 6269 let mut cmd = GuestCommand::new(&guest); 6270 cmd.args(["--cpus", "boot=1"]) 6271 .args(["--memory", "size=512M"]) 6272 .args(["--kernel", kernel_path.to_str().unwrap()]) 6273 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6274 .default_disks() 6275 .args(["--net", guest.default_net_string().as_str()]) 6276 .args(["--watchdog"]) 6277 .args(["--api-socket", &api_socket]) 6278 .capture_output(); 6279 6280 let mut child = cmd.spawn().unwrap(); 6281 6282 let r = std::panic::catch_unwind(|| { 6283 guest.wait_vm_boot(None).unwrap(); 6284 6285 let mut expected_reboot_count = 1; 6286 6287 // Enable the watchdog with a 15s timeout 6288 enable_guest_watchdog(&guest, 15); 6289 6290 // Reboot and check that systemd has activated the watchdog 6291 guest.ssh_command("sudo reboot").unwrap(); 6292 guest.wait_vm_boot(None).unwrap(); 6293 expected_reboot_count += 1; 6294 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6295 assert_eq!( 6296 guest 6297 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6298 .unwrap() 6299 .trim() 6300 .parse::<u32>() 6301 .unwrap_or_default(), 6302 2 6303 ); 6304 6305 // Allow some normal time to elapse to check we don't get spurious reboots 6306 thread::sleep(std::time::Duration::new(40, 0)); 6307 // Check no reboot 6308 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6309 6310 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6311 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6312 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6313 guest.wait_vm_boot(Some(50)).unwrap(); 6314 // Check a reboot is triggered by the watchdog 6315 expected_reboot_count += 1; 6316 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6317 6318 #[cfg(target_arch = "x86_64")] 6319 { 6320 // Now pause the VM and remain offline for 30s 6321 assert!(remote_command(&api_socket, "pause", None)); 6322 thread::sleep(std::time::Duration::new(30, 0)); 6323 assert!(remote_command(&api_socket, "resume", None)); 6324 6325 // Check no reboot 6326 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6327 } 6328 }); 6329 6330 let _ = child.kill(); 6331 let output = child.wait_with_output().unwrap(); 6332 6333 handle_child_output(r, &output); 6334 } 6335 6336 #[test] 6337 fn test_pvpanic() { 6338 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 6339 let guest = Guest::new(Box::new(jammy)); 6340 let api_socket = temp_api_path(&guest.tmp_dir); 6341 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6342 6343 let kernel_path = direct_kernel_boot_path(); 6344 6345 let mut cmd = GuestCommand::new(&guest); 6346 cmd.args(["--cpus", "boot=1"]) 6347 .args(["--memory", "size=512M"]) 6348 .args(["--kernel", kernel_path.to_str().unwrap()]) 6349 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6350 .default_disks() 6351 .args(["--net", guest.default_net_string().as_str()]) 6352 .args(["--pvpanic"]) 6353 .args(["--api-socket", &api_socket]) 6354 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6355 .capture_output(); 6356 6357 let mut child = cmd.spawn().unwrap(); 6358 6359 let r = std::panic::catch_unwind(|| { 6360 guest.wait_vm_boot(None).unwrap(); 6361 6362 // Trigger guest a panic 6363 make_guest_panic(&guest); 6364 6365 // Wait a while for guest 6366 thread::sleep(std::time::Duration::new(10, 0)); 6367 6368 let expected_sequential_events = [&MetaEvent { 6369 event: "panic".to_string(), 6370 device_id: None, 6371 }]; 6372 assert!(check_latest_events_exact( 6373 &expected_sequential_events, 6374 &event_path 6375 )); 6376 }); 6377 6378 let _ = child.kill(); 6379 let output = child.wait_with_output().unwrap(); 6380 6381 handle_child_output(r, &output); 6382 } 6383 6384 #[test] 6385 fn test_tap_from_fd() { 6386 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6387 let guest = Guest::new(Box::new(focal)); 6388 let kernel_path = direct_kernel_boot_path(); 6389 6390 // Create a TAP interface with multi-queue enabled 6391 let num_queue_pairs: usize = 2; 6392 6393 use std::str::FromStr; 6394 let taps = net_util::open_tap( 6395 Some("chtap0"), 6396 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6397 None, 6398 &mut None, 6399 None, 6400 num_queue_pairs, 6401 Some(libc::O_RDWR | libc::O_NONBLOCK), 6402 ) 6403 .unwrap(); 6404 6405 let mut child = GuestCommand::new(&guest) 6406 .args(["--cpus", &format!("boot={num_queue_pairs}")]) 6407 .args(["--memory", "size=512M"]) 6408 .args(["--kernel", kernel_path.to_str().unwrap()]) 6409 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6410 .default_disks() 6411 .args([ 6412 "--net", 6413 &format!( 6414 "fd=[{},{}],mac={},num_queues={}", 6415 taps[0].as_raw_fd(), 6416 taps[1].as_raw_fd(), 6417 guest.network.guest_mac, 6418 num_queue_pairs * 2 6419 ), 6420 ]) 6421 .capture_output() 6422 .spawn() 6423 .unwrap(); 6424 6425 let r = std::panic::catch_unwind(|| { 6426 guest.wait_vm_boot(None).unwrap(); 6427 6428 assert_eq!( 6429 guest 6430 .ssh_command("ip -o link | wc -l") 6431 .unwrap() 6432 .trim() 6433 .parse::<u32>() 6434 .unwrap_or_default(), 6435 2 6436 ); 6437 6438 guest.reboot_linux(0, None); 6439 6440 assert_eq!( 6441 guest 6442 .ssh_command("ip -o link | wc -l") 6443 .unwrap() 6444 .trim() 6445 .parse::<u32>() 6446 .unwrap_or_default(), 6447 2 6448 ); 6449 }); 6450 6451 let _ = child.kill(); 6452 let output = child.wait_with_output().unwrap(); 6453 6454 handle_child_output(r, &output); 6455 } 6456 6457 // By design, a guest VM won't be able to connect to the host 6458 // machine when using a macvtap network interface (while it can 6459 // communicate externally). As a workaround, this integration 6460 // test creates two macvtap interfaces in 'bridge' mode on the 6461 // same physical net interface, one for the guest and one for 6462 // the host. With additional setup on the IP address and the 6463 // routing table, it enables the communications between the 6464 // guest VM and the host machine. 6465 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6466 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6467 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6468 let guest = Guest::new(Box::new(focal)); 6469 let api_socket = temp_api_path(&guest.tmp_dir); 6470 6471 #[cfg(target_arch = "x86_64")] 6472 let kernel_path = direct_kernel_boot_path(); 6473 #[cfg(target_arch = "aarch64")] 6474 let kernel_path = edk2_path(); 6475 6476 let phy_net = "eth0"; 6477 6478 // Create a macvtap interface for the guest VM to use 6479 assert!(exec_host_command_status(&format!( 6480 "sudo ip link add link {phy_net} name {guest_macvtap_name} type macvtap mod bridge" 6481 )) 6482 .success()); 6483 assert!(exec_host_command_status(&format!( 6484 "sudo ip link set {} address {} up", 6485 guest_macvtap_name, guest.network.guest_mac 6486 )) 6487 .success()); 6488 assert!( 6489 exec_host_command_status(&format!("sudo ip link show {guest_macvtap_name}")).success() 6490 ); 6491 6492 let tap_index = 6493 fs::read_to_string(format!("/sys/class/net/{guest_macvtap_name}/ifindex")).unwrap(); 6494 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6495 6496 assert!(exec_host_command_status(&format!("sudo chown $UID.$UID {tap_device}")).success()); 6497 6498 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6499 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6500 assert!(tap_fd1 > 0); 6501 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6502 assert!(tap_fd2 > 0); 6503 6504 // Create a macvtap on the same physical net interface for 6505 // the host machine to use 6506 assert!(exec_host_command_status(&format!( 6507 "sudo ip link add link {phy_net} name {host_macvtap_name} type macvtap mod bridge" 6508 )) 6509 .success()); 6510 // Use default mask "255.255.255.0" 6511 assert!(exec_host_command_status(&format!( 6512 "sudo ip address add {}/24 dev {}", 6513 guest.network.host_ip, host_macvtap_name 6514 )) 6515 .success()); 6516 assert!( 6517 exec_host_command_status(&format!("sudo ip link set dev {host_macvtap_name} up")) 6518 .success() 6519 ); 6520 6521 let mut guest_command = GuestCommand::new(&guest); 6522 guest_command 6523 .args(["--cpus", "boot=2"]) 6524 .args(["--memory", "size=512M"]) 6525 .args(["--kernel", kernel_path.to_str().unwrap()]) 6526 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6527 .default_disks() 6528 .args(["--api-socket", &api_socket]); 6529 6530 let net_params = format!( 6531 "fd=[{},{}],mac={},num_queues=4", 6532 tap_fd1, tap_fd2, guest.network.guest_mac 6533 ); 6534 6535 if !hotplug { 6536 guest_command.args(["--net", &net_params]); 6537 } 6538 6539 let mut child = guest_command.capture_output().spawn().unwrap(); 6540 6541 if hotplug { 6542 // Give some time to the VMM process to listen to the API 6543 // socket. This is the only requirement to avoid the following 6544 // call to ch-remote from failing. 6545 thread::sleep(std::time::Duration::new(10, 0)); 6546 // Hotplug the virtio-net device 6547 let (cmd_success, cmd_output) = 6548 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6549 assert!(cmd_success); 6550 #[cfg(target_arch = "x86_64")] 6551 assert!(String::from_utf8_lossy(&cmd_output) 6552 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6553 #[cfg(target_arch = "aarch64")] 6554 assert!(String::from_utf8_lossy(&cmd_output) 6555 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6556 } 6557 6558 // The functional connectivity provided by the virtio-net device 6559 // gets tested through wait_vm_boot() as it expects to receive a 6560 // HTTP request, and through the SSH command as well. 6561 let r = std::panic::catch_unwind(|| { 6562 guest.wait_vm_boot(None).unwrap(); 6563 6564 assert_eq!( 6565 guest 6566 .ssh_command("ip -o link | wc -l") 6567 .unwrap() 6568 .trim() 6569 .parse::<u32>() 6570 .unwrap_or_default(), 6571 2 6572 ); 6573 6574 guest.reboot_linux(0, None); 6575 6576 assert_eq!( 6577 guest 6578 .ssh_command("ip -o link | wc -l") 6579 .unwrap() 6580 .trim() 6581 .parse::<u32>() 6582 .unwrap_or_default(), 6583 2 6584 ); 6585 }); 6586 6587 let _ = child.kill(); 6588 6589 exec_host_command_status(&format!("sudo ip link del {guest_macvtap_name}")); 6590 exec_host_command_status(&format!("sudo ip link del {host_macvtap_name}")); 6591 6592 let output = child.wait_with_output().unwrap(); 6593 6594 handle_child_output(r, &output); 6595 } 6596 6597 #[test] 6598 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6599 fn test_macvtap() { 6600 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6601 } 6602 6603 #[test] 6604 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6605 fn test_macvtap_hotplug() { 6606 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6607 } 6608 6609 #[test] 6610 #[cfg(not(feature = "mshv"))] 6611 fn test_ovs_dpdk() { 6612 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6613 let guest1 = Guest::new(Box::new(focal1)); 6614 6615 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6616 let guest2 = Guest::new(Box::new(focal2)); 6617 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6618 6619 let (mut child1, mut child2) = 6620 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6621 6622 // Create the snapshot directory 6623 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6624 6625 let r = std::panic::catch_unwind(|| { 6626 // Remove one of the two ports from the OVS bridge 6627 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6628 6629 // Spawn a new netcat listener in the first VM 6630 let guest_ip = guest1.network.guest_ip.clone(); 6631 thread::spawn(move || { 6632 ssh_command_ip( 6633 "nc -l 12345", 6634 &guest_ip, 6635 DEFAULT_SSH_RETRIES, 6636 DEFAULT_SSH_TIMEOUT, 6637 ) 6638 .unwrap(); 6639 }); 6640 6641 // Wait for the server to be listening 6642 thread::sleep(std::time::Duration::new(5, 0)); 6643 6644 // Check the connection fails this time 6645 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6646 6647 // Add the OVS port back 6648 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6649 6650 // And finally check the connection is functional again 6651 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6652 6653 // Pause the VM 6654 assert!(remote_command(&api_socket_source, "pause", None)); 6655 6656 // Take a snapshot from the VM 6657 assert!(remote_command( 6658 &api_socket_source, 6659 "snapshot", 6660 Some(format!("file://{snapshot_dir}").as_str()), 6661 )); 6662 6663 // Wait to make sure the snapshot is completed 6664 thread::sleep(std::time::Duration::new(10, 0)); 6665 }); 6666 6667 // Shutdown the source VM 6668 let _ = child2.kill(); 6669 let output = child2.wait_with_output().unwrap(); 6670 handle_child_output(r, &output); 6671 6672 // Remove the vhost-user socket file. 6673 Command::new("rm") 6674 .arg("-f") 6675 .arg("/tmp/dpdkvhostclient2") 6676 .output() 6677 .unwrap(); 6678 6679 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6680 // Restore the VM from the snapshot 6681 let mut child2 = GuestCommand::new(&guest2) 6682 .args(["--api-socket", &api_socket_restored]) 6683 .args([ 6684 "--restore", 6685 format!("source_url=file://{snapshot_dir}").as_str(), 6686 ]) 6687 .capture_output() 6688 .spawn() 6689 .unwrap(); 6690 6691 // Wait for the VM to be restored 6692 thread::sleep(std::time::Duration::new(10, 0)); 6693 6694 let r = std::panic::catch_unwind(|| { 6695 // Resume the VM 6696 assert!(remote_command(&api_socket_restored, "resume", None)); 6697 6698 // Spawn a new netcat listener in the first VM 6699 let guest_ip = guest1.network.guest_ip.clone(); 6700 thread::spawn(move || { 6701 ssh_command_ip( 6702 "nc -l 12345", 6703 &guest_ip, 6704 DEFAULT_SSH_RETRIES, 6705 DEFAULT_SSH_TIMEOUT, 6706 ) 6707 .unwrap(); 6708 }); 6709 6710 // Wait for the server to be listening 6711 thread::sleep(std::time::Duration::new(5, 0)); 6712 6713 // And check the connection is still functional after restore 6714 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6715 }); 6716 6717 let _ = child1.kill(); 6718 let _ = child2.kill(); 6719 6720 let output = child1.wait_with_output().unwrap(); 6721 child2.wait().unwrap(); 6722 6723 cleanup_ovs_dpdk(); 6724 6725 handle_child_output(r, &output); 6726 } 6727 6728 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6729 cleanup_spdk_nvme(); 6730 6731 assert!(exec_host_command_status(&format!( 6732 "mkdir -p {}", 6733 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6734 )) 6735 .success()); 6736 assert!(exec_host_command_status(&format!( 6737 "truncate {} -s 128M", 6738 nvme_dir.join("test-disk.raw").to_str().unwrap() 6739 )) 6740 .success()); 6741 assert!(exec_host_command_status(&format!( 6742 "mkfs.ext4 {}", 6743 nvme_dir.join("test-disk.raw").to_str().unwrap() 6744 )) 6745 .success()); 6746 6747 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6748 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6749 .args(["-i", "0", "-m", "0x1"]) 6750 .spawn() 6751 .unwrap(); 6752 thread::sleep(std::time::Duration::new(2, 0)); 6753 6754 assert!(exec_host_command_status( 6755 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER" 6756 ) 6757 .success()); 6758 assert!(exec_host_command_status(&format!( 6759 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6760 nvme_dir.join("test-disk.raw").to_str().unwrap() 6761 )) 6762 .success()); 6763 assert!(exec_host_command_status( 6764 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6765 ) 6766 .success()); 6767 assert!(exec_host_command_status( 6768 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6769 ) 6770 .success()); 6771 assert!(exec_host_command_status(&format!( 6772 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6773 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6774 )) 6775 .success()); 6776 } 6777 6778 fn cleanup_spdk_nvme() { 6779 exec_host_command_status("pkill -f nvmf_tgt"); 6780 } 6781 6782 #[test] 6783 fn test_vfio_user() { 6784 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6785 let jammy = UbuntuDiskConfig::new(jammy_image); 6786 let guest = Guest::new(Box::new(jammy)); 6787 6788 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6789 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6790 6791 let api_socket = temp_api_path(&guest.tmp_dir); 6792 let mut child = GuestCommand::new(&guest) 6793 .args(["--api-socket", &api_socket]) 6794 .args(["--cpus", "boot=1"]) 6795 .args(["--memory", "size=512M,shared=on,hugepages=on"]) 6796 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6797 .args(["--serial", "tty", "--console", "off"]) 6798 .default_disks() 6799 .default_net() 6800 .capture_output() 6801 .spawn() 6802 .unwrap(); 6803 6804 let r = std::panic::catch_unwind(|| { 6805 guest.wait_vm_boot(None).unwrap(); 6806 6807 // Hotplug the SPDK-NVMe device to the VM 6808 let (cmd_success, cmd_output) = remote_command_w_output( 6809 &api_socket, 6810 "add-user-device", 6811 Some(&format!( 6812 "socket={},id=vfio_user0", 6813 spdk_nvme_dir 6814 .as_path() 6815 .join("nvme-vfio-user/cntrl") 6816 .to_str() 6817 .unwrap(), 6818 )), 6819 ); 6820 assert!(cmd_success); 6821 assert!(String::from_utf8_lossy(&cmd_output) 6822 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6823 6824 thread::sleep(std::time::Duration::new(10, 0)); 6825 6826 // Check both if /dev/nvme exists and if the block size is 128M. 6827 assert_eq!( 6828 guest 6829 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6830 .unwrap() 6831 .trim() 6832 .parse::<u32>() 6833 .unwrap_or_default(), 6834 1 6835 ); 6836 6837 // Check changes persist after reboot 6838 assert_eq!( 6839 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6840 "" 6841 ); 6842 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6843 guest 6844 .ssh_command("echo test123 | sudo tee /mnt/test") 6845 .unwrap(); 6846 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6847 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6848 6849 guest.reboot_linux(0, None); 6850 assert_eq!( 6851 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6852 "" 6853 ); 6854 assert_eq!( 6855 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6856 "test123" 6857 ); 6858 }); 6859 6860 cleanup_spdk_nvme(); 6861 6862 let _ = child.kill(); 6863 let output = child.wait_with_output().unwrap(); 6864 6865 handle_child_output(r, &output); 6866 } 6867 6868 #[test] 6869 #[cfg(target_arch = "x86_64")] 6870 fn test_vdpa_block() { 6871 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6872 if !exec_host_command_status("lsmod | grep vdpa_sim_blk").success() { 6873 return; 6874 } 6875 6876 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6877 let guest = Guest::new(Box::new(focal)); 6878 let api_socket = temp_api_path(&guest.tmp_dir); 6879 6880 let kernel_path = direct_kernel_boot_path(); 6881 6882 let mut child = GuestCommand::new(&guest) 6883 .args(["--cpus", "boot=2"]) 6884 .args(["--memory", "size=512M,hugepages=on"]) 6885 .args(["--kernel", kernel_path.to_str().unwrap()]) 6886 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6887 .default_disks() 6888 .default_net() 6889 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6890 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 6891 .args(["--api-socket", &api_socket]) 6892 .capture_output() 6893 .spawn() 6894 .unwrap(); 6895 6896 let r = std::panic::catch_unwind(|| { 6897 guest.wait_vm_boot(None).unwrap(); 6898 6899 // Check both if /dev/vdc exists and if the block size is 128M. 6900 assert_eq!( 6901 guest 6902 .ssh_command("lsblk | grep vdc | grep -c 128M") 6903 .unwrap() 6904 .trim() 6905 .parse::<u32>() 6906 .unwrap_or_default(), 6907 1 6908 ); 6909 6910 // Check the content of the block device after we wrote to it. 6911 // The vpda-sim-blk should let us read what we previously wrote. 6912 guest 6913 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6914 .unwrap(); 6915 assert_eq!( 6916 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6917 "foobar" 6918 ); 6919 6920 // Hotplug an extra vDPA block device behind the vIOMMU 6921 // Add a new vDPA device to the VM 6922 let (cmd_success, cmd_output) = remote_command_w_output( 6923 &api_socket, 6924 "add-vdpa", 6925 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6926 ); 6927 assert!(cmd_success); 6928 assert!(String::from_utf8_lossy(&cmd_output) 6929 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6930 6931 thread::sleep(std::time::Duration::new(10, 0)); 6932 6933 // Check IOMMU setup 6934 assert!(guest 6935 .does_device_vendor_pair_match("0x1057", "0x1af4") 6936 .unwrap_or_default()); 6937 assert_eq!( 6938 guest 6939 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6940 .unwrap() 6941 .trim(), 6942 "0001:00:01.0" 6943 ); 6944 6945 // Check both if /dev/vdd exists and if the block size is 128M. 6946 assert_eq!( 6947 guest 6948 .ssh_command("lsblk | grep vdd | grep -c 128M") 6949 .unwrap() 6950 .trim() 6951 .parse::<u32>() 6952 .unwrap_or_default(), 6953 1 6954 ); 6955 6956 // Write some content to the block device we've just plugged. 6957 guest 6958 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6959 .unwrap(); 6960 6961 // Check we can read the content back. 6962 assert_eq!( 6963 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6964 "foobar" 6965 ); 6966 6967 // Unplug the device 6968 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6969 assert!(cmd_success); 6970 thread::sleep(std::time::Duration::new(10, 0)); 6971 6972 // Check /dev/vdd doesn't exist anymore 6973 assert_eq!( 6974 guest 6975 .ssh_command("lsblk | grep -c vdd || true") 6976 .unwrap() 6977 .trim() 6978 .parse::<u32>() 6979 .unwrap_or(1), 6980 0 6981 ); 6982 }); 6983 6984 let _ = child.kill(); 6985 let output = child.wait_with_output().unwrap(); 6986 6987 handle_child_output(r, &output); 6988 } 6989 6990 #[test] 6991 #[cfg(target_arch = "x86_64")] 6992 #[ignore = "See #5756"] 6993 fn test_vdpa_net() { 6994 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6995 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6996 return; 6997 } 6998 6999 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7000 let guest = Guest::new(Box::new(focal)); 7001 7002 let kernel_path = direct_kernel_boot_path(); 7003 7004 let mut child = GuestCommand::new(&guest) 7005 .args(["--cpus", "boot=2"]) 7006 .args(["--memory", "size=512M,hugepages=on"]) 7007 .args(["--kernel", kernel_path.to_str().unwrap()]) 7008 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7009 .default_disks() 7010 .default_net() 7011 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 7012 .capture_output() 7013 .spawn() 7014 .unwrap(); 7015 7016 let r = std::panic::catch_unwind(|| { 7017 guest.wait_vm_boot(None).unwrap(); 7018 7019 // Check we can find network interface related to vDPA device 7020 assert_eq!( 7021 guest 7022 .ssh_command("ip -o link | grep -c ens6") 7023 .unwrap() 7024 .trim() 7025 .parse::<u32>() 7026 .unwrap_or(0), 7027 1 7028 ); 7029 7030 guest 7031 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 7032 .unwrap(); 7033 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 7034 7035 // Check there is no packet yet on both TX/RX of the network interface 7036 assert_eq!( 7037 guest 7038 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 7039 .unwrap() 7040 .trim() 7041 .parse::<u32>() 7042 .unwrap_or(0), 7043 2 7044 ); 7045 7046 // Send 6 packets with ping command 7047 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 7048 7049 // Check we can find 6 packets on both TX/RX of the network interface 7050 assert_eq!( 7051 guest 7052 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 7053 .unwrap() 7054 .trim() 7055 .parse::<u32>() 7056 .unwrap_or(0), 7057 2 7058 ); 7059 7060 // No need to check for hotplug as we already tested it through 7061 // test_vdpa_block() 7062 }); 7063 7064 let _ = child.kill(); 7065 let output = child.wait_with_output().unwrap(); 7066 7067 handle_child_output(r, &output); 7068 } 7069 7070 #[test] 7071 #[cfg(target_arch = "x86_64")] 7072 fn test_tpm() { 7073 let focal = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7074 let guest = Guest::new(Box::new(focal)); 7075 7076 let (mut swtpm_command, swtpm_socket_path) = prepare_swtpm_daemon(&guest.tmp_dir); 7077 7078 let mut guest_cmd = GuestCommand::new(&guest); 7079 guest_cmd 7080 .args(["--cpus", "boot=1"]) 7081 .args(["--memory", "size=512M"]) 7082 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7083 .args(["--tpm", &format!("socket={swtpm_socket_path}")]) 7084 .capture_output() 7085 .default_disks() 7086 .default_net(); 7087 7088 // Start swtpm daemon 7089 let mut swtpm_child = swtpm_command.spawn().unwrap(); 7090 thread::sleep(std::time::Duration::new(10, 0)); 7091 let mut child = guest_cmd.spawn().unwrap(); 7092 let r = std::panic::catch_unwind(|| { 7093 guest.wait_vm_boot(None).unwrap(); 7094 assert_eq!( 7095 guest.ssh_command("ls /dev/tpm0").unwrap().trim(), 7096 "/dev/tpm0" 7097 ); 7098 guest.ssh_command("sudo tpm2_selftest -f").unwrap(); 7099 guest 7100 .ssh_command("echo 'hello' > /tmp/checksum_test; ") 7101 .unwrap(); 7102 guest.ssh_command("cmp <(sudo tpm2_pcrevent /tmp/checksum_test | grep sha256 | awk '{print $2}') <(sha256sum /tmp/checksum_test| awk '{print $1}')").unwrap(); 7103 }); 7104 7105 let _ = swtpm_child.kill(); 7106 let _d_out = swtpm_child.wait_with_output().unwrap(); 7107 7108 let _ = child.kill(); 7109 let output = child.wait_with_output().unwrap(); 7110 7111 handle_child_output(r, &output); 7112 } 7113 7114 #[test] 7115 #[cfg(target_arch = "x86_64")] 7116 fn test_double_tty() { 7117 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7118 let guest = Guest::new(Box::new(focal)); 7119 let mut cmd = GuestCommand::new(&guest); 7120 let api_socket = temp_api_path(&guest.tmp_dir); 7121 let tty_str: &str = "console=hvc0 earlyprintk=ttyS0 "; 7122 // linux printk module enable console log. 7123 let con_dis_str: &str = "console [hvc0] enabled"; 7124 // linux printk module disable console log. 7125 let con_enb_str: &str = "bootconsole [earlyser0] disabled"; 7126 7127 let kernel_path = direct_kernel_boot_path(); 7128 7129 cmd.args(["--cpus", "boot=1"]) 7130 .args(["--memory", "size=512M"]) 7131 .args(["--kernel", kernel_path.to_str().unwrap()]) 7132 .args([ 7133 "--cmdline", 7134 DIRECT_KERNEL_BOOT_CMDLINE 7135 .replace("console=hvc0 ", tty_str) 7136 .as_str(), 7137 ]) 7138 .capture_output() 7139 .default_disks() 7140 .default_net() 7141 .args(["--serial", "tty"]) 7142 .args(["--console", "tty"]) 7143 .args(["--api-socket", &api_socket]); 7144 7145 let mut child = cmd.spawn().unwrap(); 7146 7147 let mut r = std::panic::catch_unwind(|| { 7148 guest.wait_vm_boot(None).unwrap(); 7149 }); 7150 7151 let _ = child.kill(); 7152 let output = child.wait_with_output().unwrap(); 7153 7154 if r.is_ok() { 7155 r = std::panic::catch_unwind(|| { 7156 let s = String::from_utf8_lossy(&output.stdout); 7157 assert!(s.contains(tty_str)); 7158 assert!(s.contains(con_dis_str)); 7159 assert!(s.contains(con_enb_str)); 7160 }); 7161 } 7162 7163 handle_child_output(r, &output); 7164 } 7165 } 7166 7167 mod dbus_api { 7168 use crate::*; 7169 7170 // Start cloud-hypervisor with no VM parameters, running both the HTTP 7171 // and DBus APIs. Alternate calls to the external APIs (HTTP and DBus) 7172 // to create a VM, boot it, and verify that it can be shut down and then 7173 // booted again. 7174 #[test] 7175 fn test_api_dbus_and_http_interleaved() { 7176 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7177 let guest = Guest::new(Box::new(focal)); 7178 let dbus_api = TargetApi::new_dbus_api(&guest.tmp_dir); 7179 let http_api = TargetApi::new_http_api(&guest.tmp_dir); 7180 7181 let mut child = GuestCommand::new(&guest) 7182 .args(dbus_api.guest_args()) 7183 .args(http_api.guest_args()) 7184 .capture_output() 7185 .spawn() 7186 .unwrap(); 7187 7188 thread::sleep(std::time::Duration::new(1, 0)); 7189 7190 // Verify API servers are running 7191 assert!(dbus_api.remote_command("ping", None)); 7192 assert!(http_api.remote_command("ping", None)); 7193 7194 // Create the VM first 7195 let cpu_count: u8 = 4; 7196 let request_body = guest.api_create_body( 7197 cpu_count, 7198 direct_kernel_boot_path().to_str().unwrap(), 7199 DIRECT_KERNEL_BOOT_CMDLINE, 7200 ); 7201 7202 let temp_config_path = guest.tmp_dir.as_path().join("config"); 7203 std::fs::write(&temp_config_path, request_body).unwrap(); 7204 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 7205 7206 let r = std::panic::catch_unwind(|| { 7207 // Create the VM 7208 assert!(dbus_api.remote_command("create", Some(create_config),)); 7209 7210 // Then boot it 7211 assert!(http_api.remote_command("boot", None)); 7212 guest.wait_vm_boot(None).unwrap(); 7213 7214 // Check that the VM booted as expected 7215 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7216 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7217 7218 // Sync and shutdown without powering off to prevent filesystem 7219 // corruption. 7220 guest.ssh_command("sync").unwrap(); 7221 guest.ssh_command("sudo shutdown -H now").unwrap(); 7222 7223 // Wait for the guest to be fully shutdown 7224 thread::sleep(std::time::Duration::new(20, 0)); 7225 7226 // Then shutdown the VM 7227 assert!(dbus_api.remote_command("shutdown", None)); 7228 7229 // Then boot it again 7230 assert!(http_api.remote_command("boot", None)); 7231 guest.wait_vm_boot(None).unwrap(); 7232 7233 // Check that the VM booted as expected 7234 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7235 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7236 }); 7237 7238 let _ = child.kill(); 7239 let output = child.wait_with_output().unwrap(); 7240 7241 handle_child_output(r, &output); 7242 } 7243 7244 #[test] 7245 fn test_api_dbus_create_boot() { 7246 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7247 let guest = Guest::new(Box::new(focal)); 7248 7249 _test_api_create_boot(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7250 } 7251 7252 #[test] 7253 fn test_api_dbus_shutdown() { 7254 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7255 let guest = Guest::new(Box::new(focal)); 7256 7257 _test_api_shutdown(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7258 } 7259 7260 #[test] 7261 fn test_api_dbus_delete() { 7262 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7263 let guest = Guest::new(Box::new(focal)); 7264 7265 _test_api_delete(TargetApi::new_dbus_api(&guest.tmp_dir), guest); 7266 } 7267 7268 #[test] 7269 fn test_api_dbus_pause_resume() { 7270 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7271 let guest = Guest::new(Box::new(focal)); 7272 7273 _test_api_pause_resume(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7274 } 7275 } 7276 7277 mod common_sequential { 7278 #[cfg(not(feature = "mshv"))] 7279 use crate::*; 7280 7281 #[test] 7282 #[cfg(not(feature = "mshv"))] 7283 fn test_memory_mergeable_on() { 7284 test_memory_mergeable(true) 7285 } 7286 } 7287 7288 mod windows { 7289 use crate::*; 7290 use once_cell::sync::Lazy; 7291 7292 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 7293 7294 struct WindowsGuest { 7295 guest: Guest, 7296 auth: PasswordAuth, 7297 } 7298 7299 trait FsType { 7300 const FS_FAT: u8; 7301 const FS_NTFS: u8; 7302 } 7303 impl FsType for WindowsGuest { 7304 const FS_FAT: u8 = 0; 7305 const FS_NTFS: u8 = 1; 7306 } 7307 7308 impl WindowsGuest { 7309 fn new() -> Self { 7310 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 7311 let guest = Guest::new(Box::new(disk)); 7312 let auth = PasswordAuth { 7313 username: String::from("administrator"), 7314 password: String::from("Admin123"), 7315 }; 7316 7317 WindowsGuest { guest, auth } 7318 } 7319 7320 fn guest(&self) -> &Guest { 7321 &self.guest 7322 } 7323 7324 fn ssh_cmd(&self, cmd: &str) -> String { 7325 ssh_command_ip_with_auth( 7326 cmd, 7327 &self.auth, 7328 &self.guest.network.guest_ip, 7329 DEFAULT_SSH_RETRIES, 7330 DEFAULT_SSH_TIMEOUT, 7331 ) 7332 .unwrap() 7333 } 7334 7335 fn cpu_count(&self) -> u8 { 7336 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 7337 .trim() 7338 .parse::<u8>() 7339 .unwrap_or(0) 7340 } 7341 7342 fn ram_size(&self) -> usize { 7343 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 7344 .trim() 7345 .parse::<usize>() 7346 .unwrap_or(0) 7347 } 7348 7349 fn netdev_count(&self) -> u8 { 7350 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7351 .trim() 7352 .parse::<u8>() 7353 .unwrap_or(0) 7354 } 7355 7356 fn disk_count(&self) -> u8 { 7357 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7358 .trim() 7359 .parse::<u8>() 7360 .unwrap_or(0) 7361 } 7362 7363 fn reboot(&self) { 7364 let _ = self.ssh_cmd("shutdown /r /t 0"); 7365 } 7366 7367 fn shutdown(&self) { 7368 let _ = self.ssh_cmd("shutdown /s /t 0"); 7369 } 7370 7371 fn run_dnsmasq(&self) -> std::process::Child { 7372 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 7373 let dhcp_host = format!( 7374 "--dhcp-host={},{}", 7375 self.guest.network.guest_mac, self.guest.network.guest_ip 7376 ); 7377 let dhcp_range = format!( 7378 "--dhcp-range=eth,{},{}", 7379 self.guest.network.guest_ip, self.guest.network.guest_ip 7380 ); 7381 7382 Command::new("dnsmasq") 7383 .arg("--no-daemon") 7384 .arg("--log-queries") 7385 .arg(listen_address.as_str()) 7386 .arg("--except-interface=lo") 7387 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 7388 .arg("--conf-file=/dev/null") 7389 .arg(dhcp_host.as_str()) 7390 .arg(dhcp_range.as_str()) 7391 .spawn() 7392 .unwrap() 7393 } 7394 7395 // TODO Cleanup image file explicitly after test, if there's some space issues. 7396 fn disk_new(&self, fs: u8, sz: usize) -> String { 7397 let mut guard = NEXT_DISK_ID.lock().unwrap(); 7398 let id = *guard; 7399 *guard = id + 1; 7400 7401 let img = PathBuf::from(format!("/tmp/test-hotplug-{id}.raw")); 7402 let _ = fs::remove_file(&img); 7403 7404 // Create an image file 7405 let out = Command::new("qemu-img") 7406 .args([ 7407 "create", 7408 "-f", 7409 "raw", 7410 img.to_str().unwrap(), 7411 format!("{sz}m").as_str(), 7412 ]) 7413 .output() 7414 .expect("qemu-img command failed") 7415 .stdout; 7416 println!("{out:?}"); 7417 7418 // Associate image to a loop device 7419 let out = Command::new("losetup") 7420 .args(["--show", "-f", img.to_str().unwrap()]) 7421 .output() 7422 .expect("failed to create loop device") 7423 .stdout; 7424 let _tmp = String::from_utf8_lossy(&out); 7425 let loop_dev = _tmp.trim(); 7426 println!("{out:?}"); 7427 7428 // Create a partition table 7429 // echo 'type=7' | sudo sfdisk "${LOOP}" 7430 let mut child = Command::new("sfdisk") 7431 .args([loop_dev]) 7432 .stdin(Stdio::piped()) 7433 .spawn() 7434 .unwrap(); 7435 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 7436 stdin 7437 .write_all("type=7".as_bytes()) 7438 .expect("failed to write stdin"); 7439 let out = child.wait_with_output().expect("sfdisk failed").stdout; 7440 println!("{out:?}"); 7441 7442 // Disengage the loop device 7443 let out = Command::new("losetup") 7444 .args(["-d", loop_dev]) 7445 .output() 7446 .expect("loop device not found") 7447 .stdout; 7448 println!("{out:?}"); 7449 7450 // Re-associate loop device pointing to the partition only 7451 let out = Command::new("losetup") 7452 .args([ 7453 "--show", 7454 "--offset", 7455 (512 * 2048).to_string().as_str(), 7456 "-f", 7457 img.to_str().unwrap(), 7458 ]) 7459 .output() 7460 .expect("failed to create loop device") 7461 .stdout; 7462 let _tmp = String::from_utf8_lossy(&out); 7463 let loop_dev = _tmp.trim(); 7464 println!("{out:?}"); 7465 7466 // Create filesystem. 7467 let fs_cmd = match fs { 7468 WindowsGuest::FS_FAT => "mkfs.msdos", 7469 WindowsGuest::FS_NTFS => "mkfs.ntfs", 7470 _ => panic!("Unknown filesystem type '{fs}'"), 7471 }; 7472 let out = Command::new(fs_cmd) 7473 .args([&loop_dev]) 7474 .output() 7475 .unwrap_or_else(|_| panic!("{fs_cmd} failed")) 7476 .stdout; 7477 println!("{out:?}"); 7478 7479 // Disengage the loop device 7480 let out = Command::new("losetup") 7481 .args(["-d", loop_dev]) 7482 .output() 7483 .unwrap_or_else(|_| panic!("loop device '{loop_dev}' not found")) 7484 .stdout; 7485 println!("{out:?}"); 7486 7487 img.to_str().unwrap().to_string() 7488 } 7489 7490 fn disks_set_rw(&self) { 7491 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 7492 } 7493 7494 fn disks_online(&self) { 7495 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 7496 } 7497 7498 fn disk_file_put(&self, fname: &str, data: &str) { 7499 let _ = self.ssh_cmd(&format!( 7500 "powershell -Command \"'{data}' | Set-Content -Path {fname}\"" 7501 )); 7502 } 7503 7504 fn disk_file_read(&self, fname: &str) -> String { 7505 self.ssh_cmd(&format!( 7506 "powershell -Command \"Get-Content -Path {fname}\"" 7507 )) 7508 } 7509 7510 fn wait_for_boot(&self) -> bool { 7511 let cmd = "dir /b c:\\ | find \"Windows\""; 7512 let tmo_max = 180; 7513 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 7514 // interval must be small. 7515 let tmo_int = 2; 7516 let out = ssh_command_ip_with_auth( 7517 cmd, 7518 &self.auth, 7519 &self.guest.network.guest_ip, 7520 { 7521 let mut ret = 1; 7522 let mut tmo_acc = 0; 7523 loop { 7524 tmo_acc += tmo_int * ret; 7525 if tmo_acc >= tmo_max { 7526 break; 7527 } 7528 ret += 1; 7529 } 7530 ret 7531 }, 7532 tmo_int, 7533 ) 7534 .unwrap(); 7535 7536 if "Windows" == out.trim() { 7537 return true; 7538 } 7539 7540 false 7541 } 7542 } 7543 7544 fn vcpu_threads_count(pid: u32) -> u8 { 7545 // ps -T -p 12345 | grep vcpu | wc -l 7546 let out = Command::new("ps") 7547 .args(["-T", "-p", format!("{pid}").as_str()]) 7548 .output() 7549 .expect("ps command failed") 7550 .stdout; 7551 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 7552 } 7553 7554 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 7555 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 7556 let out = Command::new("ps") 7557 .args(["-T", "-p", format!("{pid}").as_str()]) 7558 .output() 7559 .expect("ps command failed") 7560 .stdout; 7561 let mut n = 0; 7562 String::from_utf8_lossy(&out) 7563 .split_whitespace() 7564 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 7565 n 7566 } 7567 7568 fn disk_ctrl_threads_count(pid: u32) -> u8 { 7569 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 7570 let out = Command::new("ps") 7571 .args(["-T", "-p", format!("{pid}").as_str()]) 7572 .output() 7573 .expect("ps command failed") 7574 .stdout; 7575 let mut n = 0; 7576 String::from_utf8_lossy(&out) 7577 .split_whitespace() 7578 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 7579 n 7580 } 7581 7582 #[test] 7583 fn test_windows_guest() { 7584 let windows_guest = WindowsGuest::new(); 7585 7586 let mut child = GuestCommand::new(windows_guest.guest()) 7587 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7588 .args(["--memory", "size=4G"]) 7589 .args(["--kernel", edk2_path().to_str().unwrap()]) 7590 .args(["--serial", "tty"]) 7591 .args(["--console", "off"]) 7592 .default_disks() 7593 .default_net() 7594 .capture_output() 7595 .spawn() 7596 .unwrap(); 7597 7598 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7599 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7600 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7601 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7602 7603 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7604 7605 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7606 7607 let r = std::panic::catch_unwind(|| { 7608 // Wait to make sure Windows boots up 7609 assert!(windows_guest.wait_for_boot()); 7610 7611 windows_guest.shutdown(); 7612 }); 7613 7614 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7615 let _ = child.kill(); 7616 let output = child.wait_with_output().unwrap(); 7617 7618 let _ = child_dnsmasq.kill(); 7619 let _ = child_dnsmasq.wait(); 7620 7621 handle_child_output(r, &output); 7622 } 7623 7624 #[test] 7625 fn test_windows_guest_multiple_queues() { 7626 let windows_guest = WindowsGuest::new(); 7627 7628 let mut ovmf_path = dirs::home_dir().unwrap(); 7629 ovmf_path.push("workloads"); 7630 ovmf_path.push(OVMF_NAME); 7631 7632 let mut child = GuestCommand::new(windows_guest.guest()) 7633 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 7634 .args(["--memory", "size=4G"]) 7635 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7636 .args(["--serial", "tty"]) 7637 .args(["--console", "off"]) 7638 .args([ 7639 "--disk", 7640 format!( 7641 "path={},num_queues=4", 7642 windows_guest 7643 .guest() 7644 .disk_config 7645 .disk(DiskType::OperatingSystem) 7646 .unwrap() 7647 ) 7648 .as_str(), 7649 ]) 7650 .args([ 7651 "--net", 7652 format!( 7653 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 7654 windows_guest.guest().network.guest_mac, 7655 windows_guest.guest().network.host_ip 7656 ) 7657 .as_str(), 7658 ]) 7659 .capture_output() 7660 .spawn() 7661 .unwrap(); 7662 7663 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7664 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7665 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7666 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7667 7668 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7669 7670 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7671 7672 let r = std::panic::catch_unwind(|| { 7673 // Wait to make sure Windows boots up 7674 assert!(windows_guest.wait_for_boot()); 7675 7676 windows_guest.shutdown(); 7677 }); 7678 7679 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7680 let _ = child.kill(); 7681 let output = child.wait_with_output().unwrap(); 7682 7683 let _ = child_dnsmasq.kill(); 7684 let _ = child_dnsmasq.wait(); 7685 7686 handle_child_output(r, &output); 7687 } 7688 7689 #[test] 7690 #[cfg(not(feature = "mshv"))] 7691 #[ignore = "See #4327"] 7692 fn test_windows_guest_snapshot_restore() { 7693 let windows_guest = WindowsGuest::new(); 7694 7695 let mut ovmf_path = dirs::home_dir().unwrap(); 7696 ovmf_path.push("workloads"); 7697 ovmf_path.push(OVMF_NAME); 7698 7699 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7700 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 7701 7702 let mut child = GuestCommand::new(windows_guest.guest()) 7703 .args(["--api-socket", &api_socket_source]) 7704 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7705 .args(["--memory", "size=4G"]) 7706 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7707 .args(["--serial", "tty"]) 7708 .args(["--console", "off"]) 7709 .default_disks() 7710 .default_net() 7711 .capture_output() 7712 .spawn() 7713 .unwrap(); 7714 7715 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7716 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7717 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7718 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7719 7720 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7721 7722 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7723 7724 // Wait to make sure Windows boots up 7725 assert!(windows_guest.wait_for_boot()); 7726 7727 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 7728 7729 // Pause the VM 7730 assert!(remote_command(&api_socket_source, "pause", None)); 7731 7732 // Take a snapshot from the VM 7733 assert!(remote_command( 7734 &api_socket_source, 7735 "snapshot", 7736 Some(format!("file://{snapshot_dir}").as_str()), 7737 )); 7738 7739 // Wait to make sure the snapshot is completed 7740 thread::sleep(std::time::Duration::new(30, 0)); 7741 7742 let _ = child.kill(); 7743 child.wait().unwrap(); 7744 7745 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 7746 7747 // Restore the VM from the snapshot 7748 let mut child = GuestCommand::new(windows_guest.guest()) 7749 .args(["--api-socket", &api_socket_restored]) 7750 .args([ 7751 "--restore", 7752 format!("source_url=file://{snapshot_dir}").as_str(), 7753 ]) 7754 .capture_output() 7755 .spawn() 7756 .unwrap(); 7757 7758 // Wait for the VM to be restored 7759 thread::sleep(std::time::Duration::new(20, 0)); 7760 7761 let r = std::panic::catch_unwind(|| { 7762 // Resume the VM 7763 assert!(remote_command(&api_socket_restored, "resume", None)); 7764 7765 windows_guest.shutdown(); 7766 }); 7767 7768 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7769 let _ = child.kill(); 7770 let output = child.wait_with_output().unwrap(); 7771 7772 let _ = child_dnsmasq.kill(); 7773 let _ = child_dnsmasq.wait(); 7774 7775 handle_child_output(r, &output); 7776 } 7777 7778 #[test] 7779 #[cfg(not(feature = "mshv"))] 7780 #[cfg(not(target_arch = "aarch64"))] 7781 fn test_windows_guest_cpu_hotplug() { 7782 let windows_guest = WindowsGuest::new(); 7783 7784 let mut ovmf_path = dirs::home_dir().unwrap(); 7785 ovmf_path.push("workloads"); 7786 ovmf_path.push(OVMF_NAME); 7787 7788 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7789 let api_socket = temp_api_path(&tmp_dir); 7790 7791 let mut child = GuestCommand::new(windows_guest.guest()) 7792 .args(["--api-socket", &api_socket]) 7793 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 7794 .args(["--memory", "size=4G"]) 7795 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7796 .args(["--serial", "tty"]) 7797 .args(["--console", "off"]) 7798 .default_disks() 7799 .default_net() 7800 .capture_output() 7801 .spawn() 7802 .unwrap(); 7803 7804 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7805 7806 let r = std::panic::catch_unwind(|| { 7807 // Wait to make sure Windows boots up 7808 assert!(windows_guest.wait_for_boot()); 7809 7810 let vcpu_num = 2; 7811 // Check the initial number of CPUs the guest sees 7812 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7813 // Check the initial number of vcpu threads in the CH process 7814 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7815 7816 let vcpu_num = 6; 7817 // Hotplug some CPUs 7818 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7819 // Wait to make sure CPUs are added 7820 thread::sleep(std::time::Duration::new(10, 0)); 7821 // Check the guest sees the correct number 7822 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7823 // Check the CH process has the correct number of vcpu threads 7824 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7825 7826 let vcpu_num = 4; 7827 // Remove some CPUs. Note that Windows doesn't support hot-remove. 7828 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7829 // Wait to make sure CPUs are removed 7830 thread::sleep(std::time::Duration::new(10, 0)); 7831 // Reboot to let Windows catch up 7832 windows_guest.reboot(); 7833 // Wait to make sure Windows completely rebooted 7834 thread::sleep(std::time::Duration::new(60, 0)); 7835 // Check the guest sees the correct number 7836 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7837 // Check the CH process has the correct number of vcpu threads 7838 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7839 7840 windows_guest.shutdown(); 7841 }); 7842 7843 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7844 let _ = child.kill(); 7845 let output = child.wait_with_output().unwrap(); 7846 7847 let _ = child_dnsmasq.kill(); 7848 let _ = child_dnsmasq.wait(); 7849 7850 handle_child_output(r, &output); 7851 } 7852 7853 #[test] 7854 #[cfg(not(feature = "mshv"))] 7855 #[cfg(not(target_arch = "aarch64"))] 7856 fn test_windows_guest_ram_hotplug() { 7857 let windows_guest = WindowsGuest::new(); 7858 7859 let mut ovmf_path = dirs::home_dir().unwrap(); 7860 ovmf_path.push("workloads"); 7861 ovmf_path.push(OVMF_NAME); 7862 7863 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7864 let api_socket = temp_api_path(&tmp_dir); 7865 7866 let mut child = GuestCommand::new(windows_guest.guest()) 7867 .args(["--api-socket", &api_socket]) 7868 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7869 .args(["--memory", "size=2G,hotplug_size=5G"]) 7870 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7871 .args(["--serial", "tty"]) 7872 .args(["--console", "off"]) 7873 .default_disks() 7874 .default_net() 7875 .capture_output() 7876 .spawn() 7877 .unwrap(); 7878 7879 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7880 7881 let r = std::panic::catch_unwind(|| { 7882 // Wait to make sure Windows boots up 7883 assert!(windows_guest.wait_for_boot()); 7884 7885 let ram_size = 2 * 1024 * 1024 * 1024; 7886 // Check the initial number of RAM the guest sees 7887 let current_ram_size = windows_guest.ram_size(); 7888 // This size seems to be reserved by the system and thus the 7889 // reported amount differs by this constant value. 7890 let reserved_ram_size = ram_size - current_ram_size; 7891 // Verify that there's not more than 4mb constant diff wasted 7892 // by the reserved ram. 7893 assert!(reserved_ram_size < 4 * 1024 * 1024); 7894 7895 let ram_size = 4 * 1024 * 1024 * 1024; 7896 // Hotplug some RAM 7897 resize_command(&api_socket, None, Some(ram_size), None, None); 7898 // Wait to make sure RAM has been added 7899 thread::sleep(std::time::Duration::new(10, 0)); 7900 // Check the guest sees the correct number 7901 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7902 7903 let ram_size = 3 * 1024 * 1024 * 1024; 7904 // Unplug some RAM. Note that hot-remove most likely won't work. 7905 resize_command(&api_socket, None, Some(ram_size), None, None); 7906 // Wait to make sure RAM has been added 7907 thread::sleep(std::time::Duration::new(10, 0)); 7908 // Reboot to let Windows catch up 7909 windows_guest.reboot(); 7910 // Wait to make sure guest completely rebooted 7911 thread::sleep(std::time::Duration::new(60, 0)); 7912 // Check the guest sees the correct number 7913 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 7914 7915 windows_guest.shutdown(); 7916 }); 7917 7918 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7919 let _ = child.kill(); 7920 let output = child.wait_with_output().unwrap(); 7921 7922 let _ = child_dnsmasq.kill(); 7923 let _ = child_dnsmasq.wait(); 7924 7925 handle_child_output(r, &output); 7926 } 7927 7928 #[test] 7929 #[cfg(not(feature = "mshv"))] 7930 fn test_windows_guest_netdev_hotplug() { 7931 let windows_guest = WindowsGuest::new(); 7932 7933 let mut ovmf_path = dirs::home_dir().unwrap(); 7934 ovmf_path.push("workloads"); 7935 ovmf_path.push(OVMF_NAME); 7936 7937 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7938 let api_socket = temp_api_path(&tmp_dir); 7939 7940 let mut child = GuestCommand::new(windows_guest.guest()) 7941 .args(["--api-socket", &api_socket]) 7942 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7943 .args(["--memory", "size=4G"]) 7944 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7945 .args(["--serial", "tty"]) 7946 .args(["--console", "off"]) 7947 .default_disks() 7948 .default_net() 7949 .capture_output() 7950 .spawn() 7951 .unwrap(); 7952 7953 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7954 7955 let r = std::panic::catch_unwind(|| { 7956 // Wait to make sure Windows boots up 7957 assert!(windows_guest.wait_for_boot()); 7958 7959 // Initially present network device 7960 let netdev_num = 1; 7961 assert_eq!(windows_guest.netdev_count(), netdev_num); 7962 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7963 7964 // Hotplug network device 7965 let (cmd_success, cmd_output) = remote_command_w_output( 7966 &api_socket, 7967 "add-net", 7968 Some(windows_guest.guest().default_net_string().as_str()), 7969 ); 7970 assert!(cmd_success); 7971 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 7972 thread::sleep(std::time::Duration::new(5, 0)); 7973 // Verify the device is on the system 7974 let netdev_num = 2; 7975 assert_eq!(windows_guest.netdev_count(), netdev_num); 7976 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7977 7978 // Remove network device 7979 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 7980 assert!(cmd_success); 7981 thread::sleep(std::time::Duration::new(5, 0)); 7982 // Verify the device has been removed 7983 let netdev_num = 1; 7984 assert_eq!(windows_guest.netdev_count(), netdev_num); 7985 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 7986 7987 windows_guest.shutdown(); 7988 }); 7989 7990 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7991 let _ = child.kill(); 7992 let output = child.wait_with_output().unwrap(); 7993 7994 let _ = child_dnsmasq.kill(); 7995 let _ = child_dnsmasq.wait(); 7996 7997 handle_child_output(r, &output); 7998 } 7999 8000 #[test] 8001 #[cfg(not(feature = "mshv"))] 8002 #[cfg(not(target_arch = "aarch64"))] 8003 fn test_windows_guest_disk_hotplug() { 8004 let windows_guest = WindowsGuest::new(); 8005 8006 let mut ovmf_path = dirs::home_dir().unwrap(); 8007 ovmf_path.push("workloads"); 8008 ovmf_path.push(OVMF_NAME); 8009 8010 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8011 let api_socket = temp_api_path(&tmp_dir); 8012 8013 let mut child = GuestCommand::new(windows_guest.guest()) 8014 .args(["--api-socket", &api_socket]) 8015 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8016 .args(["--memory", "size=4G"]) 8017 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8018 .args(["--serial", "tty"]) 8019 .args(["--console", "off"]) 8020 .default_disks() 8021 .default_net() 8022 .capture_output() 8023 .spawn() 8024 .unwrap(); 8025 8026 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8027 8028 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 8029 8030 let r = std::panic::catch_unwind(|| { 8031 // Wait to make sure Windows boots up 8032 assert!(windows_guest.wait_for_boot()); 8033 8034 // Initially present disk device 8035 let disk_num = 1; 8036 assert_eq!(windows_guest.disk_count(), disk_num); 8037 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8038 8039 // Hotplug disk device 8040 let (cmd_success, cmd_output) = remote_command_w_output( 8041 &api_socket, 8042 "add-disk", 8043 Some(format!("path={disk},readonly=off").as_str()), 8044 ); 8045 assert!(cmd_success); 8046 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 8047 thread::sleep(std::time::Duration::new(5, 0)); 8048 // Online disk device 8049 windows_guest.disks_set_rw(); 8050 windows_guest.disks_online(); 8051 // Verify the device is on the system 8052 let disk_num = 2; 8053 assert_eq!(windows_guest.disk_count(), disk_num); 8054 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8055 8056 let data = "hello"; 8057 let fname = "d:\\world"; 8058 windows_guest.disk_file_put(fname, data); 8059 8060 // Unmount disk device 8061 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 8062 assert!(cmd_success); 8063 thread::sleep(std::time::Duration::new(5, 0)); 8064 // Verify the device has been removed 8065 let disk_num = 1; 8066 assert_eq!(windows_guest.disk_count(), disk_num); 8067 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8068 8069 // Remount and check the file exists with the expected contents 8070 let (cmd_success, _cmd_output) = remote_command_w_output( 8071 &api_socket, 8072 "add-disk", 8073 Some(format!("path={disk},readonly=off").as_str()), 8074 ); 8075 assert!(cmd_success); 8076 thread::sleep(std::time::Duration::new(5, 0)); 8077 let out = windows_guest.disk_file_read(fname); 8078 assert_eq!(data, out.trim()); 8079 8080 // Intentionally no unmount, it'll happen at shutdown. 8081 8082 windows_guest.shutdown(); 8083 }); 8084 8085 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8086 let _ = child.kill(); 8087 let output = child.wait_with_output().unwrap(); 8088 8089 let _ = child_dnsmasq.kill(); 8090 let _ = child_dnsmasq.wait(); 8091 8092 handle_child_output(r, &output); 8093 } 8094 8095 #[test] 8096 #[cfg(not(feature = "mshv"))] 8097 #[cfg(not(target_arch = "aarch64"))] 8098 fn test_windows_guest_disk_hotplug_multi() { 8099 let windows_guest = WindowsGuest::new(); 8100 8101 let mut ovmf_path = dirs::home_dir().unwrap(); 8102 ovmf_path.push("workloads"); 8103 ovmf_path.push(OVMF_NAME); 8104 8105 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8106 let api_socket = temp_api_path(&tmp_dir); 8107 8108 let mut child = GuestCommand::new(windows_guest.guest()) 8109 .args(["--api-socket", &api_socket]) 8110 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8111 .args(["--memory", "size=2G"]) 8112 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8113 .args(["--serial", "tty"]) 8114 .args(["--console", "off"]) 8115 .default_disks() 8116 .default_net() 8117 .capture_output() 8118 .spawn() 8119 .unwrap(); 8120 8121 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8122 8123 // Predefined data to used at various test stages 8124 let disk_test_data: [[String; 4]; 2] = [ 8125 [ 8126 "_disk2".to_string(), 8127 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 8128 "d:\\world".to_string(), 8129 "hello".to_string(), 8130 ], 8131 [ 8132 "_disk3".to_string(), 8133 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 8134 "e:\\hello".to_string(), 8135 "world".to_string(), 8136 ], 8137 ]; 8138 8139 let r = std::panic::catch_unwind(|| { 8140 // Wait to make sure Windows boots up 8141 assert!(windows_guest.wait_for_boot()); 8142 8143 // Initially present disk device 8144 let disk_num = 1; 8145 assert_eq!(windows_guest.disk_count(), disk_num); 8146 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8147 8148 for it in &disk_test_data { 8149 let disk_id = it[0].as_str(); 8150 let disk = it[1].as_str(); 8151 // Hotplug disk device 8152 let (cmd_success, cmd_output) = remote_command_w_output( 8153 &api_socket, 8154 "add-disk", 8155 Some(format!("path={disk},readonly=off").as_str()), 8156 ); 8157 assert!(cmd_success); 8158 assert!(String::from_utf8_lossy(&cmd_output) 8159 .contains(format!("\"id\":\"{disk_id}\"").as_str())); 8160 thread::sleep(std::time::Duration::new(5, 0)); 8161 // Online disk devices 8162 windows_guest.disks_set_rw(); 8163 windows_guest.disks_online(); 8164 } 8165 // Verify the devices are on the system 8166 let disk_num = (disk_test_data.len() + 1) as u8; 8167 assert_eq!(windows_guest.disk_count(), disk_num); 8168 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8169 8170 // Put test data 8171 for it in &disk_test_data { 8172 let fname = it[2].as_str(); 8173 let data = it[3].as_str(); 8174 windows_guest.disk_file_put(fname, data); 8175 } 8176 8177 // Unmount disk devices 8178 for it in &disk_test_data { 8179 let disk_id = it[0].as_str(); 8180 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 8181 assert!(cmd_success); 8182 thread::sleep(std::time::Duration::new(5, 0)); 8183 } 8184 8185 // Verify the devices have been removed 8186 let disk_num = 1; 8187 assert_eq!(windows_guest.disk_count(), disk_num); 8188 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8189 8190 // Remount 8191 for it in &disk_test_data { 8192 let disk = it[1].as_str(); 8193 let (cmd_success, _cmd_output) = remote_command_w_output( 8194 &api_socket, 8195 "add-disk", 8196 Some(format!("path={disk},readonly=off").as_str()), 8197 ); 8198 assert!(cmd_success); 8199 thread::sleep(std::time::Duration::new(5, 0)); 8200 } 8201 8202 // Check the files exists with the expected contents 8203 for it in &disk_test_data { 8204 let fname = it[2].as_str(); 8205 let data = it[3].as_str(); 8206 let out = windows_guest.disk_file_read(fname); 8207 assert_eq!(data, out.trim()); 8208 } 8209 8210 // Intentionally no unmount, it'll happen at shutdown. 8211 8212 windows_guest.shutdown(); 8213 }); 8214 8215 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8216 let _ = child.kill(); 8217 let output = child.wait_with_output().unwrap(); 8218 8219 let _ = child_dnsmasq.kill(); 8220 let _ = child_dnsmasq.wait(); 8221 8222 handle_child_output(r, &output); 8223 } 8224 8225 #[test] 8226 #[cfg(not(feature = "mshv"))] 8227 #[cfg(not(target_arch = "aarch64"))] 8228 fn test_windows_guest_netdev_multi() { 8229 let windows_guest = WindowsGuest::new(); 8230 8231 let mut ovmf_path = dirs::home_dir().unwrap(); 8232 ovmf_path.push("workloads"); 8233 ovmf_path.push(OVMF_NAME); 8234 8235 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8236 let api_socket = temp_api_path(&tmp_dir); 8237 8238 let mut child = GuestCommand::new(windows_guest.guest()) 8239 .args(["--api-socket", &api_socket]) 8240 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8241 .args(["--memory", "size=4G"]) 8242 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8243 .args(["--serial", "tty"]) 8244 .args(["--console", "off"]) 8245 .default_disks() 8246 // The multi net dev config is borrowed from test_multiple_network_interfaces 8247 .args([ 8248 "--net", 8249 windows_guest.guest().default_net_string().as_str(), 8250 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 8251 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 8252 ]) 8253 .capture_output() 8254 .spawn() 8255 .unwrap(); 8256 8257 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8258 8259 let r = std::panic::catch_unwind(|| { 8260 // Wait to make sure Windows boots up 8261 assert!(windows_guest.wait_for_boot()); 8262 8263 let netdev_num = 3; 8264 assert_eq!(windows_guest.netdev_count(), netdev_num); 8265 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8266 8267 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 8268 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 8269 8270 windows_guest.shutdown(); 8271 }); 8272 8273 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8274 let _ = child.kill(); 8275 let output = child.wait_with_output().unwrap(); 8276 8277 let _ = child_dnsmasq.kill(); 8278 let _ = child_dnsmasq.wait(); 8279 8280 handle_child_output(r, &output); 8281 } 8282 } 8283 8284 #[cfg(target_arch = "x86_64")] 8285 mod sgx { 8286 use crate::*; 8287 8288 #[test] 8289 fn test_sgx() { 8290 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 8291 let jammy = UbuntuDiskConfig::new(jammy_image); 8292 let guest = Guest::new(Box::new(jammy)); 8293 8294 let mut child = GuestCommand::new(&guest) 8295 .args(["--cpus", "boot=1"]) 8296 .args(["--memory", "size=512M"]) 8297 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8298 .default_disks() 8299 .default_net() 8300 .args(["--sgx-epc", "id=epc0,size=64M"]) 8301 .capture_output() 8302 .spawn() 8303 .unwrap(); 8304 8305 let r = std::panic::catch_unwind(|| { 8306 guest.wait_vm_boot(None).unwrap(); 8307 8308 // Check if SGX is correctly detected in the guest. 8309 guest.check_sgx_support().unwrap(); 8310 8311 // Validate the SGX EPC section is 64MiB. 8312 assert_eq!( 8313 guest 8314 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 8315 .unwrap() 8316 .trim(), 8317 "0x0000000004000000" 8318 ); 8319 }); 8320 8321 let _ = child.kill(); 8322 let output = child.wait_with_output().unwrap(); 8323 8324 handle_child_output(r, &output); 8325 } 8326 } 8327 8328 #[cfg(target_arch = "x86_64")] 8329 mod vfio { 8330 use crate::*; 8331 8332 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 8333 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8334 let guest = Guest::new(Box::new(jammy)); 8335 let api_socket = temp_api_path(&guest.tmp_dir); 8336 8337 let mut child = GuestCommand::new(&guest) 8338 .args(["--cpus", "boot=4"]) 8339 .args([ 8340 "--memory", 8341 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(), 8342 ]) 8343 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8344 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8345 .args(["--api-socket", &api_socket]) 8346 .default_disks() 8347 .default_net() 8348 .capture_output() 8349 .spawn() 8350 .unwrap(); 8351 8352 let r = std::panic::catch_unwind(|| { 8353 guest.wait_vm_boot(None).unwrap(); 8354 8355 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8356 8357 guest.enable_memory_hotplug(); 8358 8359 // Add RAM to the VM 8360 let desired_ram = 6 << 30; 8361 resize_command(&api_socket, None, Some(desired_ram), None, None); 8362 thread::sleep(std::time::Duration::new(30, 0)); 8363 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8364 8365 // Check the VFIO device works when RAM is increased to 6GiB 8366 guest.check_nvidia_gpu(); 8367 }); 8368 8369 let _ = child.kill(); 8370 let output = child.wait_with_output().unwrap(); 8371 8372 handle_child_output(r, &output); 8373 } 8374 8375 #[test] 8376 fn test_nvidia_card_memory_hotplug_acpi() { 8377 test_nvidia_card_memory_hotplug("acpi") 8378 } 8379 8380 #[test] 8381 fn test_nvidia_card_memory_hotplug_virtio_mem() { 8382 test_nvidia_card_memory_hotplug("virtio-mem") 8383 } 8384 8385 #[test] 8386 fn test_nvidia_card_pci_hotplug() { 8387 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8388 let guest = Guest::new(Box::new(jammy)); 8389 let api_socket = temp_api_path(&guest.tmp_dir); 8390 8391 let mut child = GuestCommand::new(&guest) 8392 .args(["--cpus", "boot=4"]) 8393 .args(["--memory", "size=4G"]) 8394 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8395 .args(["--api-socket", &api_socket]) 8396 .default_disks() 8397 .default_net() 8398 .capture_output() 8399 .spawn() 8400 .unwrap(); 8401 8402 let r = std::panic::catch_unwind(|| { 8403 guest.wait_vm_boot(None).unwrap(); 8404 8405 // Hotplug the card to the VM 8406 let (cmd_success, cmd_output) = remote_command_w_output( 8407 &api_socket, 8408 "add-device", 8409 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 8410 ); 8411 assert!(cmd_success); 8412 assert!(String::from_utf8_lossy(&cmd_output) 8413 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 8414 8415 thread::sleep(std::time::Duration::new(10, 0)); 8416 8417 // Check the VFIO device works after hotplug 8418 guest.check_nvidia_gpu(); 8419 }); 8420 8421 let _ = child.kill(); 8422 let output = child.wait_with_output().unwrap(); 8423 8424 handle_child_output(r, &output); 8425 } 8426 8427 #[test] 8428 fn test_nvidia_card_reboot() { 8429 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8430 let guest = Guest::new(Box::new(jammy)); 8431 let api_socket = temp_api_path(&guest.tmp_dir); 8432 8433 let mut child = GuestCommand::new(&guest) 8434 .args(["--cpus", "boot=4"]) 8435 .args(["--memory", "size=4G"]) 8436 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8437 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8438 .args(["--api-socket", &api_socket]) 8439 .default_disks() 8440 .default_net() 8441 .capture_output() 8442 .spawn() 8443 .unwrap(); 8444 8445 let r = std::panic::catch_unwind(|| { 8446 guest.wait_vm_boot(None).unwrap(); 8447 8448 // Check the VFIO device works after boot 8449 guest.check_nvidia_gpu(); 8450 8451 guest.reboot_linux(0, None); 8452 8453 // Check the VFIO device works after reboot 8454 guest.check_nvidia_gpu(); 8455 }); 8456 8457 let _ = child.kill(); 8458 let output = child.wait_with_output().unwrap(); 8459 8460 handle_child_output(r, &output); 8461 } 8462 } 8463 8464 mod live_migration { 8465 use crate::*; 8466 8467 fn start_live_migration( 8468 migration_socket: &str, 8469 src_api_socket: &str, 8470 dest_api_socket: &str, 8471 local: bool, 8472 ) -> bool { 8473 // Start to receive migration from the destintion VM 8474 let mut receive_migration = Command::new(clh_command("ch-remote")) 8475 .args([ 8476 &format!("--api-socket={dest_api_socket}"), 8477 "receive-migration", 8478 &format! {"unix:{migration_socket}"}, 8479 ]) 8480 .stderr(Stdio::piped()) 8481 .stdout(Stdio::piped()) 8482 .spawn() 8483 .unwrap(); 8484 // Give it '1s' to make sure the 'migration_socket' file is properly created 8485 thread::sleep(std::time::Duration::new(1, 0)); 8486 // Start to send migration from the source VM 8487 8488 let mut args = [ 8489 format!("--api-socket={}", &src_api_socket), 8490 "send-migration".to_string(), 8491 format! {"unix:{migration_socket}"}, 8492 ] 8493 .to_vec(); 8494 8495 if local { 8496 args.insert(2, "--local".to_string()); 8497 } 8498 8499 let mut send_migration = Command::new(clh_command("ch-remote")) 8500 .args(&args) 8501 .stderr(Stdio::piped()) 8502 .stdout(Stdio::piped()) 8503 .spawn() 8504 .unwrap(); 8505 8506 // The 'send-migration' command should be executed successfully within the given timeout 8507 let send_success = if let Some(status) = send_migration 8508 .wait_timeout(std::time::Duration::from_secs(30)) 8509 .unwrap() 8510 { 8511 status.success() 8512 } else { 8513 false 8514 }; 8515 8516 if !send_success { 8517 let _ = send_migration.kill(); 8518 let output = send_migration.wait_with_output().unwrap(); 8519 eprintln!( 8520 "\n\n==== Start 'send_migration' output ==== \ 8521 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8522 \n\n==== End 'send_migration' output ====\n\n", 8523 String::from_utf8_lossy(&output.stdout), 8524 String::from_utf8_lossy(&output.stderr) 8525 ); 8526 } 8527 8528 // The 'receive-migration' command should be executed successfully within the given timeout 8529 let receive_success = if let Some(status) = receive_migration 8530 .wait_timeout(std::time::Duration::from_secs(30)) 8531 .unwrap() 8532 { 8533 status.success() 8534 } else { 8535 false 8536 }; 8537 8538 if !receive_success { 8539 let _ = receive_migration.kill(); 8540 let output = receive_migration.wait_with_output().unwrap(); 8541 eprintln!( 8542 "\n\n==== Start 'receive_migration' output ==== \ 8543 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8544 \n\n==== End 'receive_migration' output ====\n\n", 8545 String::from_utf8_lossy(&output.stdout), 8546 String::from_utf8_lossy(&output.stderr) 8547 ); 8548 } 8549 8550 send_success && receive_success 8551 } 8552 8553 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 8554 let mut src_vm = src_vm; 8555 let mut dest_vm = dest_vm; 8556 8557 let _ = src_vm.kill(); 8558 let src_output = src_vm.wait_with_output().unwrap(); 8559 eprintln!( 8560 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8561 String::from_utf8_lossy(&src_output.stdout) 8562 ); 8563 eprintln!( 8564 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8565 String::from_utf8_lossy(&src_output.stderr) 8566 ); 8567 let _ = dest_vm.kill(); 8568 let dest_output = dest_vm.wait_with_output().unwrap(); 8569 eprintln!( 8570 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8571 String::from_utf8_lossy(&dest_output.stdout) 8572 ); 8573 eprintln!( 8574 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8575 String::from_utf8_lossy(&dest_output.stderr) 8576 ); 8577 8578 if let Some(ovs_vm) = ovs_vm { 8579 let mut ovs_vm = ovs_vm; 8580 let _ = ovs_vm.kill(); 8581 let ovs_output = ovs_vm.wait_with_output().unwrap(); 8582 eprintln!( 8583 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 8584 String::from_utf8_lossy(&ovs_output.stdout) 8585 ); 8586 eprintln!( 8587 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 8588 String::from_utf8_lossy(&ovs_output.stderr) 8589 ); 8590 8591 cleanup_ovs_dpdk(); 8592 } 8593 8594 panic!("Test failed: {message}") 8595 } 8596 8597 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 8598 // same host. It ensures the following behaviors: 8599 // 1. The source VM is up and functional (including various virtio-devices are working properly); 8600 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 8601 // 3. The source VM terminated gracefully after live migration; 8602 // 4. The destination VM is functional (including various virtio-devices are working properly) after 8603 // live migration; 8604 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 8605 fn _test_live_migration(upgrade_test: bool, local: bool) { 8606 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8607 let guest = Guest::new(Box::new(focal)); 8608 let kernel_path = direct_kernel_boot_path(); 8609 let console_text = String::from("On a branch floating down river a cricket, singing."); 8610 let net_id = "net123"; 8611 let net_params = format!( 8612 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8613 net_id, guest.network.guest_mac, guest.network.host_ip 8614 ); 8615 8616 let memory_param: &[&str] = if local { 8617 &["--memory", "size=4G,shared=on"] 8618 } else { 8619 &["--memory", "size=4G"] 8620 }; 8621 8622 let boot_vcpus = 2; 8623 let max_vcpus = 4; 8624 8625 let pmem_temp_file = TempFile::new().unwrap(); 8626 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8627 std::process::Command::new("mkfs.ext4") 8628 .arg(pmem_temp_file.as_path()) 8629 .output() 8630 .expect("Expect creating disk image to succeed"); 8631 let pmem_path = String::from("/dev/pmem0"); 8632 8633 // Start the source VM 8634 let src_vm_path = if !upgrade_test { 8635 clh_command("cloud-hypervisor") 8636 } else { 8637 cloud_hypervisor_release_path() 8638 }; 8639 let src_api_socket = temp_api_path(&guest.tmp_dir); 8640 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8641 src_vm_cmd 8642 .args([ 8643 "--cpus", 8644 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8645 ]) 8646 .args(memory_param) 8647 .args(["--kernel", kernel_path.to_str().unwrap()]) 8648 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8649 .default_disks() 8650 .args(["--net", net_params.as_str()]) 8651 .args(["--api-socket", &src_api_socket]) 8652 .args([ 8653 "--pmem", 8654 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8655 ]); 8656 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8657 8658 // Start the destination VM 8659 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8660 dest_api_socket.push_str(".dest"); 8661 let mut dest_child = GuestCommand::new(&guest) 8662 .args(["--api-socket", &dest_api_socket]) 8663 .capture_output() 8664 .spawn() 8665 .unwrap(); 8666 8667 let r = std::panic::catch_unwind(|| { 8668 guest.wait_vm_boot(None).unwrap(); 8669 8670 // Make sure the source VM is functaionl 8671 // Check the number of vCPUs 8672 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8673 8674 // Check the guest RAM 8675 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8676 8677 // Check the guest virtio-devices, e.g. block, rng, console, and net 8678 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8679 8680 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8681 // to make sure that removing and adding back the virtio-net device does 8682 // not break the live-migration support for virtio-pci. 8683 #[cfg(target_arch = "x86_64")] 8684 { 8685 assert!(remote_command( 8686 &src_api_socket, 8687 "remove-device", 8688 Some(net_id), 8689 )); 8690 thread::sleep(std::time::Duration::new(10, 0)); 8691 8692 // Plug the virtio-net device again 8693 assert!(remote_command( 8694 &src_api_socket, 8695 "add-net", 8696 Some(net_params.as_str()), 8697 )); 8698 thread::sleep(std::time::Duration::new(10, 0)); 8699 } 8700 8701 // Start the live-migration 8702 let migration_socket = String::from( 8703 guest 8704 .tmp_dir 8705 .as_path() 8706 .join("live-migration.sock") 8707 .to_str() 8708 .unwrap(), 8709 ); 8710 8711 assert!( 8712 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8713 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8714 ); 8715 }); 8716 8717 // Check and report any errors occurred during the live-migration 8718 if r.is_err() { 8719 print_and_panic( 8720 src_child, 8721 dest_child, 8722 None, 8723 "Error occurred during live-migration", 8724 ); 8725 } 8726 8727 // Check the source vm has been terminated successful (give it '3s' to settle) 8728 thread::sleep(std::time::Duration::new(3, 0)); 8729 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8730 print_and_panic( 8731 src_child, 8732 dest_child, 8733 None, 8734 "source VM was not terminated successfully.", 8735 ); 8736 }; 8737 8738 // Post live-migration check to make sure the destination VM is funcational 8739 let r = std::panic::catch_unwind(|| { 8740 // Perform same checks to validate VM has been properly migrated 8741 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8742 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8743 8744 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8745 }); 8746 8747 // Clean-up the destination VM and make sure it terminated correctly 8748 let _ = dest_child.kill(); 8749 let dest_output = dest_child.wait_with_output().unwrap(); 8750 handle_child_output(r, &dest_output); 8751 8752 // Check the destination VM has the expected 'concole_text' from its output 8753 let r = std::panic::catch_unwind(|| { 8754 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8755 }); 8756 handle_child_output(r, &dest_output); 8757 } 8758 8759 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 8760 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8761 let guest = Guest::new(Box::new(focal)); 8762 let kernel_path = direct_kernel_boot_path(); 8763 let console_text = String::from("On a branch floating down river a cricket, singing."); 8764 let net_id = "net123"; 8765 let net_params = format!( 8766 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8767 net_id, guest.network.guest_mac, guest.network.host_ip 8768 ); 8769 8770 let memory_param: &[&str] = if local { 8771 &[ 8772 "--memory", 8773 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 8774 "--balloon", 8775 "size=0", 8776 ] 8777 } else { 8778 &[ 8779 "--memory", 8780 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 8781 "--balloon", 8782 "size=0", 8783 ] 8784 }; 8785 8786 let boot_vcpus = 2; 8787 let max_vcpus = 4; 8788 8789 let pmem_temp_file = TempFile::new().unwrap(); 8790 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8791 std::process::Command::new("mkfs.ext4") 8792 .arg(pmem_temp_file.as_path()) 8793 .output() 8794 .expect("Expect creating disk image to succeed"); 8795 let pmem_path = String::from("/dev/pmem0"); 8796 8797 // Start the source VM 8798 let src_vm_path = if !upgrade_test { 8799 clh_command("cloud-hypervisor") 8800 } else { 8801 cloud_hypervisor_release_path() 8802 }; 8803 let src_api_socket = temp_api_path(&guest.tmp_dir); 8804 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8805 src_vm_cmd 8806 .args([ 8807 "--cpus", 8808 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8809 ]) 8810 .args(memory_param) 8811 .args(["--kernel", kernel_path.to_str().unwrap()]) 8812 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8813 .default_disks() 8814 .args(["--net", net_params.as_str()]) 8815 .args(["--api-socket", &src_api_socket]) 8816 .args([ 8817 "--pmem", 8818 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8819 ]); 8820 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8821 8822 // Start the destination VM 8823 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8824 dest_api_socket.push_str(".dest"); 8825 let mut dest_child = GuestCommand::new(&guest) 8826 .args(["--api-socket", &dest_api_socket]) 8827 .capture_output() 8828 .spawn() 8829 .unwrap(); 8830 8831 let r = std::panic::catch_unwind(|| { 8832 guest.wait_vm_boot(None).unwrap(); 8833 8834 // Make sure the source VM is functaionl 8835 // Check the number of vCPUs 8836 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8837 8838 // Check the guest RAM 8839 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8840 // Increase the guest RAM 8841 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 8842 thread::sleep(std::time::Duration::new(5, 0)); 8843 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8844 // Use balloon to remove RAM from the VM 8845 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 8846 thread::sleep(std::time::Duration::new(5, 0)); 8847 let total_memory = guest.get_total_memory().unwrap_or_default(); 8848 assert!(total_memory > 4_800_000); 8849 assert!(total_memory < 5_760_000); 8850 8851 // Check the guest virtio-devices, e.g. block, rng, console, and net 8852 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8853 8854 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8855 // to make sure that removing and adding back the virtio-net device does 8856 // not break the live-migration support for virtio-pci. 8857 #[cfg(target_arch = "x86_64")] 8858 { 8859 assert!(remote_command( 8860 &src_api_socket, 8861 "remove-device", 8862 Some(net_id), 8863 )); 8864 thread::sleep(std::time::Duration::new(10, 0)); 8865 8866 // Plug the virtio-net device again 8867 assert!(remote_command( 8868 &src_api_socket, 8869 "add-net", 8870 Some(net_params.as_str()), 8871 )); 8872 thread::sleep(std::time::Duration::new(10, 0)); 8873 } 8874 8875 // Start the live-migration 8876 let migration_socket = String::from( 8877 guest 8878 .tmp_dir 8879 .as_path() 8880 .join("live-migration.sock") 8881 .to_str() 8882 .unwrap(), 8883 ); 8884 8885 assert!( 8886 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8887 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8888 ); 8889 }); 8890 8891 // Check and report any errors occurred during the live-migration 8892 if r.is_err() { 8893 print_and_panic( 8894 src_child, 8895 dest_child, 8896 None, 8897 "Error occurred during live-migration", 8898 ); 8899 } 8900 8901 // Check the source vm has been terminated successful (give it '3s' to settle) 8902 thread::sleep(std::time::Duration::new(3, 0)); 8903 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8904 print_and_panic( 8905 src_child, 8906 dest_child, 8907 None, 8908 "source VM was not terminated successfully.", 8909 ); 8910 }; 8911 8912 // Post live-migration check to make sure the destination VM is funcational 8913 let r = std::panic::catch_unwind(|| { 8914 // Perform same checks to validate VM has been properly migrated 8915 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8916 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8917 8918 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8919 8920 // Perform checks on guest RAM using balloon 8921 let total_memory = guest.get_total_memory().unwrap_or_default(); 8922 assert!(total_memory > 4_800_000); 8923 assert!(total_memory < 5_760_000); 8924 // Deflate balloon to restore entire RAM to the VM 8925 resize_command(&dest_api_socket, None, None, Some(0), None); 8926 thread::sleep(std::time::Duration::new(5, 0)); 8927 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8928 // Decrease guest RAM with virtio-mem 8929 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 8930 thread::sleep(std::time::Duration::new(5, 0)); 8931 let total_memory = guest.get_total_memory().unwrap_or_default(); 8932 assert!(total_memory > 4_800_000); 8933 assert!(total_memory < 5_760_000); 8934 }); 8935 8936 // Clean-up the destination VM and make sure it terminated correctly 8937 let _ = dest_child.kill(); 8938 let dest_output = dest_child.wait_with_output().unwrap(); 8939 handle_child_output(r, &dest_output); 8940 8941 // Check the destination VM has the expected 'concole_text' from its output 8942 let r = std::panic::catch_unwind(|| { 8943 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8944 }); 8945 handle_child_output(r, &dest_output); 8946 } 8947 8948 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 8949 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8950 let guest = Guest::new(Box::new(focal)); 8951 let kernel_path = direct_kernel_boot_path(); 8952 let console_text = String::from("On a branch floating down river a cricket, singing."); 8953 let net_id = "net123"; 8954 let net_params = format!( 8955 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8956 net_id, guest.network.guest_mac, guest.network.host_ip 8957 ); 8958 8959 let memory_param: &[&str] = if local { 8960 &[ 8961 "--memory", 8962 "size=0,hotplug_method=virtio-mem,shared=on", 8963 "--memory-zone", 8964 "id=mem0,size=1G,hotplug_size=4G,shared=on", 8965 "id=mem1,size=1G,hotplug_size=4G,shared=on", 8966 "id=mem2,size=2G,hotplug_size=4G,shared=on", 8967 "--numa", 8968 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 8969 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 8970 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 8971 ] 8972 } else { 8973 &[ 8974 "--memory", 8975 "size=0,hotplug_method=virtio-mem", 8976 "--memory-zone", 8977 "id=mem0,size=1G,hotplug_size=4G", 8978 "id=mem1,size=1G,hotplug_size=4G", 8979 "id=mem2,size=2G,hotplug_size=4G", 8980 "--numa", 8981 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 8982 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 8983 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 8984 ] 8985 }; 8986 8987 let boot_vcpus = 6; 8988 let max_vcpus = 12; 8989 8990 let pmem_temp_file = TempFile::new().unwrap(); 8991 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8992 std::process::Command::new("mkfs.ext4") 8993 .arg(pmem_temp_file.as_path()) 8994 .output() 8995 .expect("Expect creating disk image to succeed"); 8996 let pmem_path = String::from("/dev/pmem0"); 8997 8998 // Start the source VM 8999 let src_vm_path = if !upgrade_test { 9000 clh_command("cloud-hypervisor") 9001 } else { 9002 cloud_hypervisor_release_path() 9003 }; 9004 let src_api_socket = temp_api_path(&guest.tmp_dir); 9005 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9006 src_vm_cmd 9007 .args([ 9008 "--cpus", 9009 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9010 ]) 9011 .args(memory_param) 9012 .args(["--kernel", kernel_path.to_str().unwrap()]) 9013 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9014 .default_disks() 9015 .args(["--net", net_params.as_str()]) 9016 .args(["--api-socket", &src_api_socket]) 9017 .args([ 9018 "--pmem", 9019 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9020 ]); 9021 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9022 9023 // Start the destination VM 9024 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9025 dest_api_socket.push_str(".dest"); 9026 let mut dest_child = GuestCommand::new(&guest) 9027 .args(["--api-socket", &dest_api_socket]) 9028 .capture_output() 9029 .spawn() 9030 .unwrap(); 9031 9032 let r = std::panic::catch_unwind(|| { 9033 guest.wait_vm_boot(None).unwrap(); 9034 9035 // Make sure the source VM is functaionl 9036 // Check the number of vCPUs 9037 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9038 9039 // Check the guest RAM 9040 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 9041 9042 // Check the guest virtio-devices, e.g. block, rng, console, and net 9043 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9044 9045 // Check the NUMA parameters are applied correctly and resize 9046 // each zone to test the case where we migrate a VM with the 9047 // virtio-mem regions being used. 9048 { 9049 guest.check_numa_common( 9050 Some(&[960_000, 960_000, 1_920_000]), 9051 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9052 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9053 ); 9054 9055 // AArch64 currently does not support hotplug, and therefore we only 9056 // test hotplug-related function on x86_64 here. 9057 #[cfg(target_arch = "x86_64")] 9058 { 9059 guest.enable_memory_hotplug(); 9060 9061 // Resize every memory zone and check each associated NUMA node 9062 // has been assigned the right amount of memory. 9063 resize_zone_command(&src_api_socket, "mem0", "2G"); 9064 resize_zone_command(&src_api_socket, "mem1", "2G"); 9065 resize_zone_command(&src_api_socket, "mem2", "3G"); 9066 thread::sleep(std::time::Duration::new(5, 0)); 9067 9068 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 9069 } 9070 } 9071 9072 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9073 // to make sure that removing and adding back the virtio-net device does 9074 // not break the live-migration support for virtio-pci. 9075 #[cfg(target_arch = "x86_64")] 9076 { 9077 assert!(remote_command( 9078 &src_api_socket, 9079 "remove-device", 9080 Some(net_id), 9081 )); 9082 thread::sleep(std::time::Duration::new(10, 0)); 9083 9084 // Plug the virtio-net device again 9085 assert!(remote_command( 9086 &src_api_socket, 9087 "add-net", 9088 Some(net_params.as_str()), 9089 )); 9090 thread::sleep(std::time::Duration::new(10, 0)); 9091 } 9092 9093 // Start the live-migration 9094 let migration_socket = String::from( 9095 guest 9096 .tmp_dir 9097 .as_path() 9098 .join("live-migration.sock") 9099 .to_str() 9100 .unwrap(), 9101 ); 9102 9103 assert!( 9104 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9105 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9106 ); 9107 }); 9108 9109 // Check and report any errors occurred during the live-migration 9110 if r.is_err() { 9111 print_and_panic( 9112 src_child, 9113 dest_child, 9114 None, 9115 "Error occurred during live-migration", 9116 ); 9117 } 9118 9119 // Check the source vm has been terminated successful (give it '3s' to settle) 9120 thread::sleep(std::time::Duration::new(3, 0)); 9121 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9122 print_and_panic( 9123 src_child, 9124 dest_child, 9125 None, 9126 "source VM was not terminated successfully.", 9127 ); 9128 }; 9129 9130 // Post live-migration check to make sure the destination VM is funcational 9131 let r = std::panic::catch_unwind(|| { 9132 // Perform same checks to validate VM has been properly migrated 9133 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9134 #[cfg(target_arch = "x86_64")] 9135 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 9136 #[cfg(target_arch = "aarch64")] 9137 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9138 9139 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9140 9141 // Perform NUMA related checks 9142 { 9143 #[cfg(target_arch = "aarch64")] 9144 { 9145 guest.check_numa_common( 9146 Some(&[960_000, 960_000, 1_920_000]), 9147 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9148 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9149 ); 9150 } 9151 9152 // AArch64 currently does not support hotplug, and therefore we only 9153 // test hotplug-related function on x86_64 here. 9154 #[cfg(target_arch = "x86_64")] 9155 { 9156 guest.check_numa_common( 9157 Some(&[1_920_000, 1_920_000, 2_880_000]), 9158 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9159 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9160 ); 9161 9162 guest.enable_memory_hotplug(); 9163 9164 // Resize every memory zone and check each associated NUMA node 9165 // has been assigned the right amount of memory. 9166 resize_zone_command(&dest_api_socket, "mem0", "4G"); 9167 resize_zone_command(&dest_api_socket, "mem1", "4G"); 9168 resize_zone_command(&dest_api_socket, "mem2", "4G"); 9169 // Resize to the maximum amount of CPUs and check each NUMA 9170 // node has been assigned the right CPUs set. 9171 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 9172 thread::sleep(std::time::Duration::new(5, 0)); 9173 9174 guest.check_numa_common( 9175 Some(&[3_840_000, 3_840_000, 3_840_000]), 9176 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 9177 None, 9178 ); 9179 } 9180 } 9181 }); 9182 9183 // Clean-up the destination VM and make sure it terminated correctly 9184 let _ = dest_child.kill(); 9185 let dest_output = dest_child.wait_with_output().unwrap(); 9186 handle_child_output(r, &dest_output); 9187 9188 // Check the destination VM has the expected 'concole_text' from its output 9189 let r = std::panic::catch_unwind(|| { 9190 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9191 }); 9192 handle_child_output(r, &dest_output); 9193 } 9194 9195 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 9196 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9197 let guest = Guest::new(Box::new(focal)); 9198 let kernel_path = direct_kernel_boot_path(); 9199 let console_text = String::from("On a branch floating down river a cricket, singing."); 9200 let net_id = "net123"; 9201 let net_params = format!( 9202 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9203 net_id, guest.network.guest_mac, guest.network.host_ip 9204 ); 9205 9206 let memory_param: &[&str] = if local { 9207 &["--memory", "size=4G,shared=on"] 9208 } else { 9209 &["--memory", "size=4G"] 9210 }; 9211 9212 let boot_vcpus = 2; 9213 let max_vcpus = 4; 9214 9215 let pmem_temp_file = TempFile::new().unwrap(); 9216 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9217 std::process::Command::new("mkfs.ext4") 9218 .arg(pmem_temp_file.as_path()) 9219 .output() 9220 .expect("Expect creating disk image to succeed"); 9221 let pmem_path = String::from("/dev/pmem0"); 9222 9223 // Start the source VM 9224 let src_vm_path = if !upgrade_test { 9225 clh_command("cloud-hypervisor") 9226 } else { 9227 cloud_hypervisor_release_path() 9228 }; 9229 let src_api_socket = temp_api_path(&guest.tmp_dir); 9230 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9231 src_vm_cmd 9232 .args([ 9233 "--cpus", 9234 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9235 ]) 9236 .args(memory_param) 9237 .args(["--kernel", kernel_path.to_str().unwrap()]) 9238 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9239 .default_disks() 9240 .args(["--net", net_params.as_str()]) 9241 .args(["--api-socket", &src_api_socket]) 9242 .args([ 9243 "--pmem", 9244 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9245 ]) 9246 .args(["--watchdog"]); 9247 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9248 9249 // Start the destination VM 9250 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9251 dest_api_socket.push_str(".dest"); 9252 let mut dest_child = GuestCommand::new(&guest) 9253 .args(["--api-socket", &dest_api_socket]) 9254 .capture_output() 9255 .spawn() 9256 .unwrap(); 9257 9258 let r = std::panic::catch_unwind(|| { 9259 guest.wait_vm_boot(None).unwrap(); 9260 9261 // Make sure the source VM is functaionl 9262 // Check the number of vCPUs 9263 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9264 // Check the guest RAM 9265 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9266 // Check the guest virtio-devices, e.g. block, rng, console, and net 9267 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9268 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9269 // to make sure that removing and adding back the virtio-net device does 9270 // not break the live-migration support for virtio-pci. 9271 #[cfg(target_arch = "x86_64")] 9272 { 9273 assert!(remote_command( 9274 &src_api_socket, 9275 "remove-device", 9276 Some(net_id), 9277 )); 9278 thread::sleep(std::time::Duration::new(10, 0)); 9279 9280 // Plug the virtio-net device again 9281 assert!(remote_command( 9282 &src_api_socket, 9283 "add-net", 9284 Some(net_params.as_str()), 9285 )); 9286 thread::sleep(std::time::Duration::new(10, 0)); 9287 } 9288 9289 // Enable watchdog and ensure its functional 9290 let mut expected_reboot_count = 1; 9291 // Enable the watchdog with a 15s timeout 9292 enable_guest_watchdog(&guest, 15); 9293 // Reboot and check that systemd has activated the watchdog 9294 guest.ssh_command("sudo reboot").unwrap(); 9295 guest.wait_vm_boot(None).unwrap(); 9296 expected_reboot_count += 1; 9297 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9298 assert_eq!( 9299 guest 9300 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 9301 .unwrap() 9302 .trim() 9303 .parse::<u32>() 9304 .unwrap_or_default(), 9305 2 9306 ); 9307 // Allow some normal time to elapse to check we don't get spurious reboots 9308 thread::sleep(std::time::Duration::new(40, 0)); 9309 // Check no reboot 9310 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9311 9312 // Start the live-migration 9313 let migration_socket = String::from( 9314 guest 9315 .tmp_dir 9316 .as_path() 9317 .join("live-migration.sock") 9318 .to_str() 9319 .unwrap(), 9320 ); 9321 9322 assert!( 9323 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9324 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9325 ); 9326 }); 9327 9328 // Check and report any errors occurred during the live-migration 9329 if r.is_err() { 9330 print_and_panic( 9331 src_child, 9332 dest_child, 9333 None, 9334 "Error occurred during live-migration", 9335 ); 9336 } 9337 9338 // Check the source vm has been terminated successful (give it '3s' to settle) 9339 thread::sleep(std::time::Duration::new(3, 0)); 9340 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9341 print_and_panic( 9342 src_child, 9343 dest_child, 9344 None, 9345 "source VM was not terminated successfully.", 9346 ); 9347 }; 9348 9349 // Post live-migration check to make sure the destination VM is funcational 9350 let r = std::panic::catch_unwind(|| { 9351 // Perform same checks to validate VM has been properly migrated 9352 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9353 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9354 9355 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9356 9357 // Perform checks on watchdog 9358 let mut expected_reboot_count = 2; 9359 9360 // Allow some normal time to elapse to check we don't get spurious reboots 9361 thread::sleep(std::time::Duration::new(40, 0)); 9362 // Check no reboot 9363 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9364 9365 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 9366 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 9367 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 9368 guest.wait_vm_boot(Some(50)).unwrap(); 9369 // Check a reboot is triggered by the watchdog 9370 expected_reboot_count += 1; 9371 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9372 9373 #[cfg(target_arch = "x86_64")] 9374 { 9375 // Now pause the VM and remain offline for 30s 9376 assert!(remote_command(&dest_api_socket, "pause", None)); 9377 thread::sleep(std::time::Duration::new(30, 0)); 9378 assert!(remote_command(&dest_api_socket, "resume", None)); 9379 9380 // Check no reboot 9381 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9382 } 9383 }); 9384 9385 // Clean-up the destination VM and make sure it terminated correctly 9386 let _ = dest_child.kill(); 9387 let dest_output = dest_child.wait_with_output().unwrap(); 9388 handle_child_output(r, &dest_output); 9389 9390 // Check the destination VM has the expected 'concole_text' from its output 9391 let r = std::panic::catch_unwind(|| { 9392 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9393 }); 9394 handle_child_output(r, &dest_output); 9395 } 9396 9397 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 9398 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9399 let ovs_guest = Guest::new(Box::new(ovs_focal)); 9400 9401 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9402 let migration_guest = Guest::new(Box::new(migration_focal)); 9403 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 9404 9405 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 9406 let (mut ovs_child, mut src_child) = 9407 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 9408 9409 // Start the destination VM 9410 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 9411 dest_api_socket.push_str(".dest"); 9412 let mut dest_child = GuestCommand::new(&migration_guest) 9413 .args(["--api-socket", &dest_api_socket]) 9414 .capture_output() 9415 .spawn() 9416 .unwrap(); 9417 9418 let r = std::panic::catch_unwind(|| { 9419 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 9420 thread::sleep(std::time::Duration::new(1, 0)); 9421 9422 // Start the live-migration 9423 let migration_socket = String::from( 9424 migration_guest 9425 .tmp_dir 9426 .as_path() 9427 .join("live-migration.sock") 9428 .to_str() 9429 .unwrap(), 9430 ); 9431 9432 assert!( 9433 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9434 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9435 ); 9436 }); 9437 9438 // Check and report any errors occurred during the live-migration 9439 if r.is_err() { 9440 print_and_panic( 9441 src_child, 9442 dest_child, 9443 Some(ovs_child), 9444 "Error occurred during live-migration", 9445 ); 9446 } 9447 9448 // Check the source vm has been terminated successful (give it '3s' to settle) 9449 thread::sleep(std::time::Duration::new(3, 0)); 9450 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9451 print_and_panic( 9452 src_child, 9453 dest_child, 9454 Some(ovs_child), 9455 "source VM was not terminated successfully.", 9456 ); 9457 }; 9458 9459 // Post live-migration check to make sure the destination VM is funcational 9460 let r = std::panic::catch_unwind(|| { 9461 // Perform same checks to validate VM has been properly migrated 9462 // Spawn a new netcat listener in the OVS VM 9463 let guest_ip = ovs_guest.network.guest_ip.clone(); 9464 thread::spawn(move || { 9465 ssh_command_ip( 9466 "nc -l 12345", 9467 &guest_ip, 9468 DEFAULT_SSH_RETRIES, 9469 DEFAULT_SSH_TIMEOUT, 9470 ) 9471 .unwrap(); 9472 }); 9473 9474 // Wait for the server to be listening 9475 thread::sleep(std::time::Duration::new(5, 0)); 9476 9477 // And check the connection is still functional after live-migration 9478 migration_guest 9479 .ssh_command("nc -vz 172.100.0.1 12345") 9480 .unwrap(); 9481 }); 9482 9483 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 9484 let _ = dest_child.kill(); 9485 let _ = ovs_child.kill(); 9486 let dest_output = dest_child.wait_with_output().unwrap(); 9487 let ovs_output = ovs_child.wait_with_output().unwrap(); 9488 9489 cleanup_ovs_dpdk(); 9490 9491 handle_child_output(r, &dest_output); 9492 handle_child_output(Ok(()), &ovs_output); 9493 } 9494 9495 mod live_migration_parallel { 9496 use super::*; 9497 #[test] 9498 fn test_live_migration_basic() { 9499 _test_live_migration(false, false) 9500 } 9501 9502 #[test] 9503 fn test_live_migration_local() { 9504 _test_live_migration(false, true) 9505 } 9506 9507 #[test] 9508 #[cfg(not(feature = "mshv"))] 9509 fn test_live_migration_numa() { 9510 _test_live_migration_numa(false, false) 9511 } 9512 9513 #[test] 9514 #[cfg(not(feature = "mshv"))] 9515 fn test_live_migration_numa_local() { 9516 _test_live_migration_numa(false, true) 9517 } 9518 9519 #[test] 9520 fn test_live_migration_watchdog() { 9521 _test_live_migration_watchdog(false, false) 9522 } 9523 9524 #[test] 9525 fn test_live_migration_watchdog_local() { 9526 _test_live_migration_watchdog(false, true) 9527 } 9528 9529 #[test] 9530 fn test_live_migration_balloon() { 9531 _test_live_migration_balloon(false, false) 9532 } 9533 9534 #[test] 9535 fn test_live_migration_balloon_local() { 9536 _test_live_migration_balloon(false, true) 9537 } 9538 9539 #[test] 9540 fn test_live_upgrade_basic() { 9541 _test_live_migration(true, false) 9542 } 9543 9544 #[test] 9545 fn test_live_upgrade_local() { 9546 _test_live_migration(true, true) 9547 } 9548 9549 #[test] 9550 #[cfg(not(feature = "mshv"))] 9551 fn test_live_upgrade_numa() { 9552 _test_live_migration_numa(true, false) 9553 } 9554 9555 #[test] 9556 #[cfg(not(feature = "mshv"))] 9557 fn test_live_upgrade_numa_local() { 9558 _test_live_migration_numa(true, true) 9559 } 9560 9561 #[test] 9562 fn test_live_upgrade_watchdog() { 9563 _test_live_migration_watchdog(true, false) 9564 } 9565 9566 #[test] 9567 fn test_live_upgrade_watchdog_local() { 9568 _test_live_migration_watchdog(true, true) 9569 } 9570 9571 #[test] 9572 fn test_live_upgrade_balloon() { 9573 _test_live_migration_balloon(true, false) 9574 } 9575 9576 #[test] 9577 fn test_live_upgrade_balloon_local() { 9578 _test_live_migration_balloon(true, true) 9579 } 9580 } 9581 9582 mod live_migration_sequential { 9583 #[cfg(target_arch = "x86_64")] 9584 #[cfg(not(feature = "mshv"))] 9585 use super::*; 9586 9587 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 9588 #[test] 9589 #[cfg(target_arch = "x86_64")] 9590 #[cfg(not(feature = "mshv"))] 9591 fn test_live_migration_ovs_dpdk() { 9592 _test_live_migration_ovs_dpdk(false, false); 9593 } 9594 9595 #[test] 9596 #[cfg(target_arch = "x86_64")] 9597 #[cfg(not(feature = "mshv"))] 9598 fn test_live_migration_ovs_dpdk_local() { 9599 _test_live_migration_ovs_dpdk(false, true); 9600 } 9601 9602 #[test] 9603 #[cfg(target_arch = "x86_64")] 9604 #[cfg(not(feature = "mshv"))] 9605 fn test_live_upgrade_ovs_dpdk() { 9606 _test_live_migration_ovs_dpdk(true, false); 9607 } 9608 9609 #[test] 9610 #[cfg(target_arch = "x86_64")] 9611 #[cfg(not(feature = "mshv"))] 9612 fn test_live_upgrade_ovs_dpdk_local() { 9613 _test_live_migration_ovs_dpdk(true, true); 9614 } 9615 } 9616 } 9617 9618 #[cfg(target_arch = "aarch64")] 9619 mod aarch64_acpi { 9620 use crate::*; 9621 9622 #[test] 9623 fn test_simple_launch_acpi() { 9624 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9625 9626 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 9627 let guest = Guest::new(disk_config); 9628 9629 let mut child = GuestCommand::new(&guest) 9630 .args(["--cpus", "boot=1"]) 9631 .args(["--memory", "size=512M"]) 9632 .args(["--kernel", edk2_path().to_str().unwrap()]) 9633 .default_disks() 9634 .default_net() 9635 .args(["--serial", "tty", "--console", "off"]) 9636 .capture_output() 9637 .spawn() 9638 .unwrap(); 9639 9640 let r = std::panic::catch_unwind(|| { 9641 guest.wait_vm_boot(Some(120)).unwrap(); 9642 9643 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 9644 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 9645 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 9646 }); 9647 9648 let _ = child.kill(); 9649 let output = child.wait_with_output().unwrap(); 9650 9651 handle_child_output(r, &output); 9652 }); 9653 } 9654 9655 #[test] 9656 fn test_guest_numa_nodes_acpi() { 9657 _test_guest_numa_nodes(true); 9658 } 9659 9660 #[test] 9661 fn test_cpu_topology_421_acpi() { 9662 test_cpu_topology(4, 2, 1, true); 9663 } 9664 9665 #[test] 9666 fn test_cpu_topology_142_acpi() { 9667 test_cpu_topology(1, 4, 2, true); 9668 } 9669 9670 #[test] 9671 fn test_cpu_topology_262_acpi() { 9672 test_cpu_topology(2, 6, 2, true); 9673 } 9674 9675 #[test] 9676 fn test_power_button_acpi() { 9677 _test_power_button(true); 9678 } 9679 9680 #[test] 9681 fn test_virtio_iommu() { 9682 _test_virtio_iommu(true) 9683 } 9684 } 9685 9686 mod rate_limiter { 9687 use super::*; 9688 9689 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 9690 // compared to given 'limit' rate. 9691 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 9692 let upper_limit = limit * (1_f64 + difference); 9693 let lower_limit = limit * (1_f64 - difference); 9694 9695 if measured > lower_limit && measured < upper_limit { 9696 return true; 9697 } 9698 9699 eprintln!( 9700 "\n\n==== Start 'check_rate_limit' failed ==== \ 9701 \n\nmeasured={measured}, , lower_limit={lower_limit}, upper_limit={upper_limit} \ 9702 \n\n==== End 'check_rate_limit' failed ====\n\n" 9703 ); 9704 9705 false 9706 } 9707 9708 fn _test_rate_limiter_net(rx: bool) { 9709 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9710 let guest = Guest::new(Box::new(focal)); 9711 9712 let test_timeout = 10; 9713 let num_queues = 2; 9714 let queue_size = 256; 9715 let bw_size = 10485760_u64; // bytes 9716 let bw_refill_time = 100; // ms 9717 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 9718 9719 let net_params = format!( 9720 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 9721 guest.network.guest_mac, 9722 guest.network.host_ip, 9723 num_queues, 9724 queue_size, 9725 bw_size, 9726 bw_refill_time, 9727 ); 9728 9729 let mut child = GuestCommand::new(&guest) 9730 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 9731 .args(["--memory", "size=4G"]) 9732 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9733 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9734 .default_disks() 9735 .args(["--net", net_params.as_str()]) 9736 .capture_output() 9737 .spawn() 9738 .unwrap(); 9739 9740 let r = std::panic::catch_unwind(|| { 9741 guest.wait_vm_boot(None).unwrap(); 9742 let measured_bps = 9743 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, true) 9744 .unwrap(); 9745 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 9746 }); 9747 9748 let _ = child.kill(); 9749 let output = child.wait_with_output().unwrap(); 9750 handle_child_output(r, &output); 9751 } 9752 9753 #[test] 9754 fn test_rate_limiter_net_rx() { 9755 _test_rate_limiter_net(true); 9756 } 9757 9758 #[test] 9759 fn test_rate_limiter_net_tx() { 9760 _test_rate_limiter_net(false); 9761 } 9762 9763 fn _test_rate_limiter_block(bandwidth: bool) { 9764 let test_timeout = 10; 9765 let num_queues = 1; 9766 let fio_ops = FioOps::RandRW; 9767 9768 let bw_size = if bandwidth { 9769 10485760_u64 // bytes 9770 } else { 9771 100_u64 // I/O 9772 }; 9773 let bw_refill_time = 100; // ms 9774 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 9775 9776 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9777 let guest = Guest::new(Box::new(focal)); 9778 let api_socket = temp_api_path(&guest.tmp_dir); 9779 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 9780 let blk_rate_limiter_test_img = 9781 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 9782 9783 // Create the test block image 9784 assert!(exec_host_command_output(&format!( 9785 "dd if=/dev/zero of={blk_rate_limiter_test_img} bs=1M count=1024" 9786 )) 9787 .status 9788 .success()); 9789 9790 let test_blk_params = if bandwidth { 9791 format!( 9792 "path={blk_rate_limiter_test_img},bw_size={bw_size},bw_refill_time={bw_refill_time}" 9793 ) 9794 } else { 9795 format!( 9796 "path={blk_rate_limiter_test_img},ops_size={bw_size},ops_refill_time={bw_refill_time}" 9797 ) 9798 }; 9799 9800 let mut child = GuestCommand::new(&guest) 9801 .args(["--cpus", &format!("boot={num_queues}")]) 9802 .args(["--memory", "size=4G"]) 9803 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9804 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9805 .args([ 9806 "--disk", 9807 format!( 9808 "path={}", 9809 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 9810 ) 9811 .as_str(), 9812 format!( 9813 "path={}", 9814 guest.disk_config.disk(DiskType::CloudInit).unwrap() 9815 ) 9816 .as_str(), 9817 test_blk_params.as_str(), 9818 ]) 9819 .default_net() 9820 .args(["--api-socket", &api_socket]) 9821 .capture_output() 9822 .spawn() 9823 .unwrap(); 9824 9825 let r = std::panic::catch_unwind(|| { 9826 guest.wait_vm_boot(None).unwrap(); 9827 9828 let fio_command = format!( 9829 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 9830 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 9831 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 9832 ); 9833 let output = guest.ssh_command(&fio_command).unwrap(); 9834 9835 // Parse fio output 9836 let measured_rate = if bandwidth { 9837 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 9838 } else { 9839 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 9840 }; 9841 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 9842 }); 9843 9844 let _ = child.kill(); 9845 let output = child.wait_with_output().unwrap(); 9846 handle_child_output(r, &output); 9847 } 9848 9849 #[test] 9850 fn test_rate_limiter_block_bandwidth() { 9851 _test_rate_limiter_block(true) 9852 } 9853 9854 #[test] 9855 fn test_rate_limiter_block_iops() { 9856 _test_rate_limiter_block(false) 9857 } 9858 } 9859