1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use std::collections::HashMap; 14 use std::io::{BufRead, Read, Seek, Write}; 15 use std::os::unix::io::AsRawFd; 16 use std::path::PathBuf; 17 use std::process::{Child, Command, Stdio}; 18 use std::string::String; 19 use std::sync::mpsc::Receiver; 20 use std::sync::{mpsc, Mutex}; 21 use std::{fs, io, thread}; 22 23 use net_util::MacAddr; 24 use test_infra::*; 25 use vmm_sys_util::tempdir::TempDir; 26 use vmm_sys_util::tempfile::TempFile; 27 use wait_timeout::ChildExt; 28 29 // Constant taken from the VMM crate. 30 const MAX_NUM_PCI_SEGMENTS: u16 = 96; 31 32 #[cfg(target_arch = "x86_64")] 33 mod x86_64 { 34 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 35 pub const JAMMY_NVIDIA_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-nvidia.raw"; 36 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 37 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 38 "focal-server-cloudimg-amd64-custom-20210609-0-backing.qcow2"; 39 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 40 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 41 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20230119-0.raw"; 42 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2022-amd64-2.raw"; 43 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 44 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 45 } 46 47 #[cfg(target_arch = "x86_64")] 48 use x86_64::*; 49 50 #[cfg(target_arch = "aarch64")] 51 mod aarch64 { 52 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 53 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 54 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 55 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 56 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 57 "focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2"; 58 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 59 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 60 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 61 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 62 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 63 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 64 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 65 } 66 67 #[cfg(target_arch = "aarch64")] 68 use aarch64::*; 69 70 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 71 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 72 73 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 74 75 // This enum exists to make it more convenient to 76 // implement test for both D-Bus and REST APIs. 77 enum TargetApi { 78 // API socket 79 HttpApi(String), 80 // well known service name, object path 81 DBusApi(String, String), 82 } 83 84 impl TargetApi { 85 fn new_http_api(tmp_dir: &TempDir) -> Self { 86 Self::HttpApi(temp_api_path(tmp_dir)) 87 } 88 89 fn new_dbus_api(tmp_dir: &TempDir) -> Self { 90 // `tmp_dir` is in the form of "/tmp/chXXXXXX" 91 // and we take the `chXXXXXX` part as a unique identifier for the guest 92 let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); 93 94 Self::DBusApi( 95 format!("org.cloudhypervisor.{id}"), 96 format!("/org/cloudhypervisor/{id}"), 97 ) 98 } 99 100 fn guest_args(&self) -> Vec<String> { 101 match self { 102 TargetApi::HttpApi(api_socket) => { 103 vec![format!("--api-socket={}", api_socket.as_str())] 104 } 105 TargetApi::DBusApi(service_name, object_path) => { 106 vec![ 107 format!("--dbus-service-name={}", service_name.as_str()), 108 format!("--dbus-object-path={}", object_path.as_str()), 109 ] 110 } 111 } 112 } 113 114 fn remote_args(&self) -> Vec<String> { 115 // `guest_args` and `remote_args` are consistent with each other 116 self.guest_args() 117 } 118 119 fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { 120 let mut cmd = Command::new(clh_command("ch-remote")); 121 cmd.args(self.remote_args()); 122 cmd.arg(command); 123 124 if let Some(arg) = arg { 125 cmd.arg(arg); 126 } 127 128 let output = cmd.output().unwrap(); 129 if output.status.success() { 130 true 131 } else { 132 eprintln!("Error running ch-remote command: {:?}", &cmd); 133 let stderr = String::from_utf8_lossy(&output.stderr); 134 eprintln!("stderr: {stderr}"); 135 false 136 } 137 } 138 } 139 140 // Start cloud-hypervisor with no VM parameters, only the API server running. 141 // From the API: Create a VM, boot it and check that it looks as expected. 142 fn _test_api_create_boot(target_api: TargetApi, guest: Guest) { 143 let mut child = GuestCommand::new(&guest) 144 .args(target_api.guest_args()) 145 .capture_output() 146 .spawn() 147 .unwrap(); 148 149 thread::sleep(std::time::Duration::new(1, 0)); 150 151 // Verify API server is running 152 assert!(target_api.remote_command("ping", None)); 153 154 // Create the VM first 155 let cpu_count: u8 = 4; 156 let request_body = guest.api_create_body( 157 cpu_count, 158 direct_kernel_boot_path().to_str().unwrap(), 159 DIRECT_KERNEL_BOOT_CMDLINE, 160 ); 161 162 let temp_config_path = guest.tmp_dir.as_path().join("config"); 163 std::fs::write(&temp_config_path, request_body).unwrap(); 164 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 165 166 assert!(target_api.remote_command("create", Some(create_config),)); 167 168 // Then boot it 169 assert!(target_api.remote_command("boot", None)); 170 thread::sleep(std::time::Duration::new(20, 0)); 171 172 let r = std::panic::catch_unwind(|| { 173 // Check that the VM booted as expected 174 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 175 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 176 }); 177 178 kill_child(&mut child); 179 let output = child.wait_with_output().unwrap(); 180 181 handle_child_output(r, &output); 182 } 183 184 // Start cloud-hypervisor with no VM parameters, only the API server running. 185 // From the API: Create a VM, boot it and check it can be shutdown and then 186 // booted again 187 fn _test_api_shutdown(target_api: TargetApi, guest: Guest) { 188 let mut child = GuestCommand::new(&guest) 189 .args(target_api.guest_args()) 190 .capture_output() 191 .spawn() 192 .unwrap(); 193 194 thread::sleep(std::time::Duration::new(1, 0)); 195 196 // Verify API server is running 197 assert!(target_api.remote_command("ping", None)); 198 199 // Create the VM first 200 let cpu_count: u8 = 4; 201 let request_body = guest.api_create_body( 202 cpu_count, 203 direct_kernel_boot_path().to_str().unwrap(), 204 DIRECT_KERNEL_BOOT_CMDLINE, 205 ); 206 207 let temp_config_path = guest.tmp_dir.as_path().join("config"); 208 std::fs::write(&temp_config_path, request_body).unwrap(); 209 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 210 211 let r = std::panic::catch_unwind(|| { 212 assert!(target_api.remote_command("create", Some(create_config))); 213 214 // Then boot it 215 assert!(target_api.remote_command("boot", None)); 216 217 guest.wait_vm_boot(None).unwrap(); 218 219 // Check that the VM booted as expected 220 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 221 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 222 223 // Sync and shutdown without powering off to prevent filesystem 224 // corruption. 225 guest.ssh_command("sync").unwrap(); 226 guest.ssh_command("sudo shutdown -H now").unwrap(); 227 228 // Wait for the guest to be fully shutdown 229 thread::sleep(std::time::Duration::new(20, 0)); 230 231 // Then shut it down 232 assert!(target_api.remote_command("shutdown", None)); 233 234 // Then boot it again 235 assert!(target_api.remote_command("boot", None)); 236 237 guest.wait_vm_boot(None).unwrap(); 238 239 // Check that the VM booted as expected 240 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 241 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 242 }); 243 244 kill_child(&mut child); 245 let output = child.wait_with_output().unwrap(); 246 247 handle_child_output(r, &output); 248 } 249 250 // Start cloud-hypervisor with no VM parameters, only the API server running. 251 // From the API: Create a VM, boot it and check it can be deleted and then recreated 252 // booted again. 253 fn _test_api_delete(target_api: TargetApi, guest: Guest) { 254 let mut child = GuestCommand::new(&guest) 255 .args(target_api.guest_args()) 256 .capture_output() 257 .spawn() 258 .unwrap(); 259 260 thread::sleep(std::time::Duration::new(1, 0)); 261 262 // Verify API server is running 263 assert!(target_api.remote_command("ping", None)); 264 265 // Create the VM first 266 let cpu_count: u8 = 4; 267 let request_body = guest.api_create_body( 268 cpu_count, 269 direct_kernel_boot_path().to_str().unwrap(), 270 DIRECT_KERNEL_BOOT_CMDLINE, 271 ); 272 let temp_config_path = guest.tmp_dir.as_path().join("config"); 273 std::fs::write(&temp_config_path, request_body).unwrap(); 274 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 275 276 let r = std::panic::catch_unwind(|| { 277 assert!(target_api.remote_command("create", Some(create_config))); 278 279 // Then boot it 280 assert!(target_api.remote_command("boot", None)); 281 282 guest.wait_vm_boot(None).unwrap(); 283 284 // Check that the VM booted as expected 285 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 286 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 287 288 // Sync and shutdown without powering off to prevent filesystem 289 // corruption. 290 guest.ssh_command("sync").unwrap(); 291 guest.ssh_command("sudo shutdown -H now").unwrap(); 292 293 // Wait for the guest to be fully shutdown 294 thread::sleep(std::time::Duration::new(20, 0)); 295 296 // Then delete it 297 assert!(target_api.remote_command("delete", None)); 298 299 assert!(target_api.remote_command("create", Some(create_config))); 300 301 // Then boot it again 302 assert!(target_api.remote_command("boot", None)); 303 304 guest.wait_vm_boot(None).unwrap(); 305 306 // Check that the VM booted as expected 307 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 308 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 309 }); 310 311 kill_child(&mut child); 312 let output = child.wait_with_output().unwrap(); 313 314 handle_child_output(r, &output); 315 } 316 317 // Start cloud-hypervisor with no VM parameters, only the API server running. 318 // From the API: Create a VM, boot it and check that it looks as expected. 319 // Then we pause the VM, check that it's no longer available. 320 // Finally we resume the VM and check that it's available. 321 fn _test_api_pause_resume(target_api: TargetApi, guest: Guest) { 322 let mut child = GuestCommand::new(&guest) 323 .args(target_api.guest_args()) 324 .capture_output() 325 .spawn() 326 .unwrap(); 327 328 thread::sleep(std::time::Duration::new(1, 0)); 329 330 // Verify API server is running 331 assert!(target_api.remote_command("ping", None)); 332 333 // Create the VM first 334 let cpu_count: u8 = 4; 335 let request_body = guest.api_create_body( 336 cpu_count, 337 direct_kernel_boot_path().to_str().unwrap(), 338 DIRECT_KERNEL_BOOT_CMDLINE, 339 ); 340 341 let temp_config_path = guest.tmp_dir.as_path().join("config"); 342 std::fs::write(&temp_config_path, request_body).unwrap(); 343 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 344 345 assert!(target_api.remote_command("create", Some(create_config))); 346 347 // Then boot it 348 assert!(target_api.remote_command("boot", None)); 349 thread::sleep(std::time::Duration::new(20, 0)); 350 351 let r = std::panic::catch_unwind(|| { 352 // Check that the VM booted as expected 353 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 354 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 355 356 // We now pause the VM 357 assert!(target_api.remote_command("pause", None)); 358 359 // Check pausing again fails 360 assert!(!target_api.remote_command("pause", None)); 361 362 thread::sleep(std::time::Duration::new(2, 0)); 363 364 // SSH into the VM should fail 365 assert!(ssh_command_ip( 366 "grep -c processor /proc/cpuinfo", 367 &guest.network.guest_ip, 368 2, 369 5 370 ) 371 .is_err()); 372 373 // Resume the VM 374 assert!(target_api.remote_command("resume", None)); 375 376 // Check resuming again fails 377 assert!(!target_api.remote_command("resume", None)); 378 379 thread::sleep(std::time::Duration::new(2, 0)); 380 381 // Now we should be able to SSH back in and get the right number of CPUs 382 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 383 }); 384 385 kill_child(&mut child); 386 let output = child.wait_with_output().unwrap(); 387 388 handle_child_output(r, &output); 389 } 390 391 fn _test_pty_interaction(pty_path: PathBuf) { 392 let mut cf = std::fs::OpenOptions::new() 393 .write(true) 394 .read(true) 395 .open(pty_path) 396 .unwrap(); 397 398 // Some dumb sleeps but we don't want to write 399 // before the console is up and we don't want 400 // to try and write the next line before the 401 // login process is ready. 402 thread::sleep(std::time::Duration::new(5, 0)); 403 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 404 thread::sleep(std::time::Duration::new(2, 0)); 405 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 406 thread::sleep(std::time::Duration::new(2, 0)); 407 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 408 thread::sleep(std::time::Duration::new(2, 0)); 409 410 // read pty and ensure they have a login shell 411 // some fairly hacky workarounds to avoid looping 412 // forever in case the channel is blocked getting output 413 let ptyc = pty_read(cf); 414 let mut empty = 0; 415 let mut prev = String::new(); 416 loop { 417 thread::sleep(std::time::Duration::new(2, 0)); 418 match ptyc.try_recv() { 419 Ok(line) => { 420 empty = 0; 421 prev = prev + &line; 422 if prev.contains("test_pty_console") { 423 break; 424 } 425 } 426 Err(mpsc::TryRecvError::Empty) => { 427 empty += 1; 428 assert!(empty <= 5, "No login on pty"); 429 } 430 _ => { 431 panic!("No login on pty") 432 } 433 } 434 } 435 } 436 437 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 438 let mut workload_path = dirs::home_dir().unwrap(); 439 workload_path.push("workloads"); 440 441 let mut virtiofsd_path = workload_path; 442 virtiofsd_path.push("virtiofsd"); 443 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 444 445 let virtiofsd_socket_path = 446 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 447 448 // Start the daemon 449 let child = Command::new(virtiofsd_path.as_str()) 450 .args(["--shared-dir", shared_dir]) 451 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 452 .args(["--cache", "never"]) 453 .spawn() 454 .unwrap(); 455 456 thread::sleep(std::time::Duration::new(10, 0)); 457 458 (child, virtiofsd_socket_path) 459 } 460 461 fn prepare_vubd( 462 tmp_dir: &TempDir, 463 blk_img: &str, 464 num_queues: usize, 465 rdonly: bool, 466 direct: bool, 467 ) -> (std::process::Child, String) { 468 let mut workload_path = dirs::home_dir().unwrap(); 469 workload_path.push("workloads"); 470 471 let mut blk_file_path = workload_path; 472 blk_file_path.push(blk_img); 473 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 474 475 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 476 477 // Start the daemon 478 let child = Command::new(clh_command("vhost_user_block")) 479 .args([ 480 "--block-backend", 481 format!( 482 "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" 483 ) 484 .as_str(), 485 ]) 486 .spawn() 487 .unwrap(); 488 489 thread::sleep(std::time::Duration::new(10, 0)); 490 491 (child, vubd_socket_path) 492 } 493 494 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 495 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 496 } 497 498 fn temp_api_path(tmp_dir: &TempDir) -> String { 499 String::from( 500 tmp_dir 501 .as_path() 502 .join("cloud-hypervisor.sock") 503 .to_str() 504 .unwrap(), 505 ) 506 } 507 508 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 509 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 510 } 511 512 // Creates the directory and returns the path. 513 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 514 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 515 std::fs::create_dir(&snapshot_dir).unwrap(); 516 snapshot_dir 517 } 518 519 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 520 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 521 vmcore_file 522 } 523 524 // Creates the path for direct kernel boot and return the path. 525 // For x86_64, this function returns the vmlinux kernel path. 526 // For AArch64, this function returns the PE kernel path. 527 fn direct_kernel_boot_path() -> PathBuf { 528 let mut workload_path = dirs::home_dir().unwrap(); 529 workload_path.push("workloads"); 530 531 let mut kernel_path = workload_path; 532 #[cfg(target_arch = "x86_64")] 533 kernel_path.push("vmlinux"); 534 #[cfg(target_arch = "aarch64")] 535 kernel_path.push("Image"); 536 537 kernel_path 538 } 539 540 fn edk2_path() -> PathBuf { 541 let mut workload_path = dirs::home_dir().unwrap(); 542 workload_path.push("workloads"); 543 let mut edk2_path = workload_path; 544 edk2_path.push(OVMF_NAME); 545 546 edk2_path 547 } 548 549 fn cloud_hypervisor_release_path() -> String { 550 let mut workload_path = dirs::home_dir().unwrap(); 551 workload_path.push("workloads"); 552 553 let mut ch_release_path = workload_path; 554 #[cfg(target_arch = "x86_64")] 555 ch_release_path.push("cloud-hypervisor-static"); 556 #[cfg(target_arch = "aarch64")] 557 ch_release_path.push("cloud-hypervisor-static-aarch64"); 558 559 ch_release_path.into_os_string().into_string().unwrap() 560 } 561 562 fn prepare_vhost_user_net_daemon( 563 tmp_dir: &TempDir, 564 ip: &str, 565 tap: Option<&str>, 566 mtu: Option<u16>, 567 num_queues: usize, 568 client_mode: bool, 569 ) -> (std::process::Command, String) { 570 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 571 572 // Start the daemon 573 let mut net_params = format!( 574 "ip={ip},mask=255.255.255.0,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" 575 ); 576 577 if let Some(tap) = tap { 578 net_params.push_str(format!(",tap={tap}").as_str()); 579 } 580 581 if let Some(mtu) = mtu { 582 net_params.push_str(format!(",mtu={mtu}").as_str()); 583 } 584 585 let mut command = Command::new(clh_command("vhost_user_net")); 586 command.args(["--net-backend", net_params.as_str()]); 587 588 (command, vunet_socket_path) 589 } 590 591 fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { 592 let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); 593 let swtpm_socket_path = String::from( 594 tmp_dir 595 .as_path() 596 .join("swtpm") 597 .join("swtpm.sock") 598 .to_str() 599 .unwrap(), 600 ); 601 std::fs::create_dir(&swtpm_tpm_dir).unwrap(); 602 603 let mut swtpm_command = Command::new("swtpm"); 604 let swtpm_args = [ 605 "socket", 606 "--tpmstate", 607 &format!("dir={swtpm_tpm_dir}"), 608 "--ctrl", 609 &format!("type=unixio,path={swtpm_socket_path}"), 610 "--flags", 611 "startup-clear", 612 "--tpm2", 613 ]; 614 swtpm_command.args(swtpm_args); 615 616 (swtpm_command, swtpm_socket_path) 617 } 618 619 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 620 let mut cmd = Command::new(clh_command("ch-remote")); 621 cmd.args([&format!("--api-socket={api_socket}"), command]); 622 623 if let Some(arg) = arg { 624 cmd.arg(arg); 625 } 626 let output = cmd.output().unwrap(); 627 if output.status.success() { 628 true 629 } else { 630 eprintln!("Error running ch-remote command: {:?}", &cmd); 631 let stderr = String::from_utf8_lossy(&output.stderr); 632 eprintln!("stderr: {stderr}"); 633 false 634 } 635 } 636 637 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 638 let mut cmd = Command::new(clh_command("ch-remote")); 639 cmd.args([&format!("--api-socket={api_socket}"), command]); 640 641 if let Some(arg) = arg { 642 cmd.arg(arg); 643 } 644 645 let output = cmd.output().expect("Failed to launch ch-remote"); 646 647 (output.status.success(), output.stdout) 648 } 649 650 fn resize_command( 651 api_socket: &str, 652 desired_vcpus: Option<u8>, 653 desired_ram: Option<usize>, 654 desired_balloon: Option<usize>, 655 event_file: Option<&str>, 656 ) -> bool { 657 let mut cmd = Command::new(clh_command("ch-remote")); 658 cmd.args([&format!("--api-socket={api_socket}"), "resize"]); 659 660 if let Some(desired_vcpus) = desired_vcpus { 661 cmd.arg(format!("--cpus={desired_vcpus}")); 662 } 663 664 if let Some(desired_ram) = desired_ram { 665 cmd.arg(format!("--memory={desired_ram}")); 666 } 667 668 if let Some(desired_balloon) = desired_balloon { 669 cmd.arg(format!("--balloon={desired_balloon}")); 670 } 671 672 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 673 674 if let Some(event_path) = event_file { 675 let latest_events = [ 676 &MetaEvent { 677 event: "resizing".to_string(), 678 device_id: None, 679 }, 680 &MetaEvent { 681 event: "resized".to_string(), 682 device_id: None, 683 }, 684 ]; 685 // See: #5938 686 thread::sleep(std::time::Duration::new(1, 0)); 687 assert!(check_latest_events_exact(&latest_events, event_path)); 688 } 689 690 ret 691 } 692 693 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 694 let mut cmd = Command::new(clh_command("ch-remote")); 695 cmd.args([ 696 &format!("--api-socket={api_socket}"), 697 "resize-zone", 698 &format!("--id={id}"), 699 &format!("--size={desired_size}"), 700 ]); 701 702 cmd.status().expect("Failed to launch ch-remote").success() 703 } 704 705 // setup OVS-DPDK bridge and ports 706 fn setup_ovs_dpdk() { 707 // setup OVS-DPDK 708 assert!(exec_host_command_status("service openvswitch-switch start").success()); 709 assert!(exec_host_command_status("ovs-vsctl init").success()); 710 assert!( 711 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 712 .success() 713 ); 714 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 715 716 // Create OVS-DPDK bridge and ports 717 assert!(exec_host_command_status( 718 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 719 ) 720 .success()); 721 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 722 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 723 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 724 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 725 } 726 fn cleanup_ovs_dpdk() { 727 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 728 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 729 } 730 // Setup two guests and ensure they are connected through ovs-dpdk 731 fn setup_ovs_dpdk_guests( 732 guest1: &Guest, 733 guest2: &Guest, 734 api_socket: &str, 735 release_binary: bool, 736 ) -> (Child, Child) { 737 setup_ovs_dpdk(); 738 739 let clh_path = if !release_binary { 740 clh_command("cloud-hypervisor") 741 } else { 742 cloud_hypervisor_release_path() 743 }; 744 745 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 746 .args(["--cpus", "boot=2"]) 747 .args(["--memory", "size=0,shared=on"]) 748 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 749 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 750 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 751 .default_disks() 752 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 753 .capture_output() 754 .spawn() 755 .unwrap(); 756 757 #[cfg(target_arch = "x86_64")] 758 let guest_net_iface = "ens5"; 759 #[cfg(target_arch = "aarch64")] 760 let guest_net_iface = "enp0s5"; 761 762 let r = std::panic::catch_unwind(|| { 763 guest1.wait_vm_boot(None).unwrap(); 764 765 guest1 766 .ssh_command(&format!( 767 "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" 768 )) 769 .unwrap(); 770 guest1 771 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 772 .unwrap(); 773 774 let guest_ip = guest1.network.guest_ip.clone(); 775 thread::spawn(move || { 776 ssh_command_ip( 777 "nc -l 12345", 778 &guest_ip, 779 DEFAULT_SSH_RETRIES, 780 DEFAULT_SSH_TIMEOUT, 781 ) 782 .unwrap(); 783 }); 784 }); 785 if r.is_err() { 786 cleanup_ovs_dpdk(); 787 788 let _ = child1.kill(); 789 let output = child1.wait_with_output().unwrap(); 790 handle_child_output(r, &output); 791 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 792 } 793 794 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 795 .args(["--api-socket", api_socket]) 796 .args(["--cpus", "boot=2"]) 797 .args(["--memory", "size=0,shared=on"]) 798 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 799 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 800 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 801 .default_disks() 802 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 803 .capture_output() 804 .spawn() 805 .unwrap(); 806 807 let r = std::panic::catch_unwind(|| { 808 guest2.wait_vm_boot(None).unwrap(); 809 810 guest2 811 .ssh_command(&format!( 812 "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" 813 )) 814 .unwrap(); 815 guest2 816 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 817 .unwrap(); 818 819 // Check the connection works properly between the two VMs 820 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 821 }); 822 if r.is_err() { 823 cleanup_ovs_dpdk(); 824 825 let _ = child1.kill(); 826 let _ = child2.kill(); 827 let output = child2.wait_with_output().unwrap(); 828 handle_child_output(r, &output); 829 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 830 } 831 832 (child1, child2) 833 } 834 835 enum FwType { 836 Ovmf, 837 RustHypervisorFirmware, 838 } 839 840 fn fw_path(_fw_type: FwType) -> String { 841 let mut workload_path = dirs::home_dir().unwrap(); 842 workload_path.push("workloads"); 843 844 let mut fw_path = workload_path; 845 #[cfg(target_arch = "aarch64")] 846 fw_path.push("CLOUDHV_EFI.fd"); 847 #[cfg(target_arch = "x86_64")] 848 { 849 match _fw_type { 850 FwType::Ovmf => fw_path.push(OVMF_NAME), 851 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 852 } 853 } 854 855 fw_path.to_str().unwrap().to_string() 856 } 857 858 #[derive(Debug)] 859 struct MetaEvent { 860 event: String, 861 device_id: Option<String>, 862 } 863 864 impl MetaEvent { 865 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 866 let mut matched = false; 867 if v["event"].as_str().unwrap() == self.event { 868 if let Some(device_id) = &self.device_id { 869 if v["properties"]["id"].as_str().unwrap() == device_id { 870 matched = true 871 } 872 } else { 873 matched = true; 874 } 875 } 876 matched 877 } 878 } 879 880 // Parse the event_monitor file based on the format that each event 881 // is followed by a double newline 882 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 883 let content = fs::read(event_file).unwrap(); 884 let mut ret = Vec::new(); 885 for entry in String::from_utf8_lossy(&content) 886 .trim() 887 .split("\n\n") 888 .collect::<Vec<&str>>() 889 { 890 ret.push(serde_json::from_str(entry).unwrap()); 891 } 892 893 ret 894 } 895 896 // Return true if all events from the input 'expected_events' are matched sequentially 897 // with events from the 'event_file' 898 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 899 let json_events = parse_event_file(event_file); 900 let len = expected_events.len(); 901 let mut idx = 0; 902 for e in &json_events { 903 if idx == len { 904 break; 905 } 906 if expected_events[idx].match_with_json_event(e) { 907 idx += 1; 908 } 909 } 910 911 let ret = idx == len; 912 913 if !ret { 914 eprintln!( 915 "\n\n==== Start 'check_sequential_events' failed ==== \ 916 \n\nexpected_events={:?}\nactual_events={:?} \ 917 \n\n==== End 'check_sequential_events' failed ====", 918 expected_events, json_events, 919 ); 920 } 921 922 ret 923 } 924 925 // Return true if all events from the input 'expected_events' are matched exactly 926 // with events from the 'event_file' 927 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 928 let json_events = parse_event_file(event_file); 929 assert!(expected_events.len() <= json_events.len()); 930 let json_events = &json_events[..expected_events.len()]; 931 932 for (idx, e) in json_events.iter().enumerate() { 933 if !expected_events[idx].match_with_json_event(e) { 934 eprintln!( 935 "\n\n==== Start 'check_sequential_events_exact' failed ==== \ 936 \n\nexpected_events={:?}\nactual_events={:?} \ 937 \n\n==== End 'check_sequential_events_exact' failed ====", 938 expected_events, json_events, 939 ); 940 941 return false; 942 } 943 } 944 945 true 946 } 947 948 // Return true if events from the input 'latest_events' are matched exactly 949 // with the most recent events from the 'event_file' 950 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 951 let json_events = parse_event_file(event_file); 952 assert!(latest_events.len() <= json_events.len()); 953 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 954 955 for (idx, e) in json_events.iter().enumerate() { 956 if !latest_events[idx].match_with_json_event(e) { 957 eprintln!( 958 "\n\n==== Start 'check_latest_events_exact' failed ==== \ 959 \n\nexpected_events={:?}\nactual_events={:?} \ 960 \n\n==== End 'check_latest_events_exact' failed ====", 961 latest_events, json_events, 962 ); 963 964 return false; 965 } 966 } 967 968 true 969 } 970 971 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 972 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 973 let guest = Guest::new(Box::new(focal)); 974 let total_vcpus = threads_per_core * cores_per_package * packages; 975 let direct_kernel_boot_path = direct_kernel_boot_path(); 976 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 977 let fw_path = fw_path(FwType::RustHypervisorFirmware); 978 if use_fw { 979 kernel_path = fw_path.as_str(); 980 } 981 982 let mut child = GuestCommand::new(&guest) 983 .args([ 984 "--cpus", 985 &format!( 986 "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" 987 ), 988 ]) 989 .args(["--memory", "size=512M"]) 990 .args(["--kernel", kernel_path]) 991 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 992 .default_disks() 993 .default_net() 994 .capture_output() 995 .spawn() 996 .unwrap(); 997 998 let r = std::panic::catch_unwind(|| { 999 guest.wait_vm_boot(None).unwrap(); 1000 assert_eq!( 1001 guest.get_cpu_count().unwrap_or_default(), 1002 u32::from(total_vcpus) 1003 ); 1004 assert_eq!( 1005 guest 1006 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1007 .unwrap() 1008 .trim() 1009 .parse::<u8>() 1010 .unwrap_or(0), 1011 threads_per_core 1012 ); 1013 1014 assert_eq!( 1015 guest 1016 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1017 .unwrap() 1018 .trim() 1019 .parse::<u8>() 1020 .unwrap_or(0), 1021 cores_per_package 1022 ); 1023 1024 assert_eq!( 1025 guest 1026 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1027 .unwrap() 1028 .trim() 1029 .parse::<u8>() 1030 .unwrap_or(0), 1031 packages 1032 ); 1033 1034 #[cfg(target_arch = "x86_64")] 1035 { 1036 let mut cpu_id = 0; 1037 for package_id in 0..packages { 1038 for core_id in 0..cores_per_package { 1039 for _ in 0..threads_per_core { 1040 assert_eq!( 1041 guest 1042 .ssh_command(&format!("cat /sys/devices/system/cpu/cpu{cpu_id}/topology/physical_package_id")) 1043 .unwrap() 1044 .trim() 1045 .parse::<u8>() 1046 .unwrap_or(0), 1047 package_id 1048 ); 1049 1050 assert_eq!( 1051 guest 1052 .ssh_command(&format!( 1053 "cat /sys/devices/system/cpu/cpu{cpu_id}/topology/core_id" 1054 )) 1055 .unwrap() 1056 .trim() 1057 .parse::<u8>() 1058 .unwrap_or(0), 1059 core_id 1060 ); 1061 1062 cpu_id += 1; 1063 } 1064 } 1065 } 1066 } 1067 }); 1068 1069 kill_child(&mut child); 1070 let output = child.wait_with_output().unwrap(); 1071 1072 handle_child_output(r, &output); 1073 } 1074 1075 #[allow(unused_variables)] 1076 fn _test_guest_numa_nodes(acpi: bool) { 1077 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1078 let guest = Guest::new(Box::new(focal)); 1079 let api_socket = temp_api_path(&guest.tmp_dir); 1080 #[cfg(target_arch = "x86_64")] 1081 let kernel_path = direct_kernel_boot_path(); 1082 #[cfg(target_arch = "aarch64")] 1083 let kernel_path = if acpi { 1084 edk2_path() 1085 } else { 1086 direct_kernel_boot_path() 1087 }; 1088 1089 let mut child = GuestCommand::new(&guest) 1090 .args(["--cpus", "boot=6,max=12"]) 1091 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 1092 .args([ 1093 "--memory-zone", 1094 "id=mem0,size=1G,hotplug_size=3G", 1095 "id=mem1,size=2G,hotplug_size=3G", 1096 "id=mem2,size=3G,hotplug_size=3G", 1097 ]) 1098 .args([ 1099 "--numa", 1100 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 1101 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 1102 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 1103 ]) 1104 .args(["--kernel", kernel_path.to_str().unwrap()]) 1105 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1106 .args(["--api-socket", &api_socket]) 1107 .capture_output() 1108 .default_disks() 1109 .default_net() 1110 .spawn() 1111 .unwrap(); 1112 1113 let r = std::panic::catch_unwind(|| { 1114 guest.wait_vm_boot(None).unwrap(); 1115 1116 guest.check_numa_common( 1117 Some(&[960_000, 1_920_000, 2_880_000]), 1118 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 1119 Some(&["10 15 20", "20 10 25", "25 30 10"]), 1120 ); 1121 1122 // AArch64 currently does not support hotplug, and therefore we only 1123 // test hotplug-related function on x86_64 here. 1124 #[cfg(target_arch = "x86_64")] 1125 { 1126 guest.enable_memory_hotplug(); 1127 1128 // Resize every memory zone and check each associated NUMA node 1129 // has been assigned the right amount of memory. 1130 resize_zone_command(&api_socket, "mem0", "4G"); 1131 resize_zone_command(&api_socket, "mem1", "4G"); 1132 resize_zone_command(&api_socket, "mem2", "4G"); 1133 // Resize to the maximum amount of CPUs and check each NUMA 1134 // node has been assigned the right CPUs set. 1135 resize_command(&api_socket, Some(12), None, None, None); 1136 thread::sleep(std::time::Duration::new(5, 0)); 1137 1138 guest.check_numa_common( 1139 Some(&[3_840_000, 3_840_000, 3_840_000]), 1140 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 1141 None, 1142 ); 1143 } 1144 }); 1145 1146 kill_child(&mut child); 1147 let output = child.wait_with_output().unwrap(); 1148 1149 handle_child_output(r, &output); 1150 } 1151 1152 #[allow(unused_variables)] 1153 fn _test_power_button(acpi: bool) { 1154 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1155 let guest = Guest::new(Box::new(focal)); 1156 let mut cmd = GuestCommand::new(&guest); 1157 let api_socket = temp_api_path(&guest.tmp_dir); 1158 1159 #[cfg(target_arch = "x86_64")] 1160 let kernel_path = direct_kernel_boot_path(); 1161 #[cfg(target_arch = "aarch64")] 1162 let kernel_path = if acpi { 1163 edk2_path() 1164 } else { 1165 direct_kernel_boot_path() 1166 }; 1167 1168 cmd.args(["--cpus", "boot=1"]) 1169 .args(["--memory", "size=512M"]) 1170 .args(["--kernel", kernel_path.to_str().unwrap()]) 1171 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1172 .capture_output() 1173 .default_disks() 1174 .default_net() 1175 .args(["--api-socket", &api_socket]); 1176 1177 let child = cmd.spawn().unwrap(); 1178 1179 let r = std::panic::catch_unwind(|| { 1180 guest.wait_vm_boot(None).unwrap(); 1181 assert!(remote_command(&api_socket, "power-button", None)); 1182 }); 1183 1184 let output = child.wait_with_output().unwrap(); 1185 assert!(output.status.success()); 1186 handle_child_output(r, &output); 1187 } 1188 1189 type PrepareNetDaemon = dyn Fn( 1190 &TempDir, 1191 &str, 1192 Option<&str>, 1193 Option<u16>, 1194 usize, 1195 bool, 1196 ) -> (std::process::Command, String); 1197 1198 fn test_vhost_user_net( 1199 tap: Option<&str>, 1200 num_queues: usize, 1201 prepare_daemon: &PrepareNetDaemon, 1202 generate_host_mac: bool, 1203 client_mode_daemon: bool, 1204 ) { 1205 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1206 let guest = Guest::new(Box::new(focal)); 1207 let api_socket = temp_api_path(&guest.tmp_dir); 1208 1209 let kernel_path = direct_kernel_boot_path(); 1210 1211 let host_mac = if generate_host_mac { 1212 Some(MacAddr::local_random()) 1213 } else { 1214 None 1215 }; 1216 1217 let mtu = Some(3000); 1218 1219 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 1220 &guest.tmp_dir, 1221 &guest.network.host_ip, 1222 tap, 1223 mtu, 1224 num_queues, 1225 client_mode_daemon, 1226 ); 1227 1228 let net_params = format!( 1229 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 1230 guest.network.guest_mac, 1231 vunet_socket_path, 1232 num_queues, 1233 if let Some(host_mac) = host_mac { 1234 format!(",host_mac={host_mac}") 1235 } else { 1236 "".to_owned() 1237 }, 1238 if client_mode_daemon { 1239 "server" 1240 } else { 1241 "client" 1242 }, 1243 ); 1244 1245 let mut ch_command = GuestCommand::new(&guest); 1246 ch_command 1247 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 1248 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1249 .args(["--kernel", kernel_path.to_str().unwrap()]) 1250 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1251 .default_disks() 1252 .args(["--net", net_params.as_str()]) 1253 .args(["--api-socket", &api_socket]) 1254 .capture_output(); 1255 1256 let mut daemon_child: std::process::Child; 1257 let mut child: std::process::Child; 1258 1259 if client_mode_daemon { 1260 child = ch_command.spawn().unwrap(); 1261 // Make sure the VMM is waiting for the backend to connect 1262 thread::sleep(std::time::Duration::new(10, 0)); 1263 daemon_child = daemon_command.spawn().unwrap(); 1264 } else { 1265 daemon_child = daemon_command.spawn().unwrap(); 1266 // Make sure the backend is waiting for the VMM to connect 1267 thread::sleep(std::time::Duration::new(10, 0)); 1268 child = ch_command.spawn().unwrap(); 1269 } 1270 1271 let r = std::panic::catch_unwind(|| { 1272 guest.wait_vm_boot(None).unwrap(); 1273 1274 if let Some(tap_name) = tap { 1275 let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); 1276 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1277 } 1278 1279 if let Some(host_mac) = tap { 1280 let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); 1281 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1282 } 1283 1284 #[cfg(target_arch = "aarch64")] 1285 let iface = "enp0s4"; 1286 #[cfg(target_arch = "x86_64")] 1287 let iface = "ens4"; 1288 1289 assert_eq!( 1290 guest 1291 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 1292 .unwrap() 1293 .trim(), 1294 "3000" 1295 ); 1296 1297 // 1 network interface + default localhost ==> 2 interfaces 1298 // It's important to note that this test is fully exercising the 1299 // vhost-user-net implementation and the associated backend since 1300 // it does not define any --net network interface. That means all 1301 // the ssh communication in that test happens through the network 1302 // interface backed by vhost-user-net. 1303 assert_eq!( 1304 guest 1305 .ssh_command("ip -o link | wc -l") 1306 .unwrap() 1307 .trim() 1308 .parse::<u32>() 1309 .unwrap_or_default(), 1310 2 1311 ); 1312 1313 // The following pci devices will appear on guest with PCI-MSI 1314 // interrupt vectors assigned. 1315 // 1 virtio-console with 3 vectors: config, Rx, Tx 1316 // 1 virtio-blk with 2 vectors: config, Request 1317 // 1 virtio-blk with 2 vectors: config, Request 1318 // 1 virtio-rng with 2 vectors: config, Request 1319 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1320 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1321 // Based on the above, the total vectors should 14. 1322 #[cfg(target_arch = "x86_64")] 1323 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1324 #[cfg(target_arch = "aarch64")] 1325 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1326 assert_eq!( 1327 guest 1328 .ssh_command(grep_cmd) 1329 .unwrap() 1330 .trim() 1331 .parse::<u32>() 1332 .unwrap_or_default(), 1333 10 + (num_queues as u32) 1334 ); 1335 1336 // ACPI feature is needed. 1337 #[cfg(target_arch = "x86_64")] 1338 { 1339 guest.enable_memory_hotplug(); 1340 1341 // Add RAM to the VM 1342 let desired_ram = 1024 << 20; 1343 resize_command(&api_socket, None, Some(desired_ram), None, None); 1344 1345 thread::sleep(std::time::Duration::new(10, 0)); 1346 1347 // Here by simply checking the size (through ssh), we validate 1348 // the connection is still working, which means vhost-user-net 1349 // keeps working after the resize. 1350 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1351 } 1352 }); 1353 1354 kill_child(&mut child); 1355 let output = child.wait_with_output().unwrap(); 1356 1357 thread::sleep(std::time::Duration::new(5, 0)); 1358 let _ = daemon_child.kill(); 1359 let _ = daemon_child.wait(); 1360 1361 handle_child_output(r, &output); 1362 } 1363 1364 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1365 1366 fn test_vhost_user_blk( 1367 num_queues: usize, 1368 readonly: bool, 1369 direct: bool, 1370 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1371 ) { 1372 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1373 let guest = Guest::new(Box::new(focal)); 1374 let api_socket = temp_api_path(&guest.tmp_dir); 1375 1376 let kernel_path = direct_kernel_boot_path(); 1377 1378 let (blk_params, daemon_child) = { 1379 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1380 // Start the daemon 1381 let (daemon_child, vubd_socket_path) = 1382 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1383 1384 ( 1385 format!( 1386 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1387 ), 1388 Some(daemon_child), 1389 ) 1390 }; 1391 1392 let mut child = GuestCommand::new(&guest) 1393 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1394 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1395 .args(["--kernel", kernel_path.to_str().unwrap()]) 1396 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1397 .args([ 1398 "--disk", 1399 format!( 1400 "path={}", 1401 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1402 ) 1403 .as_str(), 1404 format!( 1405 "path={}", 1406 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1407 ) 1408 .as_str(), 1409 blk_params.as_str(), 1410 ]) 1411 .default_net() 1412 .args(["--api-socket", &api_socket]) 1413 .capture_output() 1414 .spawn() 1415 .unwrap(); 1416 1417 let r = std::panic::catch_unwind(|| { 1418 guest.wait_vm_boot(None).unwrap(); 1419 1420 // Check both if /dev/vdc exists and if the block size is 16M. 1421 assert_eq!( 1422 guest 1423 .ssh_command("lsblk | grep vdc | grep -c 16M") 1424 .unwrap() 1425 .trim() 1426 .parse::<u32>() 1427 .unwrap_or_default(), 1428 1 1429 ); 1430 1431 // Check if this block is RO or RW. 1432 assert_eq!( 1433 guest 1434 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1435 .unwrap() 1436 .trim() 1437 .parse::<u32>() 1438 .unwrap_or_default(), 1439 readonly as u32 1440 ); 1441 1442 // Check if the number of queues in /sys/block/vdc/mq matches the 1443 // expected num_queues. 1444 assert_eq!( 1445 guest 1446 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1447 .unwrap() 1448 .trim() 1449 .parse::<u32>() 1450 .unwrap_or_default(), 1451 num_queues as u32 1452 ); 1453 1454 // Mount the device 1455 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1456 guest.ssh_command("mkdir mount_image").unwrap(); 1457 guest 1458 .ssh_command( 1459 format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), 1460 ) 1461 .unwrap(); 1462 1463 // Check the content of the block device. The file "foo" should 1464 // contain "bar". 1465 assert_eq!( 1466 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1467 "bar" 1468 ); 1469 1470 // ACPI feature is needed. 1471 #[cfg(target_arch = "x86_64")] 1472 { 1473 guest.enable_memory_hotplug(); 1474 1475 // Add RAM to the VM 1476 let desired_ram = 1024 << 20; 1477 resize_command(&api_socket, None, Some(desired_ram), None, None); 1478 1479 thread::sleep(std::time::Duration::new(10, 0)); 1480 1481 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1482 1483 // Check again the content of the block device after the resize 1484 // has been performed. 1485 assert_eq!( 1486 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1487 "bar" 1488 ); 1489 } 1490 1491 // Unmount the device 1492 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1493 guest.ssh_command("rm -r mount_image").unwrap(); 1494 }); 1495 1496 kill_child(&mut child); 1497 let output = child.wait_with_output().unwrap(); 1498 1499 if let Some(mut daemon_child) = daemon_child { 1500 thread::sleep(std::time::Duration::new(5, 0)); 1501 let _ = daemon_child.kill(); 1502 let _ = daemon_child.wait(); 1503 } 1504 1505 handle_child_output(r, &output); 1506 } 1507 1508 fn test_boot_from_vhost_user_blk( 1509 num_queues: usize, 1510 readonly: bool, 1511 direct: bool, 1512 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1513 ) { 1514 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1515 let guest = Guest::new(Box::new(focal)); 1516 1517 let kernel_path = direct_kernel_boot_path(); 1518 1519 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1520 1521 let (blk_boot_params, daemon_child) = { 1522 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1523 // Start the daemon 1524 let (daemon_child, vubd_socket_path) = prepare_daemon( 1525 &guest.tmp_dir, 1526 disk_path.as_str(), 1527 num_queues, 1528 readonly, 1529 direct, 1530 ); 1531 1532 ( 1533 format!( 1534 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1535 ), 1536 Some(daemon_child), 1537 ) 1538 }; 1539 1540 let mut child = GuestCommand::new(&guest) 1541 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1542 .args(["--memory", "size=512M,shared=on"]) 1543 .args(["--kernel", kernel_path.to_str().unwrap()]) 1544 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1545 .args([ 1546 "--disk", 1547 blk_boot_params.as_str(), 1548 format!( 1549 "path={}", 1550 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1551 ) 1552 .as_str(), 1553 ]) 1554 .default_net() 1555 .capture_output() 1556 .spawn() 1557 .unwrap(); 1558 1559 let r = std::panic::catch_unwind(|| { 1560 guest.wait_vm_boot(None).unwrap(); 1561 1562 // Just check the VM booted correctly. 1563 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1564 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1565 }); 1566 kill_child(&mut child); 1567 let output = child.wait_with_output().unwrap(); 1568 1569 if let Some(mut daemon_child) = daemon_child { 1570 thread::sleep(std::time::Duration::new(5, 0)); 1571 let _ = daemon_child.kill(); 1572 let _ = daemon_child.wait(); 1573 } 1574 1575 handle_child_output(r, &output); 1576 } 1577 1578 fn _test_virtio_fs( 1579 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1580 hotplug: bool, 1581 pci_segment: Option<u16>, 1582 ) { 1583 #[cfg(target_arch = "aarch64")] 1584 let focal_image = if hotplug { 1585 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1586 } else { 1587 FOCAL_IMAGE_NAME.to_string() 1588 }; 1589 #[cfg(target_arch = "x86_64")] 1590 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1591 let focal = UbuntuDiskConfig::new(focal_image); 1592 let guest = Guest::new(Box::new(focal)); 1593 let api_socket = temp_api_path(&guest.tmp_dir); 1594 1595 let mut workload_path = dirs::home_dir().unwrap(); 1596 workload_path.push("workloads"); 1597 1598 let mut shared_dir = workload_path; 1599 shared_dir.push("shared_dir"); 1600 1601 #[cfg(target_arch = "x86_64")] 1602 let kernel_path = direct_kernel_boot_path(); 1603 #[cfg(target_arch = "aarch64")] 1604 let kernel_path = if hotplug { 1605 edk2_path() 1606 } else { 1607 direct_kernel_boot_path() 1608 }; 1609 1610 let (mut daemon_child, virtiofsd_socket_path) = 1611 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1612 1613 let mut guest_command = GuestCommand::new(&guest); 1614 guest_command 1615 .args(["--cpus", "boot=1"]) 1616 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1617 .args(["--kernel", kernel_path.to_str().unwrap()]) 1618 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1619 .default_disks() 1620 .default_net() 1621 .args(["--api-socket", &api_socket]); 1622 if pci_segment.is_some() { 1623 guest_command.args([ 1624 "--platform", 1625 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 1626 ]); 1627 } 1628 1629 let fs_params = format!( 1630 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1631 virtiofsd_socket_path, 1632 if let Some(pci_segment) = pci_segment { 1633 format!(",pci_segment={pci_segment}") 1634 } else { 1635 "".to_owned() 1636 } 1637 ); 1638 1639 if !hotplug { 1640 guest_command.args(["--fs", fs_params.as_str()]); 1641 } 1642 1643 let mut child = guest_command.capture_output().spawn().unwrap(); 1644 1645 let r = std::panic::catch_unwind(|| { 1646 guest.wait_vm_boot(None).unwrap(); 1647 1648 if hotplug { 1649 // Add fs to the VM 1650 let (cmd_success, cmd_output) = 1651 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1652 assert!(cmd_success); 1653 1654 if let Some(pci_segment) = pci_segment { 1655 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1656 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1657 ))); 1658 } else { 1659 assert!(String::from_utf8_lossy(&cmd_output) 1660 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1661 } 1662 1663 thread::sleep(std::time::Duration::new(10, 0)); 1664 } 1665 1666 // Mount shared directory through virtio_fs filesystem 1667 guest 1668 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1669 .unwrap(); 1670 1671 // Check file1 exists and its content is "foo" 1672 assert_eq!( 1673 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1674 "foo" 1675 ); 1676 // Check file2 does not exist 1677 guest 1678 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1679 .unwrap(); 1680 1681 // Check file3 exists and its content is "bar" 1682 assert_eq!( 1683 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1684 "bar" 1685 ); 1686 1687 // ACPI feature is needed. 1688 #[cfg(target_arch = "x86_64")] 1689 { 1690 guest.enable_memory_hotplug(); 1691 1692 // Add RAM to the VM 1693 let desired_ram = 1024 << 20; 1694 resize_command(&api_socket, None, Some(desired_ram), None, None); 1695 1696 thread::sleep(std::time::Duration::new(30, 0)); 1697 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1698 1699 // After the resize, check again that file1 exists and its 1700 // content is "foo". 1701 assert_eq!( 1702 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1703 "foo" 1704 ); 1705 } 1706 1707 if hotplug { 1708 // Remove from VM 1709 guest.ssh_command("sudo umount mount_dir").unwrap(); 1710 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1711 } 1712 }); 1713 1714 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1715 thread::sleep(std::time::Duration::new(10, 0)); 1716 let (daemon_child, virtiofsd_socket_path) = 1717 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1718 1719 let r = std::panic::catch_unwind(|| { 1720 thread::sleep(std::time::Duration::new(10, 0)); 1721 let fs_params = format!( 1722 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1723 virtiofsd_socket_path, 1724 if let Some(pci_segment) = pci_segment { 1725 format!(",pci_segment={pci_segment}") 1726 } else { 1727 "".to_owned() 1728 } 1729 ); 1730 1731 // Add back and check it works 1732 let (cmd_success, cmd_output) = 1733 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1734 assert!(cmd_success); 1735 if let Some(pci_segment) = pci_segment { 1736 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1737 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1738 ))); 1739 } else { 1740 assert!(String::from_utf8_lossy(&cmd_output) 1741 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1742 } 1743 1744 thread::sleep(std::time::Duration::new(10, 0)); 1745 // Mount shared directory through virtio_fs filesystem 1746 guest 1747 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1748 .unwrap(); 1749 1750 // Check file1 exists and its content is "foo" 1751 assert_eq!( 1752 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1753 "foo" 1754 ); 1755 }); 1756 1757 (r, Some(daemon_child)) 1758 } else { 1759 (r, None) 1760 }; 1761 1762 kill_child(&mut child); 1763 let output = child.wait_with_output().unwrap(); 1764 1765 let _ = daemon_child.kill(); 1766 let _ = daemon_child.wait(); 1767 1768 if let Some(mut daemon_child) = hotplug_daemon_child { 1769 let _ = daemon_child.kill(); 1770 let _ = daemon_child.wait(); 1771 } 1772 1773 handle_child_output(r, &output); 1774 } 1775 1776 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1777 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1778 let guest = Guest::new(Box::new(focal)); 1779 1780 let kernel_path = direct_kernel_boot_path(); 1781 1782 let pmem_temp_file = TempFile::new().unwrap(); 1783 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1784 1785 std::process::Command::new("mkfs.ext4") 1786 .arg(pmem_temp_file.as_path()) 1787 .output() 1788 .expect("Expect creating disk image to succeed"); 1789 1790 let mut child = GuestCommand::new(&guest) 1791 .args(["--cpus", "boot=1"]) 1792 .args(["--memory", "size=512M"]) 1793 .args(["--kernel", kernel_path.to_str().unwrap()]) 1794 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1795 .default_disks() 1796 .default_net() 1797 .args([ 1798 "--pmem", 1799 format!( 1800 "file={}{}{}", 1801 pmem_temp_file.as_path().to_str().unwrap(), 1802 if specify_size { ",size=128M" } else { "" }, 1803 if discard_writes { 1804 ",discard_writes=on" 1805 } else { 1806 "" 1807 } 1808 ) 1809 .as_str(), 1810 ]) 1811 .capture_output() 1812 .spawn() 1813 .unwrap(); 1814 1815 let r = std::panic::catch_unwind(|| { 1816 guest.wait_vm_boot(None).unwrap(); 1817 1818 // Check for the presence of /dev/pmem0 1819 assert_eq!( 1820 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1821 "/dev/pmem0" 1822 ); 1823 1824 // Check changes persist after reboot 1825 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1826 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1827 guest 1828 .ssh_command("echo test123 | sudo tee /mnt/test") 1829 .unwrap(); 1830 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1831 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1832 1833 guest.reboot_linux(0, None); 1834 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1835 assert_eq!( 1836 guest 1837 .ssh_command("sudo cat /mnt/test || true") 1838 .unwrap() 1839 .trim(), 1840 if discard_writes { "" } else { "test123" } 1841 ); 1842 }); 1843 1844 kill_child(&mut child); 1845 let output = child.wait_with_output().unwrap(); 1846 1847 handle_child_output(r, &output); 1848 } 1849 1850 fn get_fd_count(pid: u32) -> usize { 1851 fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() 1852 } 1853 1854 fn _test_virtio_vsock(hotplug: bool) { 1855 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1856 let guest = Guest::new(Box::new(focal)); 1857 1858 #[cfg(target_arch = "x86_64")] 1859 let kernel_path = direct_kernel_boot_path(); 1860 #[cfg(target_arch = "aarch64")] 1861 let kernel_path = if hotplug { 1862 edk2_path() 1863 } else { 1864 direct_kernel_boot_path() 1865 }; 1866 1867 let socket = temp_vsock_path(&guest.tmp_dir); 1868 let api_socket = temp_api_path(&guest.tmp_dir); 1869 1870 let mut cmd = GuestCommand::new(&guest); 1871 cmd.args(["--api-socket", &api_socket]); 1872 cmd.args(["--cpus", "boot=1"]); 1873 cmd.args(["--memory", "size=512M"]); 1874 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1875 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1876 cmd.default_disks(); 1877 cmd.default_net(); 1878 1879 if !hotplug { 1880 cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); 1881 } 1882 1883 let mut child = cmd.capture_output().spawn().unwrap(); 1884 1885 let r = std::panic::catch_unwind(|| { 1886 guest.wait_vm_boot(None).unwrap(); 1887 1888 if hotplug { 1889 let (cmd_success, cmd_output) = remote_command_w_output( 1890 &api_socket, 1891 "add-vsock", 1892 Some(format!("cid=3,socket={socket},id=test0").as_str()), 1893 ); 1894 assert!(cmd_success); 1895 assert!(String::from_utf8_lossy(&cmd_output) 1896 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1897 thread::sleep(std::time::Duration::new(10, 0)); 1898 // Check adding a second one fails 1899 assert!(!remote_command( 1900 &api_socket, 1901 "add-vsock", 1902 Some("cid=1234,socket=/tmp/fail") 1903 )); 1904 } 1905 1906 // Validate vsock works as expected. 1907 guest.check_vsock(socket.as_str()); 1908 guest.reboot_linux(0, None); 1909 // Validate vsock still works after a reboot. 1910 guest.check_vsock(socket.as_str()); 1911 1912 if hotplug { 1913 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1914 } 1915 }); 1916 1917 kill_child(&mut child); 1918 let output = child.wait_with_output().unwrap(); 1919 1920 handle_child_output(r, &output); 1921 } 1922 1923 fn get_ksm_pages_shared() -> u32 { 1924 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1925 .unwrap() 1926 .trim() 1927 .parse::<u32>() 1928 .unwrap() 1929 } 1930 1931 fn test_memory_mergeable(mergeable: bool) { 1932 let memory_param = if mergeable { 1933 "mergeable=on" 1934 } else { 1935 "mergeable=off" 1936 }; 1937 1938 // We are assuming the rest of the system in our CI is not using mergeable memory 1939 let ksm_ps_init = get_ksm_pages_shared(); 1940 assert!(ksm_ps_init == 0); 1941 1942 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1943 let guest1 = Guest::new(Box::new(focal1)); 1944 let mut child1 = GuestCommand::new(&guest1) 1945 .args(["--cpus", "boot=1"]) 1946 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1947 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1948 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1949 .default_disks() 1950 .args(["--net", guest1.default_net_string().as_str()]) 1951 .args(["--serial", "tty", "--console", "off"]) 1952 .capture_output() 1953 .spawn() 1954 .unwrap(); 1955 1956 let r = std::panic::catch_unwind(|| { 1957 guest1.wait_vm_boot(None).unwrap(); 1958 }); 1959 if r.is_err() { 1960 kill_child(&mut child1); 1961 let output = child1.wait_with_output().unwrap(); 1962 handle_child_output(r, &output); 1963 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1964 } 1965 1966 let ksm_ps_guest1 = get_ksm_pages_shared(); 1967 1968 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1969 let guest2 = Guest::new(Box::new(focal2)); 1970 let mut child2 = GuestCommand::new(&guest2) 1971 .args(["--cpus", "boot=1"]) 1972 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1973 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1974 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1975 .default_disks() 1976 .args(["--net", guest2.default_net_string().as_str()]) 1977 .args(["--serial", "tty", "--console", "off"]) 1978 .capture_output() 1979 .spawn() 1980 .unwrap(); 1981 1982 let r = std::panic::catch_unwind(|| { 1983 guest2.wait_vm_boot(None).unwrap(); 1984 let ksm_ps_guest2 = get_ksm_pages_shared(); 1985 1986 if mergeable { 1987 println!( 1988 "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" 1989 ); 1990 // We are expecting the number of shared pages to increase as the number of VM increases 1991 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1992 } else { 1993 assert!(ksm_ps_guest1 == 0); 1994 assert!(ksm_ps_guest2 == 0); 1995 } 1996 }); 1997 1998 kill_child(&mut child1); 1999 kill_child(&mut child2); 2000 2001 let output = child1.wait_with_output().unwrap(); 2002 child2.wait().unwrap(); 2003 2004 handle_child_output(r, &output); 2005 } 2006 2007 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 2008 let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); 2009 let reader = io::BufReader::new(smaps); 2010 2011 let mut skip_map: bool = false; 2012 let mut region_name: String = "".to_string(); 2013 let mut region_maps = HashMap::new(); 2014 for line in reader.lines() { 2015 let l = line.unwrap(); 2016 2017 if l.contains('-') { 2018 let values: Vec<&str> = l.split_whitespace().collect(); 2019 region_name = values.last().unwrap().trim().to_string(); 2020 if region_name == "0" { 2021 region_name = "anonymous".to_string() 2022 } 2023 } 2024 2025 // Each section begins with something that looks like: 2026 // Size: 2184 kB 2027 if l.starts_with("Size:") { 2028 let values: Vec<&str> = l.split_whitespace().collect(); 2029 let map_size = values[1].parse::<u32>().unwrap(); 2030 // We skip the assigned guest RAM map, its RSS is only 2031 // dependent on the guest actual memory usage. 2032 // Everything else can be added to the VMM overhead. 2033 skip_map = map_size >= guest_memory_size; 2034 continue; 2035 } 2036 2037 // If this is a map we're taking into account, then we only 2038 // count the RSS. The sum of all counted RSS is the VMM overhead. 2039 if !skip_map && l.starts_with("Rss:") { 2040 let values: Vec<&str> = l.split_whitespace().collect(); 2041 let value = values[1].trim().parse::<u32>().unwrap(); 2042 *region_maps.entry(region_name.clone()).or_insert(0) += value; 2043 } 2044 } 2045 2046 region_maps 2047 } 2048 2049 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 2050 let mut total = 0; 2051 2052 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 2053 eprintln!("{region_name}: {value}"); 2054 total += value; 2055 } 2056 2057 total 2058 } 2059 2060 fn process_rss_kib(pid: u32) -> usize { 2061 let command = format!("ps -q {pid} -o rss="); 2062 let rss = exec_host_command_output(&command); 2063 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 2064 } 2065 2066 // 10MB is our maximum accepted overhead. 2067 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 2068 2069 #[derive(PartialEq, Eq, PartialOrd)] 2070 struct Counters { 2071 rx_bytes: u64, 2072 rx_frames: u64, 2073 tx_bytes: u64, 2074 tx_frames: u64, 2075 read_bytes: u64, 2076 write_bytes: u64, 2077 read_ops: u64, 2078 write_ops: u64, 2079 } 2080 2081 fn get_counters(api_socket: &str) -> Counters { 2082 // Get counters 2083 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 2084 assert!(cmd_success); 2085 2086 let counters: HashMap<&str, HashMap<&str, u64>> = 2087 serde_json::from_slice(&cmd_output).unwrap_or_default(); 2088 2089 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 2090 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 2091 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 2092 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 2093 2094 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 2095 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 2096 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 2097 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 2098 2099 Counters { 2100 rx_bytes, 2101 rx_frames, 2102 tx_bytes, 2103 tx_frames, 2104 read_bytes, 2105 write_bytes, 2106 read_ops, 2107 write_ops, 2108 } 2109 } 2110 2111 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 2112 let (tx, rx) = mpsc::channel::<String>(); 2113 thread::spawn(move || loop { 2114 thread::sleep(std::time::Duration::new(1, 0)); 2115 let mut buf = [0; 512]; 2116 match pty.read(&mut buf) { 2117 Ok(_bytes) => { 2118 let output = std::str::from_utf8(&buf).unwrap().to_string(); 2119 match tx.send(output) { 2120 Ok(_) => (), 2121 Err(_) => break, 2122 } 2123 } 2124 Err(_) => break, 2125 } 2126 }); 2127 rx 2128 } 2129 2130 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 2131 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2132 assert!(cmd_success); 2133 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2134 assert_eq!("Pty", info["config"][pty_type]["mode"]); 2135 PathBuf::from( 2136 info["config"][pty_type]["file"] 2137 .as_str() 2138 .expect("Missing pty path"), 2139 ) 2140 } 2141 2142 // VFIO test network setup. 2143 // We reserve a different IP class for it: 172.18.0.0/24. 2144 #[cfg(target_arch = "x86_64")] 2145 fn setup_vfio_network_interfaces() { 2146 // 'vfio-br0' 2147 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 2148 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 2149 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 2150 // 'vfio-tap0' 2151 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 2152 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 2153 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 2154 // 'vfio-tap1' 2155 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 2156 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 2157 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 2158 // 'vfio-tap2' 2159 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 2160 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 2161 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 2162 // 'vfio-tap3' 2163 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 2164 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 2165 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 2166 } 2167 2168 // Tear VFIO test network down 2169 #[cfg(target_arch = "x86_64")] 2170 fn cleanup_vfio_network_interfaces() { 2171 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 2172 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 2173 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 2174 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 2175 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 2176 } 2177 2178 fn balloon_size(api_socket: &str) -> u64 { 2179 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2180 assert!(cmd_success); 2181 2182 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2183 let total_mem = &info["config"]["memory"]["size"] 2184 .to_string() 2185 .parse::<u64>() 2186 .unwrap(); 2187 let actual_mem = &info["memory_actual_size"] 2188 .to_string() 2189 .parse::<u64>() 2190 .unwrap(); 2191 total_mem - actual_mem 2192 } 2193 2194 fn vm_state(api_socket: &str) -> String { 2195 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2196 assert!(cmd_success); 2197 2198 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2199 let state = &info["state"].as_str().unwrap(); 2200 2201 state.to_string() 2202 } 2203 2204 // This test validates that it can find the virtio-iommu device at first. 2205 // It also verifies that both disks and the network card are attached to 2206 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 2207 // The last interesting part of this test is that it exercises the network 2208 // interface attached to the virtual IOMMU since this is the one used to 2209 // send all commands through SSH. 2210 fn _test_virtio_iommu(acpi: bool) { 2211 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 2212 // Focal image is still old. 2213 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 2214 // the kernel binary has been updated. 2215 #[cfg(target_arch = "aarch64")] 2216 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 2217 #[cfg(target_arch = "x86_64")] 2218 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2219 let focal = UbuntuDiskConfig::new(focal_image); 2220 let guest = Guest::new(Box::new(focal)); 2221 2222 #[cfg(target_arch = "x86_64")] 2223 let kernel_path = direct_kernel_boot_path(); 2224 #[cfg(target_arch = "aarch64")] 2225 let kernel_path = if acpi { 2226 edk2_path() 2227 } else { 2228 direct_kernel_boot_path() 2229 }; 2230 2231 let mut child = GuestCommand::new(&guest) 2232 .args(["--cpus", "boot=1"]) 2233 .args(["--memory", "size=512M"]) 2234 .args(["--kernel", kernel_path.to_str().unwrap()]) 2235 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2236 .args([ 2237 "--disk", 2238 format!( 2239 "path={},iommu=on", 2240 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2241 ) 2242 .as_str(), 2243 format!( 2244 "path={},iommu=on", 2245 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2246 ) 2247 .as_str(), 2248 ]) 2249 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 2250 .capture_output() 2251 .spawn() 2252 .unwrap(); 2253 2254 let r = std::panic::catch_unwind(|| { 2255 guest.wait_vm_boot(None).unwrap(); 2256 2257 // Verify the virtio-iommu device is present. 2258 assert!(guest 2259 .does_device_vendor_pair_match("0x1057", "0x1af4") 2260 .unwrap_or_default()); 2261 2262 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 2263 // different with ACPI. 2264 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 2265 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 2266 // and appear under folder '/sys/kernel/iommu_groups/'. 2267 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 2268 // which is the console. The first disk "0000:00:02.0" is in group '1'. 2269 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 2270 // contains "0000:00:02.0" which is the first disk. 2271 // 2272 // Verify the iommu group of the first disk. 2273 let iommu_group = !acpi as i32; 2274 assert_eq!( 2275 guest 2276 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2277 .unwrap() 2278 .trim(), 2279 "0000:00:02.0" 2280 ); 2281 2282 // Verify the iommu group of the second disk. 2283 let iommu_group = if acpi { 1 } else { 2 }; 2284 assert_eq!( 2285 guest 2286 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2287 .unwrap() 2288 .trim(), 2289 "0000:00:03.0" 2290 ); 2291 2292 // Verify the iommu group of the network card. 2293 let iommu_group = if acpi { 2 } else { 3 }; 2294 assert_eq!( 2295 guest 2296 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2297 .unwrap() 2298 .trim(), 2299 "0000:00:04.0" 2300 ); 2301 }); 2302 2303 kill_child(&mut child); 2304 let output = child.wait_with_output().unwrap(); 2305 2306 handle_child_output(r, &output); 2307 } 2308 2309 fn get_reboot_count(guest: &Guest) -> u32 { 2310 guest 2311 .ssh_command("sudo last | grep -c reboot") 2312 .unwrap() 2313 .trim() 2314 .parse::<u32>() 2315 .unwrap_or_default() 2316 } 2317 2318 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 2319 // Check for PCI device 2320 assert!(guest 2321 .does_device_vendor_pair_match("0x1063", "0x1af4") 2322 .unwrap_or_default()); 2323 2324 // Enable systemd watchdog 2325 guest 2326 .ssh_command(&format!( 2327 "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" 2328 )) 2329 .unwrap(); 2330 2331 guest.ssh_command("sudo systemctl daemon-reexec").unwrap(); 2332 } 2333 2334 fn make_guest_panic(guest: &Guest) { 2335 // Check for pvpanic device 2336 assert!(guest 2337 .does_device_vendor_pair_match("0x0011", "0x1b36") 2338 .unwrap_or_default()); 2339 2340 // Trigger guest a panic 2341 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 2342 } 2343 2344 mod common_parallel { 2345 use std::fs::OpenOptions; 2346 use std::io::SeekFrom; 2347 2348 use crate::*; 2349 2350 #[test] 2351 #[cfg(target_arch = "x86_64")] 2352 fn test_focal_hypervisor_fw() { 2353 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 2354 } 2355 2356 #[test] 2357 #[cfg(target_arch = "x86_64")] 2358 fn test_focal_ovmf() { 2359 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 2360 } 2361 2362 #[cfg(target_arch = "x86_64")] 2363 fn test_simple_launch(fw_path: String, disk_path: &str) { 2364 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 2365 let guest = Guest::new(disk_config); 2366 let event_path = temp_event_monitor_path(&guest.tmp_dir); 2367 2368 let mut child = GuestCommand::new(&guest) 2369 .args(["--cpus", "boot=1"]) 2370 .args(["--memory", "size=512M"]) 2371 .args(["--kernel", fw_path.as_str()]) 2372 .default_disks() 2373 .default_net() 2374 .args(["--serial", "tty", "--console", "off"]) 2375 .args(["--event-monitor", format!("path={event_path}").as_str()]) 2376 .capture_output() 2377 .spawn() 2378 .unwrap(); 2379 2380 let r = std::panic::catch_unwind(|| { 2381 guest.wait_vm_boot(Some(120)).unwrap(); 2382 2383 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2384 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2385 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2386 2387 let expected_sequential_events = [ 2388 &MetaEvent { 2389 event: "starting".to_string(), 2390 device_id: None, 2391 }, 2392 &MetaEvent { 2393 event: "booting".to_string(), 2394 device_id: None, 2395 }, 2396 &MetaEvent { 2397 event: "booted".to_string(), 2398 device_id: None, 2399 }, 2400 &MetaEvent { 2401 event: "activated".to_string(), 2402 device_id: Some("_disk0".to_string()), 2403 }, 2404 &MetaEvent { 2405 event: "reset".to_string(), 2406 device_id: Some("_disk0".to_string()), 2407 }, 2408 ]; 2409 assert!(check_sequential_events( 2410 &expected_sequential_events, 2411 &event_path 2412 )); 2413 2414 // It's been observed on the Bionic image that udev and snapd 2415 // services can cause some delay in the VM's shutdown. Disabling 2416 // them improves the reliability of this test. 2417 let _ = guest.ssh_command("sudo systemctl disable udev"); 2418 let _ = guest.ssh_command("sudo systemctl stop udev"); 2419 let _ = guest.ssh_command("sudo systemctl disable snapd"); 2420 let _ = guest.ssh_command("sudo systemctl stop snapd"); 2421 2422 guest.ssh_command("sudo poweroff").unwrap(); 2423 thread::sleep(std::time::Duration::new(20, 0)); 2424 let latest_events = [ 2425 &MetaEvent { 2426 event: "shutdown".to_string(), 2427 device_id: None, 2428 }, 2429 &MetaEvent { 2430 event: "deleted".to_string(), 2431 device_id: None, 2432 }, 2433 &MetaEvent { 2434 event: "shutdown".to_string(), 2435 device_id: None, 2436 }, 2437 ]; 2438 assert!(check_latest_events_exact(&latest_events, &event_path)); 2439 }); 2440 2441 kill_child(&mut child); 2442 let output = child.wait_with_output().unwrap(); 2443 2444 handle_child_output(r, &output); 2445 } 2446 2447 #[test] 2448 fn test_multi_cpu() { 2449 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2450 let jammy = UbuntuDiskConfig::new(jammy_image); 2451 let guest = Guest::new(Box::new(jammy)); 2452 2453 let mut cmd = GuestCommand::new(&guest); 2454 cmd.args(["--cpus", "boot=2,max=4"]) 2455 .args(["--memory", "size=512M"]) 2456 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2457 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2458 .capture_output() 2459 .default_disks() 2460 .default_net(); 2461 2462 let mut child = cmd.spawn().unwrap(); 2463 2464 let r = std::panic::catch_unwind(|| { 2465 guest.wait_vm_boot(Some(120)).unwrap(); 2466 2467 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2468 2469 assert_eq!( 2470 guest 2471 .ssh_command( 2472 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2473 ) 2474 .unwrap() 2475 .trim(), 2476 "smp: Brought up 1 node, 2 CPUs" 2477 ); 2478 }); 2479 2480 kill_child(&mut child); 2481 let output = child.wait_with_output().unwrap(); 2482 2483 handle_child_output(r, &output); 2484 } 2485 2486 #[test] 2487 fn test_cpu_topology_421() { 2488 test_cpu_topology(4, 2, 1, false); 2489 } 2490 2491 #[test] 2492 fn test_cpu_topology_142() { 2493 test_cpu_topology(1, 4, 2, false); 2494 } 2495 2496 #[test] 2497 fn test_cpu_topology_262() { 2498 test_cpu_topology(2, 6, 2, false); 2499 } 2500 2501 #[test] 2502 #[cfg(target_arch = "x86_64")] 2503 #[cfg(not(feature = "mshv"))] 2504 fn test_cpu_physical_bits() { 2505 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2506 let guest = Guest::new(Box::new(focal)); 2507 let max_phys_bits: u8 = 36; 2508 let mut child = GuestCommand::new(&guest) 2509 .args(["--cpus", &format!("max_phys_bits={max_phys_bits}")]) 2510 .args(["--memory", "size=512M"]) 2511 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2512 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2513 .default_disks() 2514 .default_net() 2515 .capture_output() 2516 .spawn() 2517 .unwrap(); 2518 2519 let r = std::panic::catch_unwind(|| { 2520 guest.wait_vm_boot(None).unwrap(); 2521 2522 assert!( 2523 guest 2524 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2525 .unwrap() 2526 .trim() 2527 .parse::<u8>() 2528 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2529 ); 2530 }); 2531 2532 kill_child(&mut child); 2533 let output = child.wait_with_output().unwrap(); 2534 2535 handle_child_output(r, &output); 2536 } 2537 2538 #[test] 2539 fn test_cpu_affinity() { 2540 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2541 let guest = Guest::new(Box::new(focal)); 2542 2543 // We need the host to have at least 4 CPUs if we want to be able 2544 // to run this test. 2545 let host_cpus_count = exec_host_command_output("nproc"); 2546 assert!( 2547 String::from_utf8_lossy(&host_cpus_count.stdout) 2548 .trim() 2549 .parse::<u16>() 2550 .unwrap_or(0) 2551 >= 4 2552 ); 2553 2554 let mut child = GuestCommand::new(&guest) 2555 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2556 .args(["--memory", "size=512M"]) 2557 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2558 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2559 .default_disks() 2560 .default_net() 2561 .capture_output() 2562 .spawn() 2563 .unwrap(); 2564 2565 let r = std::panic::catch_unwind(|| { 2566 guest.wait_vm_boot(None).unwrap(); 2567 let pid = child.id(); 2568 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2569 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2570 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2571 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2572 }); 2573 2574 kill_child(&mut child); 2575 let output = child.wait_with_output().unwrap(); 2576 handle_child_output(r, &output); 2577 } 2578 2579 #[test] 2580 fn test_virtio_queue_affinity() { 2581 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2582 let guest = Guest::new(Box::new(focal)); 2583 2584 // We need the host to have at least 4 CPUs if we want to be able 2585 // to run this test. 2586 let host_cpus_count = exec_host_command_output("nproc"); 2587 assert!( 2588 String::from_utf8_lossy(&host_cpus_count.stdout) 2589 .trim() 2590 .parse::<u16>() 2591 .unwrap_or(0) 2592 >= 4 2593 ); 2594 2595 let mut child = GuestCommand::new(&guest) 2596 .args(["--cpus", "boot=4"]) 2597 .args(["--memory", "size=512M"]) 2598 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2599 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2600 .args([ 2601 "--disk", 2602 format!( 2603 "path={}", 2604 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2605 ) 2606 .as_str(), 2607 format!( 2608 "path={},num_queues=4,queue_affinity=[0@[0,2],1@[1,3],2@[1],3@[3]]", 2609 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2610 ) 2611 .as_str(), 2612 ]) 2613 .default_net() 2614 .capture_output() 2615 .spawn() 2616 .unwrap(); 2617 2618 let r = std::panic::catch_unwind(|| { 2619 guest.wait_vm_boot(None).unwrap(); 2620 let pid = child.id(); 2621 let taskset_q0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2622 assert_eq!(String::from_utf8_lossy(&taskset_q0.stdout).trim(), "0,2"); 2623 let taskset_q1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2624 assert_eq!(String::from_utf8_lossy(&taskset_q1.stdout).trim(), "1,3"); 2625 let taskset_q2 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q2 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2626 assert_eq!(String::from_utf8_lossy(&taskset_q2.stdout).trim(), "1"); 2627 let taskset_q3 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q3 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2628 assert_eq!(String::from_utf8_lossy(&taskset_q3.stdout).trim(), "3"); 2629 }); 2630 2631 kill_child(&mut child); 2632 let output = child.wait_with_output().unwrap(); 2633 handle_child_output(r, &output); 2634 } 2635 2636 #[test] 2637 #[cfg(not(feature = "mshv"))] 2638 fn test_large_vm() { 2639 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2640 let guest = Guest::new(Box::new(focal)); 2641 let mut cmd = GuestCommand::new(&guest); 2642 cmd.args(["--cpus", "boot=48"]) 2643 .args(["--memory", "size=5120M"]) 2644 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2645 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2646 .args(["--serial", "tty"]) 2647 .args(["--console", "off"]) 2648 .capture_output() 2649 .default_disks() 2650 .default_net(); 2651 2652 let mut child = cmd.spawn().unwrap(); 2653 2654 guest.wait_vm_boot(None).unwrap(); 2655 2656 let r = std::panic::catch_unwind(|| { 2657 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2658 assert_eq!( 2659 guest 2660 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2661 .unwrap() 2662 .trim(), 2663 "0-47" 2664 ); 2665 2666 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2667 }); 2668 2669 kill_child(&mut child); 2670 let output = child.wait_with_output().unwrap(); 2671 2672 handle_child_output(r, &output); 2673 } 2674 2675 #[test] 2676 #[cfg(not(feature = "mshv"))] 2677 fn test_huge_memory() { 2678 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2679 let guest = Guest::new(Box::new(focal)); 2680 let mut cmd = GuestCommand::new(&guest); 2681 cmd.args(["--cpus", "boot=1"]) 2682 .args(["--memory", "size=128G"]) 2683 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2684 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2685 .capture_output() 2686 .default_disks() 2687 .default_net(); 2688 2689 let mut child = cmd.spawn().unwrap(); 2690 2691 guest.wait_vm_boot(Some(120)).unwrap(); 2692 2693 let r = std::panic::catch_unwind(|| { 2694 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2695 }); 2696 2697 kill_child(&mut child); 2698 let output = child.wait_with_output().unwrap(); 2699 2700 handle_child_output(r, &output); 2701 } 2702 2703 #[test] 2704 fn test_power_button() { 2705 _test_power_button(false); 2706 } 2707 2708 #[test] 2709 #[cfg(not(feature = "mshv"))] 2710 fn test_user_defined_memory_regions() { 2711 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2712 let guest = Guest::new(Box::new(focal)); 2713 let api_socket = temp_api_path(&guest.tmp_dir); 2714 2715 let kernel_path = direct_kernel_boot_path(); 2716 2717 let mut child = GuestCommand::new(&guest) 2718 .args(["--cpus", "boot=1"]) 2719 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2720 .args([ 2721 "--memory-zone", 2722 "id=mem0,size=1G,hotplug_size=2G", 2723 "id=mem1,size=1G,shared=on", 2724 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2725 ]) 2726 .args(["--kernel", kernel_path.to_str().unwrap()]) 2727 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2728 .args(["--api-socket", &api_socket]) 2729 .capture_output() 2730 .default_disks() 2731 .default_net() 2732 .spawn() 2733 .unwrap(); 2734 2735 let r = std::panic::catch_unwind(|| { 2736 guest.wait_vm_boot(None).unwrap(); 2737 2738 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2739 2740 guest.enable_memory_hotplug(); 2741 2742 resize_zone_command(&api_socket, "mem0", "3G"); 2743 thread::sleep(std::time::Duration::new(5, 0)); 2744 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2745 resize_zone_command(&api_socket, "mem2", "3G"); 2746 thread::sleep(std::time::Duration::new(5, 0)); 2747 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2748 resize_zone_command(&api_socket, "mem0", "2G"); 2749 thread::sleep(std::time::Duration::new(5, 0)); 2750 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2751 resize_zone_command(&api_socket, "mem2", "2G"); 2752 thread::sleep(std::time::Duration::new(5, 0)); 2753 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2754 2755 guest.reboot_linux(0, None); 2756 2757 // Check the amount of RAM after reboot 2758 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2759 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2760 2761 // Check if we can still resize down to the initial 'boot'size 2762 resize_zone_command(&api_socket, "mem0", "1G"); 2763 thread::sleep(std::time::Duration::new(5, 0)); 2764 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2765 resize_zone_command(&api_socket, "mem2", "1G"); 2766 thread::sleep(std::time::Duration::new(5, 0)); 2767 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2768 }); 2769 2770 kill_child(&mut child); 2771 let output = child.wait_with_output().unwrap(); 2772 2773 handle_child_output(r, &output); 2774 } 2775 2776 #[test] 2777 #[cfg(not(feature = "mshv"))] 2778 fn test_guest_numa_nodes() { 2779 _test_guest_numa_nodes(false); 2780 } 2781 2782 #[test] 2783 #[cfg(target_arch = "x86_64")] 2784 fn test_iommu_segments() { 2785 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2786 let focal = UbuntuDiskConfig::new(focal_image); 2787 let guest = Guest::new(Box::new(focal)); 2788 2789 // Prepare another disk file for the virtio-disk device 2790 let test_disk_path = String::from( 2791 guest 2792 .tmp_dir 2793 .as_path() 2794 .join("test-disk.raw") 2795 .to_str() 2796 .unwrap(), 2797 ); 2798 assert!( 2799 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2800 ); 2801 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2802 2803 let api_socket = temp_api_path(&guest.tmp_dir); 2804 let mut cmd = GuestCommand::new(&guest); 2805 2806 cmd.args(["--cpus", "boot=1"]) 2807 .args(["--api-socket", &api_socket]) 2808 .args(["--memory", "size=512M"]) 2809 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2810 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2811 .args([ 2812 "--platform", 2813 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS},iommu_segments=[1]"), 2814 ]) 2815 .default_disks() 2816 .capture_output() 2817 .default_net(); 2818 2819 let mut child = cmd.spawn().unwrap(); 2820 2821 guest.wait_vm_boot(None).unwrap(); 2822 2823 let r = std::panic::catch_unwind(|| { 2824 let (cmd_success, cmd_output) = remote_command_w_output( 2825 &api_socket, 2826 "add-disk", 2827 Some( 2828 format!( 2829 "path={},id=test0,pci_segment=1,iommu=on", 2830 test_disk_path.as_str() 2831 ) 2832 .as_str(), 2833 ), 2834 ); 2835 assert!(cmd_success); 2836 assert!(String::from_utf8_lossy(&cmd_output) 2837 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2838 2839 // Check IOMMU setup 2840 assert!(guest 2841 .does_device_vendor_pair_match("0x1057", "0x1af4") 2842 .unwrap_or_default()); 2843 assert_eq!( 2844 guest 2845 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2846 .unwrap() 2847 .trim(), 2848 "0001:00:01.0" 2849 ); 2850 }); 2851 2852 kill_child(&mut child); 2853 let output = child.wait_with_output().unwrap(); 2854 2855 handle_child_output(r, &output); 2856 } 2857 2858 #[test] 2859 fn test_pci_msi() { 2860 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2861 let guest = Guest::new(Box::new(focal)); 2862 let mut cmd = GuestCommand::new(&guest); 2863 cmd.args(["--cpus", "boot=1"]) 2864 .args(["--memory", "size=512M"]) 2865 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2866 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2867 .capture_output() 2868 .default_disks() 2869 .default_net(); 2870 2871 let mut child = cmd.spawn().unwrap(); 2872 2873 guest.wait_vm_boot(None).unwrap(); 2874 2875 #[cfg(target_arch = "x86_64")] 2876 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2877 #[cfg(target_arch = "aarch64")] 2878 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2879 2880 let r = std::panic::catch_unwind(|| { 2881 assert_eq!( 2882 guest 2883 .ssh_command(grep_cmd) 2884 .unwrap() 2885 .trim() 2886 .parse::<u32>() 2887 .unwrap_or_default(), 2888 12 2889 ); 2890 }); 2891 2892 kill_child(&mut child); 2893 let output = child.wait_with_output().unwrap(); 2894 2895 handle_child_output(r, &output); 2896 } 2897 2898 #[test] 2899 fn test_virtio_net_ctrl_queue() { 2900 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2901 let guest = Guest::new(Box::new(focal)); 2902 let mut cmd = GuestCommand::new(&guest); 2903 cmd.args(["--cpus", "boot=1"]) 2904 .args(["--memory", "size=512M"]) 2905 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2906 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2907 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2908 .capture_output() 2909 .default_disks(); 2910 2911 let mut child = cmd.spawn().unwrap(); 2912 2913 guest.wait_vm_boot(None).unwrap(); 2914 2915 #[cfg(target_arch = "aarch64")] 2916 let iface = "enp0s4"; 2917 #[cfg(target_arch = "x86_64")] 2918 let iface = "ens4"; 2919 2920 let r = std::panic::catch_unwind(|| { 2921 assert_eq!( 2922 guest 2923 .ssh_command( 2924 format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() 2925 ) 2926 .unwrap() 2927 .trim(), 2928 "success" 2929 ); 2930 assert_eq!( 2931 guest 2932 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 2933 .unwrap() 2934 .trim(), 2935 "3000" 2936 ); 2937 }); 2938 2939 kill_child(&mut child); 2940 let output = child.wait_with_output().unwrap(); 2941 2942 handle_child_output(r, &output); 2943 } 2944 2945 #[test] 2946 fn test_pci_multiple_segments() { 2947 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2948 let guest = Guest::new(Box::new(focal)); 2949 2950 // Prepare another disk file for the virtio-disk device 2951 let test_disk_path = String::from( 2952 guest 2953 .tmp_dir 2954 .as_path() 2955 .join("test-disk.raw") 2956 .to_str() 2957 .unwrap(), 2958 ); 2959 assert!( 2960 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2961 ); 2962 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2963 2964 let mut cmd = GuestCommand::new(&guest); 2965 cmd.args(["--cpus", "boot=1"]) 2966 .args(["--memory", "size=512M"]) 2967 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2968 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2969 .args([ 2970 "--platform", 2971 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 2972 ]) 2973 .args([ 2974 "--disk", 2975 format!( 2976 "path={}", 2977 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2978 ) 2979 .as_str(), 2980 format!( 2981 "path={}", 2982 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2983 ) 2984 .as_str(), 2985 format!("path={test_disk_path},pci_segment=15").as_str(), 2986 ]) 2987 .capture_output() 2988 .default_net(); 2989 2990 let mut child = cmd.spawn().unwrap(); 2991 2992 guest.wait_vm_boot(None).unwrap(); 2993 2994 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2995 2996 let r = std::panic::catch_unwind(|| { 2997 // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. 2998 assert_eq!( 2999 guest 3000 .ssh_command(grep_cmd) 3001 .unwrap() 3002 .trim() 3003 .parse::<u16>() 3004 .unwrap_or_default(), 3005 MAX_NUM_PCI_SEGMENTS 3006 ); 3007 3008 // Check both if /dev/vdc exists and if the block size is 4M. 3009 assert_eq!( 3010 guest 3011 .ssh_command("lsblk | grep vdc | grep -c 4M") 3012 .unwrap() 3013 .trim() 3014 .parse::<u32>() 3015 .unwrap_or_default(), 3016 1 3017 ); 3018 3019 // Mount the device. 3020 guest.ssh_command("mkdir mount_image").unwrap(); 3021 guest 3022 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 3023 .unwrap(); 3024 // Grant all users with write permission. 3025 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 3026 3027 // Write something to the device. 3028 guest 3029 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 3030 .unwrap(); 3031 3032 // Check the content of the block device. The file "foo" should 3033 // contain "bar". 3034 assert_eq!( 3035 guest 3036 .ssh_command("sudo cat mount_image/foo") 3037 .unwrap() 3038 .trim(), 3039 "bar" 3040 ); 3041 }); 3042 3043 kill_child(&mut child); 3044 let output = child.wait_with_output().unwrap(); 3045 3046 handle_child_output(r, &output); 3047 } 3048 3049 #[test] 3050 fn test_pci_multiple_segments_numa_node() { 3051 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3052 let guest = Guest::new(Box::new(focal)); 3053 let api_socket = temp_api_path(&guest.tmp_dir); 3054 #[cfg(target_arch = "x86_64")] 3055 let kernel_path = direct_kernel_boot_path(); 3056 #[cfg(target_arch = "aarch64")] 3057 let kernel_path = edk2_path(); 3058 3059 // Prepare another disk file for the virtio-disk device 3060 let test_disk_path = String::from( 3061 guest 3062 .tmp_dir 3063 .as_path() 3064 .join("test-disk.raw") 3065 .to_str() 3066 .unwrap(), 3067 ); 3068 assert!( 3069 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 3070 ); 3071 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 3072 const TEST_DISK_NODE: u16 = 1; 3073 3074 let mut child = GuestCommand::new(&guest) 3075 .args(["--platform", "num_pci_segments=2"]) 3076 .args(["--cpus", "boot=2"]) 3077 .args(["--memory", "size=0"]) 3078 .args(["--memory-zone", "id=mem0,size=256M", "id=mem1,size=256M"]) 3079 .args([ 3080 "--numa", 3081 "guest_numa_id=0,cpus=[0],distances=[1@20],memory_zones=mem0,pci_segments=[0]", 3082 "guest_numa_id=1,cpus=[1],distances=[0@20],memory_zones=mem1,pci_segments=[1]", 3083 ]) 3084 .args(["--kernel", kernel_path.to_str().unwrap()]) 3085 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3086 .args(["--api-socket", &api_socket]) 3087 .capture_output() 3088 .args([ 3089 "--disk", 3090 format!( 3091 "path={}", 3092 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3093 ) 3094 .as_str(), 3095 format!( 3096 "path={}", 3097 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3098 ) 3099 .as_str(), 3100 format!("path={test_disk_path},pci_segment={TEST_DISK_NODE}").as_str(), 3101 ]) 3102 .default_net() 3103 .spawn() 3104 .unwrap(); 3105 3106 let cmd = "cat /sys/block/vdc/device/../numa_node"; 3107 3108 let r = std::panic::catch_unwind(|| { 3109 guest.wait_vm_boot(None).unwrap(); 3110 3111 assert_eq!( 3112 guest 3113 .ssh_command(cmd) 3114 .unwrap() 3115 .trim() 3116 .parse::<u16>() 3117 .unwrap_or_default(), 3118 TEST_DISK_NODE 3119 ); 3120 }); 3121 3122 kill_child(&mut child); 3123 let output = child.wait_with_output().unwrap(); 3124 3125 handle_child_output(r, &output); 3126 } 3127 3128 #[test] 3129 fn test_direct_kernel_boot() { 3130 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3131 let guest = Guest::new(Box::new(focal)); 3132 3133 let kernel_path = direct_kernel_boot_path(); 3134 3135 let mut child = GuestCommand::new(&guest) 3136 .args(["--cpus", "boot=1"]) 3137 .args(["--memory", "size=512M"]) 3138 .args(["--kernel", kernel_path.to_str().unwrap()]) 3139 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3140 .default_disks() 3141 .default_net() 3142 .capture_output() 3143 .spawn() 3144 .unwrap(); 3145 3146 let r = std::panic::catch_unwind(|| { 3147 guest.wait_vm_boot(None).unwrap(); 3148 3149 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3150 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3151 3152 let grep_cmd = if cfg!(target_arch = "x86_64") { 3153 "grep -c PCI-MSI /proc/interrupts" 3154 } else { 3155 "grep -c ITS-MSI /proc/interrupts" 3156 }; 3157 assert_eq!( 3158 guest 3159 .ssh_command(grep_cmd) 3160 .unwrap() 3161 .trim() 3162 .parse::<u32>() 3163 .unwrap_or_default(), 3164 12 3165 ); 3166 }); 3167 3168 kill_child(&mut child); 3169 let output = child.wait_with_output().unwrap(); 3170 3171 handle_child_output(r, &output); 3172 } 3173 3174 #[test] 3175 #[cfg(target_arch = "x86_64")] 3176 fn test_direct_kernel_boot_bzimage() { 3177 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3178 let guest = Guest::new(Box::new(focal)); 3179 3180 let mut kernel_path = direct_kernel_boot_path(); 3181 // Replace the default kernel with the bzImage. 3182 kernel_path.pop(); 3183 kernel_path.push("bzImage"); 3184 3185 let mut child = GuestCommand::new(&guest) 3186 .args(["--cpus", "boot=1"]) 3187 .args(["--memory", "size=512M"]) 3188 .args(["--kernel", kernel_path.to_str().unwrap()]) 3189 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3190 .default_disks() 3191 .default_net() 3192 .capture_output() 3193 .spawn() 3194 .unwrap(); 3195 3196 let r = std::panic::catch_unwind(|| { 3197 guest.wait_vm_boot(None).unwrap(); 3198 3199 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3200 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3201 3202 let grep_cmd = if cfg!(target_arch = "x86_64") { 3203 "grep -c PCI-MSI /proc/interrupts" 3204 } else { 3205 "grep -c ITS-MSI /proc/interrupts" 3206 }; 3207 assert_eq!( 3208 guest 3209 .ssh_command(grep_cmd) 3210 .unwrap() 3211 .trim() 3212 .parse::<u32>() 3213 .unwrap_or_default(), 3214 12 3215 ); 3216 }); 3217 3218 kill_child(&mut child); 3219 let output = child.wait_with_output().unwrap(); 3220 3221 handle_child_output(r, &output); 3222 } 3223 3224 fn _test_virtio_block(image_name: &str, disable_io_uring: bool, disable_aio: bool) { 3225 let focal = UbuntuDiskConfig::new(image_name.to_string()); 3226 let guest = Guest::new(Box::new(focal)); 3227 3228 let mut workload_path = dirs::home_dir().unwrap(); 3229 workload_path.push("workloads"); 3230 3231 let mut blk_file_path = workload_path; 3232 blk_file_path.push("blk.img"); 3233 3234 let kernel_path = direct_kernel_boot_path(); 3235 3236 let mut cloud_child = GuestCommand::new(&guest) 3237 .args(["--cpus", "boot=4"]) 3238 .args(["--memory", "size=512M,shared=on"]) 3239 .args(["--kernel", kernel_path.to_str().unwrap()]) 3240 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3241 .args([ 3242 "--disk", 3243 format!( 3244 "path={}", 3245 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3246 ) 3247 .as_str(), 3248 format!( 3249 "path={}", 3250 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3251 ) 3252 .as_str(), 3253 format!( 3254 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", 3255 blk_file_path.to_str().unwrap(), 3256 disable_io_uring, 3257 disable_aio, 3258 ) 3259 .as_str(), 3260 ]) 3261 .default_net() 3262 .capture_output() 3263 .spawn() 3264 .unwrap(); 3265 3266 let r = std::panic::catch_unwind(|| { 3267 guest.wait_vm_boot(None).unwrap(); 3268 3269 // Check both if /dev/vdc exists and if the block size is 16M. 3270 assert_eq!( 3271 guest 3272 .ssh_command("lsblk | grep vdc | grep -c 16M") 3273 .unwrap() 3274 .trim() 3275 .parse::<u32>() 3276 .unwrap_or_default(), 3277 1 3278 ); 3279 3280 // Check both if /dev/vdc exists and if this block is RO. 3281 assert_eq!( 3282 guest 3283 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 3284 .unwrap() 3285 .trim() 3286 .parse::<u32>() 3287 .unwrap_or_default(), 3288 1 3289 ); 3290 3291 // Check if the number of queues is 4. 3292 assert_eq!( 3293 guest 3294 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 3295 .unwrap() 3296 .trim() 3297 .parse::<u32>() 3298 .unwrap_or_default(), 3299 4 3300 ); 3301 }); 3302 3303 let _ = cloud_child.kill(); 3304 let output = cloud_child.wait_with_output().unwrap(); 3305 3306 handle_child_output(r, &output); 3307 } 3308 3309 #[test] 3310 fn test_virtio_block_io_uring() { 3311 _test_virtio_block(FOCAL_IMAGE_NAME, false, true) 3312 } 3313 3314 #[test] 3315 fn test_virtio_block_aio() { 3316 _test_virtio_block(FOCAL_IMAGE_NAME, true, false) 3317 } 3318 3319 #[test] 3320 fn test_virtio_block_sync() { 3321 _test_virtio_block(FOCAL_IMAGE_NAME, true, true) 3322 } 3323 3324 #[test] 3325 fn test_virtio_block_qcow2() { 3326 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false, false) 3327 } 3328 3329 #[test] 3330 fn test_virtio_block_qcow2_backing_file() { 3331 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE, false, false) 3332 } 3333 3334 #[test] 3335 fn test_virtio_block_vhd() { 3336 let mut workload_path = dirs::home_dir().unwrap(); 3337 workload_path.push("workloads"); 3338 3339 let mut raw_file_path = workload_path.clone(); 3340 let mut vhd_file_path = workload_path; 3341 raw_file_path.push(FOCAL_IMAGE_NAME); 3342 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 3343 3344 // Generate VHD file from RAW file 3345 std::process::Command::new("qemu-img") 3346 .arg("convert") 3347 .arg("-p") 3348 .args(["-f", "raw"]) 3349 .args(["-O", "vpc"]) 3350 .args(["-o", "subformat=fixed"]) 3351 .arg(raw_file_path.to_str().unwrap()) 3352 .arg(vhd_file_path.to_str().unwrap()) 3353 .output() 3354 .expect("Expect generating VHD image from RAW image"); 3355 3356 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false, false) 3357 } 3358 3359 #[test] 3360 fn test_virtio_block_vhdx() { 3361 let mut workload_path = dirs::home_dir().unwrap(); 3362 workload_path.push("workloads"); 3363 3364 let mut raw_file_path = workload_path.clone(); 3365 let mut vhdx_file_path = workload_path; 3366 raw_file_path.push(FOCAL_IMAGE_NAME); 3367 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 3368 3369 // Generate dynamic VHDX file from RAW file 3370 std::process::Command::new("qemu-img") 3371 .arg("convert") 3372 .arg("-p") 3373 .args(["-f", "raw"]) 3374 .args(["-O", "vhdx"]) 3375 .arg(raw_file_path.to_str().unwrap()) 3376 .arg(vhdx_file_path.to_str().unwrap()) 3377 .output() 3378 .expect("Expect generating dynamic VHDx image from RAW image"); 3379 3380 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false, false) 3381 } 3382 3383 #[test] 3384 fn test_virtio_block_dynamic_vhdx_expand() { 3385 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 3386 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 3387 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 3388 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 3389 3390 let mut workload_path = dirs::home_dir().unwrap(); 3391 workload_path.push("workloads"); 3392 3393 let mut vhdx_file_path = workload_path; 3394 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 3395 let vhdx_path = vhdx_file_path.to_str().unwrap(); 3396 3397 // Generate a 100 MiB dynamic VHDX file 3398 std::process::Command::new("qemu-img") 3399 .arg("create") 3400 .args(["-f", "vhdx"]) 3401 .arg(vhdx_path) 3402 .arg(VIRTUAL_DISK_SIZE.to_string()) 3403 .output() 3404 .expect("Expect generating dynamic VHDx image from RAW image"); 3405 3406 // Check if the size matches with empty VHDx file size 3407 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 3408 3409 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3410 let guest = Guest::new(Box::new(focal)); 3411 let kernel_path = direct_kernel_boot_path(); 3412 3413 let mut cloud_child = GuestCommand::new(&guest) 3414 .args(["--cpus", "boot=1"]) 3415 .args(["--memory", "size=512M"]) 3416 .args(["--kernel", kernel_path.to_str().unwrap()]) 3417 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3418 .args([ 3419 "--disk", 3420 format!( 3421 "path={}", 3422 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3423 ) 3424 .as_str(), 3425 format!( 3426 "path={}", 3427 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3428 ) 3429 .as_str(), 3430 format!("path={vhdx_path}").as_str(), 3431 ]) 3432 .default_net() 3433 .capture_output() 3434 .spawn() 3435 .unwrap(); 3436 3437 let r = std::panic::catch_unwind(|| { 3438 guest.wait_vm_boot(None).unwrap(); 3439 3440 // Check both if /dev/vdc exists and if the block size is 100 MiB. 3441 assert_eq!( 3442 guest 3443 .ssh_command("lsblk | grep vdc | grep -c 100M") 3444 .unwrap() 3445 .trim() 3446 .parse::<u32>() 3447 .unwrap_or_default(), 3448 1 3449 ); 3450 3451 // Write 100 MB of data to the VHDx disk 3452 guest 3453 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 3454 .unwrap(); 3455 }); 3456 3457 // Check if the size matches with expected expanded VHDx file size 3458 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 3459 3460 kill_child(&mut cloud_child); 3461 let output = cloud_child.wait_with_output().unwrap(); 3462 3463 handle_child_output(r, &output); 3464 } 3465 3466 fn vhdx_image_size(disk_name: &str) -> u64 { 3467 std::fs::File::open(disk_name) 3468 .unwrap() 3469 .seek(SeekFrom::End(0)) 3470 .unwrap() 3471 } 3472 3473 #[test] 3474 fn test_virtio_block_direct_and_firmware() { 3475 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3476 let guest = Guest::new(Box::new(focal)); 3477 3478 // The OS disk must be copied to a location that is not backed by 3479 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 3480 // with EINVAL because tmpfs doesn't support this flag. 3481 let mut workloads_path = dirs::home_dir().unwrap(); 3482 workloads_path.push("workloads"); 3483 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 3484 let mut os_path = os_dir.as_path().to_path_buf(); 3485 os_path.push("osdisk.img"); 3486 rate_limited_copy( 3487 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3488 os_path.as_path(), 3489 ) 3490 .expect("copying of OS disk failed"); 3491 3492 let mut child = GuestCommand::new(&guest) 3493 .args(["--cpus", "boot=1"]) 3494 .args(["--memory", "size=512M"]) 3495 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 3496 .args([ 3497 "--disk", 3498 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 3499 format!( 3500 "path={}", 3501 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3502 ) 3503 .as_str(), 3504 ]) 3505 .default_net() 3506 .capture_output() 3507 .spawn() 3508 .unwrap(); 3509 3510 let r = std::panic::catch_unwind(|| { 3511 guest.wait_vm_boot(Some(120)).unwrap(); 3512 }); 3513 3514 kill_child(&mut child); 3515 let output = child.wait_with_output().unwrap(); 3516 3517 handle_child_output(r, &output); 3518 } 3519 3520 #[test] 3521 fn test_vhost_user_net_default() { 3522 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 3523 } 3524 3525 #[test] 3526 fn test_vhost_user_net_named_tap() { 3527 test_vhost_user_net( 3528 Some("mytap0"), 3529 2, 3530 &prepare_vhost_user_net_daemon, 3531 false, 3532 false, 3533 ) 3534 } 3535 3536 #[test] 3537 fn test_vhost_user_net_existing_tap() { 3538 test_vhost_user_net( 3539 Some("vunet-tap0"), 3540 2, 3541 &prepare_vhost_user_net_daemon, 3542 false, 3543 false, 3544 ) 3545 } 3546 3547 #[test] 3548 fn test_vhost_user_net_multiple_queues() { 3549 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 3550 } 3551 3552 #[test] 3553 fn test_vhost_user_net_tap_multiple_queues() { 3554 test_vhost_user_net( 3555 Some("vunet-tap1"), 3556 4, 3557 &prepare_vhost_user_net_daemon, 3558 false, 3559 false, 3560 ) 3561 } 3562 3563 #[test] 3564 fn test_vhost_user_net_host_mac() { 3565 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 3566 } 3567 3568 #[test] 3569 fn test_vhost_user_net_client_mode() { 3570 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 3571 } 3572 3573 #[test] 3574 #[cfg(not(target_arch = "aarch64"))] 3575 fn test_vhost_user_blk_default() { 3576 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 3577 } 3578 3579 #[test] 3580 #[cfg(not(target_arch = "aarch64"))] 3581 fn test_vhost_user_blk_readonly() { 3582 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3583 } 3584 3585 #[test] 3586 #[cfg(not(target_arch = "aarch64"))] 3587 fn test_vhost_user_blk_direct() { 3588 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3589 } 3590 3591 #[test] 3592 fn test_boot_from_vhost_user_blk_default() { 3593 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3594 } 3595 3596 #[test] 3597 #[cfg(target_arch = "x86_64")] 3598 fn test_split_irqchip() { 3599 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3600 let guest = Guest::new(Box::new(focal)); 3601 3602 let mut child = GuestCommand::new(&guest) 3603 .args(["--cpus", "boot=1"]) 3604 .args(["--memory", "size=512M"]) 3605 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3606 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3607 .default_disks() 3608 .default_net() 3609 .capture_output() 3610 .spawn() 3611 .unwrap(); 3612 3613 let r = std::panic::catch_unwind(|| { 3614 guest.wait_vm_boot(None).unwrap(); 3615 3616 assert_eq!( 3617 guest 3618 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3619 .unwrap() 3620 .trim() 3621 .parse::<u32>() 3622 .unwrap_or(1), 3623 0 3624 ); 3625 assert_eq!( 3626 guest 3627 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3628 .unwrap() 3629 .trim() 3630 .parse::<u32>() 3631 .unwrap_or(1), 3632 0 3633 ); 3634 }); 3635 3636 kill_child(&mut child); 3637 let output = child.wait_with_output().unwrap(); 3638 3639 handle_child_output(r, &output); 3640 } 3641 3642 #[test] 3643 #[cfg(target_arch = "x86_64")] 3644 fn test_dmi_serial_number() { 3645 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3646 let guest = Guest::new(Box::new(focal)); 3647 3648 let mut child = GuestCommand::new(&guest) 3649 .args(["--cpus", "boot=1"]) 3650 .args(["--memory", "size=512M"]) 3651 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3652 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3653 .args(["--platform", "serial_number=a=b;c=d"]) 3654 .default_disks() 3655 .default_net() 3656 .capture_output() 3657 .spawn() 3658 .unwrap(); 3659 3660 let r = std::panic::catch_unwind(|| { 3661 guest.wait_vm_boot(None).unwrap(); 3662 3663 assert_eq!( 3664 guest 3665 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3666 .unwrap() 3667 .trim(), 3668 "a=b;c=d" 3669 ); 3670 }); 3671 3672 kill_child(&mut child); 3673 let output = child.wait_with_output().unwrap(); 3674 3675 handle_child_output(r, &output); 3676 } 3677 3678 #[test] 3679 #[cfg(target_arch = "x86_64")] 3680 fn test_dmi_uuid() { 3681 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3682 let guest = Guest::new(Box::new(focal)); 3683 3684 let mut child = GuestCommand::new(&guest) 3685 .args(["--cpus", "boot=1"]) 3686 .args(["--memory", "size=512M"]) 3687 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3688 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3689 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3690 .default_disks() 3691 .default_net() 3692 .capture_output() 3693 .spawn() 3694 .unwrap(); 3695 3696 let r = std::panic::catch_unwind(|| { 3697 guest.wait_vm_boot(None).unwrap(); 3698 3699 assert_eq!( 3700 guest 3701 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3702 .unwrap() 3703 .trim(), 3704 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3705 ); 3706 }); 3707 3708 kill_child(&mut child); 3709 let output = child.wait_with_output().unwrap(); 3710 3711 handle_child_output(r, &output); 3712 } 3713 3714 #[test] 3715 #[cfg(target_arch = "x86_64")] 3716 fn test_dmi_oem_strings() { 3717 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3718 let guest = Guest::new(Box::new(focal)); 3719 3720 let s1 = "io.systemd.credential:xx=yy"; 3721 let s2 = "This is a test string"; 3722 3723 let oem_strings = format!("oem_strings=[{s1},{s2}]"); 3724 3725 let mut child = GuestCommand::new(&guest) 3726 .args(["--cpus", "boot=1"]) 3727 .args(["--memory", "size=512M"]) 3728 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3729 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3730 .args(["--platform", &oem_strings]) 3731 .default_disks() 3732 .default_net() 3733 .capture_output() 3734 .spawn() 3735 .unwrap(); 3736 3737 let r = std::panic::catch_unwind(|| { 3738 guest.wait_vm_boot(None).unwrap(); 3739 3740 assert_eq!( 3741 guest 3742 .ssh_command("sudo dmidecode --oem-string count") 3743 .unwrap() 3744 .trim(), 3745 "2" 3746 ); 3747 3748 assert_eq!( 3749 guest 3750 .ssh_command("sudo dmidecode --oem-string 1") 3751 .unwrap() 3752 .trim(), 3753 s1 3754 ); 3755 3756 assert_eq!( 3757 guest 3758 .ssh_command("sudo dmidecode --oem-string 2") 3759 .unwrap() 3760 .trim(), 3761 s2 3762 ); 3763 }); 3764 3765 kill_child(&mut child); 3766 let output = child.wait_with_output().unwrap(); 3767 3768 handle_child_output(r, &output); 3769 } 3770 3771 #[test] 3772 fn test_virtio_fs() { 3773 _test_virtio_fs(&prepare_virtiofsd, false, None) 3774 } 3775 3776 #[test] 3777 fn test_virtio_fs_hotplug() { 3778 _test_virtio_fs(&prepare_virtiofsd, true, None) 3779 } 3780 3781 #[test] 3782 #[cfg(not(feature = "mshv"))] 3783 fn test_virtio_fs_multi_segment_hotplug() { 3784 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3785 } 3786 3787 #[test] 3788 #[cfg(not(feature = "mshv"))] 3789 fn test_virtio_fs_multi_segment() { 3790 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3791 } 3792 3793 #[test] 3794 fn test_virtio_pmem_persist_writes() { 3795 test_virtio_pmem(false, false) 3796 } 3797 3798 #[test] 3799 fn test_virtio_pmem_discard_writes() { 3800 test_virtio_pmem(true, false) 3801 } 3802 3803 #[test] 3804 fn test_virtio_pmem_with_size() { 3805 test_virtio_pmem(true, true) 3806 } 3807 3808 #[test] 3809 fn test_boot_from_virtio_pmem() { 3810 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3811 let guest = Guest::new(Box::new(focal)); 3812 3813 let kernel_path = direct_kernel_boot_path(); 3814 3815 let mut child = GuestCommand::new(&guest) 3816 .args(["--cpus", "boot=1"]) 3817 .args(["--memory", "size=512M"]) 3818 .args(["--kernel", kernel_path.to_str().unwrap()]) 3819 .args([ 3820 "--disk", 3821 format!( 3822 "path={}", 3823 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3824 ) 3825 .as_str(), 3826 ]) 3827 .default_net() 3828 .args([ 3829 "--pmem", 3830 format!( 3831 "file={},size={}", 3832 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3833 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3834 .unwrap() 3835 .len() 3836 ) 3837 .as_str(), 3838 ]) 3839 .args([ 3840 "--cmdline", 3841 DIRECT_KERNEL_BOOT_CMDLINE 3842 .replace("vda1", "pmem0p1") 3843 .as_str(), 3844 ]) 3845 .capture_output() 3846 .spawn() 3847 .unwrap(); 3848 3849 let r = std::panic::catch_unwind(|| { 3850 guest.wait_vm_boot(None).unwrap(); 3851 3852 // Simple checks to validate the VM booted properly 3853 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3854 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3855 }); 3856 3857 kill_child(&mut child); 3858 let output = child.wait_with_output().unwrap(); 3859 3860 handle_child_output(r, &output); 3861 } 3862 3863 #[test] 3864 fn test_multiple_network_interfaces() { 3865 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3866 let guest = Guest::new(Box::new(focal)); 3867 3868 let kernel_path = direct_kernel_boot_path(); 3869 3870 let mut child = GuestCommand::new(&guest) 3871 .args(["--cpus", "boot=1"]) 3872 .args(["--memory", "size=512M"]) 3873 .args(["--kernel", kernel_path.to_str().unwrap()]) 3874 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3875 .default_disks() 3876 .args([ 3877 "--net", 3878 guest.default_net_string().as_str(), 3879 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3880 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3881 ]) 3882 .capture_output() 3883 .spawn() 3884 .unwrap(); 3885 3886 let r = std::panic::catch_unwind(|| { 3887 guest.wait_vm_boot(None).unwrap(); 3888 3889 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3890 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3891 3892 // 3 network interfaces + default localhost ==> 4 interfaces 3893 assert_eq!( 3894 guest 3895 .ssh_command("ip -o link | wc -l") 3896 .unwrap() 3897 .trim() 3898 .parse::<u32>() 3899 .unwrap_or_default(), 3900 4 3901 ); 3902 }); 3903 3904 kill_child(&mut child); 3905 let output = child.wait_with_output().unwrap(); 3906 3907 handle_child_output(r, &output); 3908 } 3909 3910 #[test] 3911 #[cfg(target_arch = "aarch64")] 3912 fn test_pmu_on() { 3913 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3914 let guest = Guest::new(Box::new(focal)); 3915 let mut child = GuestCommand::new(&guest) 3916 .args(["--cpus", "boot=1"]) 3917 .args(["--memory", "size=512M"]) 3918 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3919 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3920 .default_disks() 3921 .default_net() 3922 .capture_output() 3923 .spawn() 3924 .unwrap(); 3925 3926 let r = std::panic::catch_unwind(|| { 3927 guest.wait_vm_boot(None).unwrap(); 3928 3929 // Test that PMU exists. 3930 assert_eq!( 3931 guest 3932 .ssh_command(GREP_PMU_IRQ_CMD) 3933 .unwrap() 3934 .trim() 3935 .parse::<u32>() 3936 .unwrap_or_default(), 3937 1 3938 ); 3939 }); 3940 3941 kill_child(&mut child); 3942 let output = child.wait_with_output().unwrap(); 3943 3944 handle_child_output(r, &output); 3945 } 3946 3947 #[test] 3948 fn test_serial_off() { 3949 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3950 let guest = Guest::new(Box::new(focal)); 3951 let mut child = GuestCommand::new(&guest) 3952 .args(["--cpus", "boot=1"]) 3953 .args(["--memory", "size=512M"]) 3954 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3955 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3956 .default_disks() 3957 .default_net() 3958 .args(["--serial", "off"]) 3959 .capture_output() 3960 .spawn() 3961 .unwrap(); 3962 3963 let r = std::panic::catch_unwind(|| { 3964 guest.wait_vm_boot(None).unwrap(); 3965 3966 // Test that there is no ttyS0 3967 assert_eq!( 3968 guest 3969 .ssh_command(GREP_SERIAL_IRQ_CMD) 3970 .unwrap() 3971 .trim() 3972 .parse::<u32>() 3973 .unwrap_or(1), 3974 0 3975 ); 3976 }); 3977 3978 kill_child(&mut child); 3979 let output = child.wait_with_output().unwrap(); 3980 3981 handle_child_output(r, &output); 3982 } 3983 3984 #[test] 3985 fn test_serial_null() { 3986 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3987 let guest = Guest::new(Box::new(focal)); 3988 let mut cmd = GuestCommand::new(&guest); 3989 #[cfg(target_arch = "x86_64")] 3990 let console_str: &str = "console=ttyS0"; 3991 #[cfg(target_arch = "aarch64")] 3992 let console_str: &str = "console=ttyAMA0"; 3993 3994 cmd.args(["--cpus", "boot=1"]) 3995 .args(["--memory", "size=512M"]) 3996 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3997 .args([ 3998 "--cmdline", 3999 DIRECT_KERNEL_BOOT_CMDLINE 4000 .replace("console=hvc0 ", console_str) 4001 .as_str(), 4002 ]) 4003 .default_disks() 4004 .default_net() 4005 .args(["--serial", "null"]) 4006 .args(["--console", "off"]) 4007 .capture_output(); 4008 4009 let mut child = cmd.spawn().unwrap(); 4010 4011 let r = std::panic::catch_unwind(|| { 4012 guest.wait_vm_boot(None).unwrap(); 4013 4014 // Test that there is a ttyS0 4015 assert_eq!( 4016 guest 4017 .ssh_command(GREP_SERIAL_IRQ_CMD) 4018 .unwrap() 4019 .trim() 4020 .parse::<u32>() 4021 .unwrap_or_default(), 4022 1 4023 ); 4024 }); 4025 4026 kill_child(&mut child); 4027 let output = child.wait_with_output().unwrap(); 4028 handle_child_output(r, &output); 4029 4030 let r = std::panic::catch_unwind(|| { 4031 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4032 }); 4033 4034 handle_child_output(r, &output); 4035 } 4036 4037 #[test] 4038 fn test_serial_tty() { 4039 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4040 let guest = Guest::new(Box::new(focal)); 4041 4042 let kernel_path = direct_kernel_boot_path(); 4043 4044 #[cfg(target_arch = "x86_64")] 4045 let console_str: &str = "console=ttyS0"; 4046 #[cfg(target_arch = "aarch64")] 4047 let console_str: &str = "console=ttyAMA0"; 4048 4049 let mut child = GuestCommand::new(&guest) 4050 .args(["--cpus", "boot=1"]) 4051 .args(["--memory", "size=512M"]) 4052 .args(["--kernel", kernel_path.to_str().unwrap()]) 4053 .args([ 4054 "--cmdline", 4055 DIRECT_KERNEL_BOOT_CMDLINE 4056 .replace("console=hvc0 ", console_str) 4057 .as_str(), 4058 ]) 4059 .default_disks() 4060 .default_net() 4061 .args(["--serial", "tty"]) 4062 .args(["--console", "off"]) 4063 .capture_output() 4064 .spawn() 4065 .unwrap(); 4066 4067 let r = std::panic::catch_unwind(|| { 4068 guest.wait_vm_boot(None).unwrap(); 4069 4070 // Test that there is a ttyS0 4071 assert_eq!( 4072 guest 4073 .ssh_command(GREP_SERIAL_IRQ_CMD) 4074 .unwrap() 4075 .trim() 4076 .parse::<u32>() 4077 .unwrap_or_default(), 4078 1 4079 ); 4080 }); 4081 4082 // This sleep is needed to wait for the login prompt 4083 thread::sleep(std::time::Duration::new(2, 0)); 4084 4085 kill_child(&mut child); 4086 let output = child.wait_with_output().unwrap(); 4087 handle_child_output(r, &output); 4088 4089 let r = std::panic::catch_unwind(|| { 4090 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4091 }); 4092 4093 handle_child_output(r, &output); 4094 } 4095 4096 #[test] 4097 fn test_serial_file() { 4098 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4099 let guest = Guest::new(Box::new(focal)); 4100 4101 let serial_path = guest.tmp_dir.as_path().join("serial-output"); 4102 #[cfg(target_arch = "x86_64")] 4103 let console_str: &str = "console=ttyS0"; 4104 #[cfg(target_arch = "aarch64")] 4105 let console_str: &str = "console=ttyAMA0"; 4106 4107 let mut child = GuestCommand::new(&guest) 4108 .args(["--cpus", "boot=1"]) 4109 .args(["--memory", "size=512M"]) 4110 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4111 .args([ 4112 "--cmdline", 4113 DIRECT_KERNEL_BOOT_CMDLINE 4114 .replace("console=hvc0 ", console_str) 4115 .as_str(), 4116 ]) 4117 .default_disks() 4118 .default_net() 4119 .args([ 4120 "--serial", 4121 format!("file={}", serial_path.to_str().unwrap()).as_str(), 4122 ]) 4123 .capture_output() 4124 .spawn() 4125 .unwrap(); 4126 4127 let r = std::panic::catch_unwind(|| { 4128 guest.wait_vm_boot(None).unwrap(); 4129 4130 // Test that there is a ttyS0 4131 assert_eq!( 4132 guest 4133 .ssh_command(GREP_SERIAL_IRQ_CMD) 4134 .unwrap() 4135 .trim() 4136 .parse::<u32>() 4137 .unwrap_or_default(), 4138 1 4139 ); 4140 4141 guest.ssh_command("sudo shutdown -h now").unwrap(); 4142 }); 4143 4144 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4145 kill_child(&mut child); 4146 let output = child.wait_with_output().unwrap(); 4147 handle_child_output(r, &output); 4148 4149 let r = std::panic::catch_unwind(|| { 4150 // Check that the cloud-hypervisor binary actually terminated 4151 assert!(output.status.success()); 4152 4153 // Do this check after shutdown of the VM as an easy way to ensure 4154 // all writes are flushed to disk 4155 let mut f = std::fs::File::open(serial_path).unwrap(); 4156 let mut buf = String::new(); 4157 f.read_to_string(&mut buf).unwrap(); 4158 assert!(buf.contains(CONSOLE_TEST_STRING)); 4159 }); 4160 4161 handle_child_output(r, &output); 4162 } 4163 4164 #[test] 4165 fn test_pty_interaction() { 4166 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4167 let guest = Guest::new(Box::new(focal)); 4168 let api_socket = temp_api_path(&guest.tmp_dir); 4169 let serial_option = if cfg!(target_arch = "x86_64") { 4170 " console=ttyS0" 4171 } else { 4172 " console=ttyAMA0" 4173 }; 4174 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4175 4176 let mut child = GuestCommand::new(&guest) 4177 .args(["--cpus", "boot=1"]) 4178 .args(["--memory", "size=512M"]) 4179 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4180 .args(["--cmdline", &cmdline]) 4181 .default_disks() 4182 .default_net() 4183 .args(["--serial", "null"]) 4184 .args(["--console", "pty"]) 4185 .args(["--api-socket", &api_socket]) 4186 .spawn() 4187 .unwrap(); 4188 4189 let r = std::panic::catch_unwind(|| { 4190 guest.wait_vm_boot(None).unwrap(); 4191 // Get pty fd for console 4192 let console_path = get_pty_path(&api_socket, "console"); 4193 _test_pty_interaction(console_path); 4194 4195 guest.ssh_command("sudo shutdown -h now").unwrap(); 4196 }); 4197 4198 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4199 let _ = child.kill(); 4200 let output = child.wait_with_output().unwrap(); 4201 handle_child_output(r, &output); 4202 4203 let r = std::panic::catch_unwind(|| { 4204 // Check that the cloud-hypervisor binary actually terminated 4205 assert!(output.status.success()) 4206 }); 4207 handle_child_output(r, &output); 4208 } 4209 4210 #[test] 4211 fn test_serial_socket_interaction() { 4212 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4213 let guest = Guest::new(Box::new(focal)); 4214 let serial_socket = guest.tmp_dir.as_path().join("serial.socket"); 4215 let serial_socket_pty = guest.tmp_dir.as_path().join("serial.pty"); 4216 let serial_option = if cfg!(target_arch = "x86_64") { 4217 " console=ttyS0" 4218 } else { 4219 " console=ttyAMA0" 4220 }; 4221 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4222 4223 let mut child = GuestCommand::new(&guest) 4224 .args(["--cpus", "boot=1"]) 4225 .args(["--memory", "size=512M"]) 4226 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4227 .args(["--cmdline", &cmdline]) 4228 .default_disks() 4229 .default_net() 4230 .args(["--console", "null"]) 4231 .args([ 4232 "--serial", 4233 format!("socket={}", serial_socket.to_str().unwrap()).as_str(), 4234 ]) 4235 .spawn() 4236 .unwrap(); 4237 4238 let _ = std::panic::catch_unwind(|| { 4239 guest.wait_vm_boot(None).unwrap(); 4240 }); 4241 4242 let mut socat_command = Command::new("socat"); 4243 let socat_args = [ 4244 &format!("pty,link={},raw", serial_socket_pty.display()), 4245 &format!("UNIX-CONNECT:{}", serial_socket.display()), 4246 ]; 4247 socat_command.args(socat_args); 4248 4249 let mut socat_child = socat_command.spawn().unwrap(); 4250 thread::sleep(std::time::Duration::new(1, 0)); 4251 4252 let _ = std::panic::catch_unwind(|| { 4253 _test_pty_interaction(serial_socket_pty); 4254 }); 4255 4256 let _ = socat_child.kill(); 4257 4258 let r = std::panic::catch_unwind(|| { 4259 guest.ssh_command("sudo shutdown -h now").unwrap(); 4260 }); 4261 4262 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4263 kill_child(&mut child); 4264 let output = child.wait_with_output().unwrap(); 4265 handle_child_output(r, &output); 4266 4267 let r = std::panic::catch_unwind(|| { 4268 // Check that the cloud-hypervisor binary actually terminated 4269 if !output.status.success() { 4270 panic!( 4271 "Cloud Hypervisor process failed to terminate gracefully: {:?}", 4272 output.status 4273 ); 4274 } 4275 }); 4276 handle_child_output(r, &output); 4277 } 4278 4279 #[test] 4280 fn test_virtio_console() { 4281 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4282 let guest = Guest::new(Box::new(focal)); 4283 4284 let kernel_path = direct_kernel_boot_path(); 4285 4286 let mut child = GuestCommand::new(&guest) 4287 .args(["--cpus", "boot=1"]) 4288 .args(["--memory", "size=512M"]) 4289 .args(["--kernel", kernel_path.to_str().unwrap()]) 4290 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4291 .default_disks() 4292 .default_net() 4293 .args(["--console", "tty"]) 4294 .args(["--serial", "null"]) 4295 .capture_output() 4296 .spawn() 4297 .unwrap(); 4298 4299 let text = String::from("On a branch floating down river a cricket, singing."); 4300 let cmd = format!("echo {text} | sudo tee /dev/hvc0"); 4301 4302 let r = std::panic::catch_unwind(|| { 4303 guest.wait_vm_boot(None).unwrap(); 4304 4305 assert!(guest 4306 .does_device_vendor_pair_match("0x1043", "0x1af4") 4307 .unwrap_or_default()); 4308 4309 guest.ssh_command(&cmd).unwrap(); 4310 }); 4311 4312 kill_child(&mut child); 4313 let output = child.wait_with_output().unwrap(); 4314 handle_child_output(r, &output); 4315 4316 let r = std::panic::catch_unwind(|| { 4317 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 4318 }); 4319 4320 handle_child_output(r, &output); 4321 } 4322 4323 #[test] 4324 fn test_console_file() { 4325 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4326 let guest = Guest::new(Box::new(focal)); 4327 4328 let console_path = guest.tmp_dir.as_path().join("console-output"); 4329 let mut child = GuestCommand::new(&guest) 4330 .args(["--cpus", "boot=1"]) 4331 .args(["--memory", "size=512M"]) 4332 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4333 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4334 .default_disks() 4335 .default_net() 4336 .args([ 4337 "--console", 4338 format!("file={}", console_path.to_str().unwrap()).as_str(), 4339 ]) 4340 .capture_output() 4341 .spawn() 4342 .unwrap(); 4343 4344 guest.wait_vm_boot(None).unwrap(); 4345 4346 guest.ssh_command("sudo shutdown -h now").unwrap(); 4347 4348 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4349 kill_child(&mut child); 4350 let output = child.wait_with_output().unwrap(); 4351 4352 let r = std::panic::catch_unwind(|| { 4353 // Check that the cloud-hypervisor binary actually terminated 4354 assert!(output.status.success()); 4355 4356 // Do this check after shutdown of the VM as an easy way to ensure 4357 // all writes are flushed to disk 4358 let mut f = std::fs::File::open(console_path).unwrap(); 4359 let mut buf = String::new(); 4360 f.read_to_string(&mut buf).unwrap(); 4361 4362 if !buf.contains(CONSOLE_TEST_STRING) { 4363 eprintln!( 4364 "\n\n==== Console file output ====\n\n{buf}\n\n==== End console file output ====" 4365 ); 4366 } 4367 assert!(buf.contains(CONSOLE_TEST_STRING)); 4368 }); 4369 4370 handle_child_output(r, &output); 4371 } 4372 4373 #[test] 4374 #[cfg(target_arch = "x86_64")] 4375 #[cfg(not(feature = "mshv"))] 4376 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 4377 // backed networking interfaces, bound through a simple bridge on the host. 4378 // So if the nested cloud-hypervisor succeeds in getting a directly 4379 // assigned interface from its cloud-hypervisor host, we should be able to 4380 // ssh into it, and verify that it's running with the right kernel command 4381 // line (We tag the command line from cloud-hypervisor for that purpose). 4382 // The third device is added to validate that hotplug works correctly since 4383 // it is being added to the L2 VM through hotplugging mechanism. 4384 // Also, we pass-through a virtio-blk device to the L2 VM to test the 32-bit 4385 // vfio device support 4386 fn test_vfio() { 4387 setup_vfio_network_interfaces(); 4388 4389 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 4390 let guest = Guest::new_from_ip_range(Box::new(jammy), "172.18", 0); 4391 4392 let mut workload_path = dirs::home_dir().unwrap(); 4393 workload_path.push("workloads"); 4394 4395 let kernel_path = direct_kernel_boot_path(); 4396 4397 let mut vfio_path = workload_path.clone(); 4398 vfio_path.push("vfio"); 4399 4400 let mut cloud_init_vfio_base_path = vfio_path.clone(); 4401 cloud_init_vfio_base_path.push("cloudinit.img"); 4402 4403 // We copy our cloudinit into the vfio mount point, for the nested 4404 // cloud-hypervisor guest to use. 4405 rate_limited_copy( 4406 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 4407 &cloud_init_vfio_base_path, 4408 ) 4409 .expect("copying of cloud-init disk failed"); 4410 4411 let mut vfio_disk_path = workload_path.clone(); 4412 vfio_disk_path.push("vfio.img"); 4413 4414 // Create the vfio disk image 4415 let output = Command::new("mkfs.ext4") 4416 .arg("-d") 4417 .arg(vfio_path.to_str().unwrap()) 4418 .arg(vfio_disk_path.to_str().unwrap()) 4419 .arg("2g") 4420 .output() 4421 .unwrap(); 4422 if !output.status.success() { 4423 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 4424 panic!("mkfs.ext4 command generated an error"); 4425 } 4426 4427 let mut blk_file_path = workload_path; 4428 blk_file_path.push("blk.img"); 4429 4430 let vfio_tap0 = "vfio-tap0"; 4431 let vfio_tap1 = "vfio-tap1"; 4432 let vfio_tap2 = "vfio-tap2"; 4433 let vfio_tap3 = "vfio-tap3"; 4434 4435 let mut child = GuestCommand::new(&guest) 4436 .args(["--cpus", "boot=4"]) 4437 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 4438 .args(["--kernel", kernel_path.to_str().unwrap()]) 4439 .args([ 4440 "--disk", 4441 format!( 4442 "path={}", 4443 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4444 ) 4445 .as_str(), 4446 format!( 4447 "path={}", 4448 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4449 ) 4450 .as_str(), 4451 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 4452 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 4453 ]) 4454 .args([ 4455 "--cmdline", 4456 format!( 4457 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 4458 ) 4459 .as_str(), 4460 ]) 4461 .args([ 4462 "--net", 4463 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 4464 format!( 4465 "tap={},mac={},iommu=on", 4466 vfio_tap1, guest.network.l2_guest_mac1 4467 ) 4468 .as_str(), 4469 format!( 4470 "tap={},mac={},iommu=on", 4471 vfio_tap2, guest.network.l2_guest_mac2 4472 ) 4473 .as_str(), 4474 format!( 4475 "tap={},mac={},iommu=on", 4476 vfio_tap3, guest.network.l2_guest_mac3 4477 ) 4478 .as_str(), 4479 ]) 4480 .capture_output() 4481 .spawn() 4482 .unwrap(); 4483 4484 thread::sleep(std::time::Duration::new(30, 0)); 4485 4486 let r = std::panic::catch_unwind(|| { 4487 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 4488 thread::sleep(std::time::Duration::new(120, 0)); 4489 4490 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 4491 // added to its kernel command line. 4492 // Let's ssh into it and verify that it's there. If it is it means 4493 // we're in the right guest (The L2 one) because the QEMU L1 guest 4494 // does not have this command line tag. 4495 assert!(check_matched_lines_count( 4496 guest.ssh_command_l2_1("cat /proc/cmdline").unwrap().trim(), 4497 vec!["VFIOTAG"], 4498 1 4499 )); 4500 4501 // Let's also verify from the second virtio-net device passed to 4502 // the L2 VM. 4503 assert!(check_matched_lines_count( 4504 guest.ssh_command_l2_2("cat /proc/cmdline").unwrap().trim(), 4505 vec!["VFIOTAG"], 4506 1 4507 )); 4508 4509 // Check the amount of PCI devices appearing in L2 VM. 4510 assert!(check_lines_count( 4511 guest 4512 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4513 .unwrap() 4514 .trim(), 4515 8 4516 )); 4517 4518 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 4519 assert!(check_matched_lines_count( 4520 guest.ssh_command_l2_1("lsblk").unwrap().trim(), 4521 vec!["vdc", "16M"], 4522 1 4523 )); 4524 4525 // Hotplug an extra virtio-net device through L2 VM. 4526 guest 4527 .ssh_command_l1( 4528 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 4529 ) 4530 .unwrap(); 4531 guest 4532 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 4533 .unwrap(); 4534 let vfio_hotplug_output = guest 4535 .ssh_command_l1( 4536 "sudo /mnt/ch-remote \ 4537 --api-socket=/tmp/ch_api.sock \ 4538 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 4539 ) 4540 .unwrap(); 4541 assert!(check_matched_lines_count( 4542 vfio_hotplug_output.trim(), 4543 vec!["{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}"], 4544 1 4545 )); 4546 4547 thread::sleep(std::time::Duration::new(10, 0)); 4548 4549 // Let's also verify from the third virtio-net device passed to 4550 // the L2 VM. This third device has been hotplugged through the L2 4551 // VM, so this is our way to validate hotplug works for VFIO PCI. 4552 assert!(check_matched_lines_count( 4553 guest.ssh_command_l2_3("cat /proc/cmdline").unwrap().trim(), 4554 vec!["VFIOTAG"], 4555 1 4556 )); 4557 4558 // Check the amount of PCI devices appearing in L2 VM. 4559 // There should be one more device than before, raising the count 4560 // up to 9 PCI devices. 4561 assert!(check_lines_count( 4562 guest 4563 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4564 .unwrap() 4565 .trim(), 4566 9 4567 )); 4568 4569 // Let's now verify that we can correctly remove the virtio-net 4570 // device through the "remove-device" command responsible for 4571 // unplugging VFIO devices. 4572 guest 4573 .ssh_command_l1( 4574 "sudo /mnt/ch-remote \ 4575 --api-socket=/tmp/ch_api.sock \ 4576 remove-device vfio123", 4577 ) 4578 .unwrap(); 4579 thread::sleep(std::time::Duration::new(10, 0)); 4580 4581 // Check the amount of PCI devices appearing in L2 VM is back down 4582 // to 8 devices. 4583 assert!(check_lines_count( 4584 guest 4585 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4586 .unwrap() 4587 .trim(), 4588 8 4589 )); 4590 4591 // Perform memory hotplug in L2 and validate the memory is showing 4592 // up as expected. In order to check, we will use the virtio-net 4593 // device already passed through L2 as a VFIO device, this will 4594 // verify that VFIO devices are functional with memory hotplug. 4595 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 4596 guest 4597 .ssh_command_l2_1( 4598 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 4599 ) 4600 .unwrap(); 4601 guest 4602 .ssh_command_l1( 4603 "sudo /mnt/ch-remote \ 4604 --api-socket=/tmp/ch_api.sock \ 4605 resize --memory=1073741824", 4606 ) 4607 .unwrap(); 4608 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 4609 }); 4610 4611 kill_child(&mut child); 4612 let output = child.wait_with_output().unwrap(); 4613 4614 cleanup_vfio_network_interfaces(); 4615 4616 handle_child_output(r, &output); 4617 } 4618 4619 #[test] 4620 fn test_direct_kernel_boot_noacpi() { 4621 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4622 let guest = Guest::new(Box::new(focal)); 4623 4624 let kernel_path = direct_kernel_boot_path(); 4625 4626 let mut child = GuestCommand::new(&guest) 4627 .args(["--cpus", "boot=1"]) 4628 .args(["--memory", "size=512M"]) 4629 .args(["--kernel", kernel_path.to_str().unwrap()]) 4630 .args([ 4631 "--cmdline", 4632 format!("{DIRECT_KERNEL_BOOT_CMDLINE} acpi=off").as_str(), 4633 ]) 4634 .default_disks() 4635 .default_net() 4636 .capture_output() 4637 .spawn() 4638 .unwrap(); 4639 4640 let r = std::panic::catch_unwind(|| { 4641 guest.wait_vm_boot(None).unwrap(); 4642 4643 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4644 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4645 }); 4646 4647 kill_child(&mut child); 4648 let output = child.wait_with_output().unwrap(); 4649 4650 handle_child_output(r, &output); 4651 } 4652 4653 #[test] 4654 fn test_virtio_vsock() { 4655 _test_virtio_vsock(false) 4656 } 4657 4658 #[test] 4659 fn test_virtio_vsock_hotplug() { 4660 _test_virtio_vsock(true); 4661 } 4662 4663 #[test] 4664 fn test_api_http_shutdown() { 4665 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4666 let guest = Guest::new(Box::new(focal)); 4667 4668 _test_api_shutdown(TargetApi::new_http_api(&guest.tmp_dir), guest) 4669 } 4670 4671 #[test] 4672 fn test_api_http_delete() { 4673 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4674 let guest = Guest::new(Box::new(focal)); 4675 4676 _test_api_delete(TargetApi::new_http_api(&guest.tmp_dir), guest); 4677 } 4678 4679 #[test] 4680 fn test_api_http_pause_resume() { 4681 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4682 let guest = Guest::new(Box::new(focal)); 4683 4684 _test_api_pause_resume(TargetApi::new_http_api(&guest.tmp_dir), guest) 4685 } 4686 4687 #[test] 4688 fn test_api_http_create_boot() { 4689 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4690 let guest = Guest::new(Box::new(focal)); 4691 4692 _test_api_create_boot(TargetApi::new_http_api(&guest.tmp_dir), guest) 4693 } 4694 4695 #[test] 4696 fn test_virtio_iommu() { 4697 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4698 } 4699 4700 #[test] 4701 // We cannot force the software running in the guest to reprogram the BAR 4702 // with some different addresses, but we have a reliable way of testing it 4703 // with a standard Linux kernel. 4704 // By removing a device from the PCI tree, and then rescanning the tree, 4705 // Linux consistently chooses to reorganize the PCI device BARs to other 4706 // locations in the guest address space. 4707 // This test creates a dedicated PCI network device to be checked as being 4708 // properly probed first, then removing it, and adding it again by doing a 4709 // rescan. 4710 fn test_pci_bar_reprogramming() { 4711 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4712 let guest = Guest::new(Box::new(focal)); 4713 4714 #[cfg(target_arch = "x86_64")] 4715 let kernel_path = direct_kernel_boot_path(); 4716 #[cfg(target_arch = "aarch64")] 4717 let kernel_path = edk2_path(); 4718 4719 let mut child = GuestCommand::new(&guest) 4720 .args(["--cpus", "boot=1"]) 4721 .args(["--memory", "size=512M"]) 4722 .args(["--kernel", kernel_path.to_str().unwrap()]) 4723 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4724 .default_disks() 4725 .args([ 4726 "--net", 4727 guest.default_net_string().as_str(), 4728 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4729 ]) 4730 .capture_output() 4731 .spawn() 4732 .unwrap(); 4733 4734 let r = std::panic::catch_unwind(|| { 4735 guest.wait_vm_boot(None).unwrap(); 4736 4737 // 2 network interfaces + default localhost ==> 3 interfaces 4738 assert_eq!( 4739 guest 4740 .ssh_command("ip -o link | wc -l") 4741 .unwrap() 4742 .trim() 4743 .parse::<u32>() 4744 .unwrap_or_default(), 4745 3 4746 ); 4747 4748 let init_bar_addr = guest 4749 .ssh_command( 4750 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4751 ) 4752 .unwrap(); 4753 4754 // Remove the PCI device 4755 guest 4756 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4757 .unwrap(); 4758 4759 // Only 1 network interface left + default localhost ==> 2 interfaces 4760 assert_eq!( 4761 guest 4762 .ssh_command("ip -o link | wc -l") 4763 .unwrap() 4764 .trim() 4765 .parse::<u32>() 4766 .unwrap_or_default(), 4767 2 4768 ); 4769 4770 // Remove the PCI device 4771 guest 4772 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4773 .unwrap(); 4774 4775 // Back to 2 network interface + default localhost ==> 3 interfaces 4776 assert_eq!( 4777 guest 4778 .ssh_command("ip -o link | wc -l") 4779 .unwrap() 4780 .trim() 4781 .parse::<u32>() 4782 .unwrap_or_default(), 4783 3 4784 ); 4785 4786 let new_bar_addr = guest 4787 .ssh_command( 4788 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4789 ) 4790 .unwrap(); 4791 4792 // Let's compare the BAR addresses for our virtio-net device. 4793 // They should be different as we expect the BAR reprogramming 4794 // to have happened. 4795 assert_ne!(init_bar_addr, new_bar_addr); 4796 }); 4797 4798 kill_child(&mut child); 4799 let output = child.wait_with_output().unwrap(); 4800 4801 handle_child_output(r, &output); 4802 } 4803 4804 #[test] 4805 fn test_memory_mergeable_off() { 4806 test_memory_mergeable(false) 4807 } 4808 4809 #[test] 4810 #[cfg(target_arch = "x86_64")] 4811 fn test_cpu_hotplug() { 4812 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4813 let guest = Guest::new(Box::new(focal)); 4814 let api_socket = temp_api_path(&guest.tmp_dir); 4815 4816 let kernel_path = direct_kernel_boot_path(); 4817 4818 let mut child = GuestCommand::new(&guest) 4819 .args(["--cpus", "boot=2,max=4"]) 4820 .args(["--memory", "size=512M"]) 4821 .args(["--kernel", kernel_path.to_str().unwrap()]) 4822 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4823 .default_disks() 4824 .default_net() 4825 .args(["--api-socket", &api_socket]) 4826 .capture_output() 4827 .spawn() 4828 .unwrap(); 4829 4830 let r = std::panic::catch_unwind(|| { 4831 guest.wait_vm_boot(None).unwrap(); 4832 4833 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4834 4835 // Resize the VM 4836 let desired_vcpus = 4; 4837 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4838 4839 guest 4840 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4841 .unwrap(); 4842 guest 4843 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4844 .unwrap(); 4845 thread::sleep(std::time::Duration::new(10, 0)); 4846 assert_eq!( 4847 guest.get_cpu_count().unwrap_or_default(), 4848 u32::from(desired_vcpus) 4849 ); 4850 4851 guest.reboot_linux(0, None); 4852 4853 assert_eq!( 4854 guest.get_cpu_count().unwrap_or_default(), 4855 u32::from(desired_vcpus) 4856 ); 4857 4858 // Resize the VM 4859 let desired_vcpus = 2; 4860 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4861 4862 thread::sleep(std::time::Duration::new(10, 0)); 4863 assert_eq!( 4864 guest.get_cpu_count().unwrap_or_default(), 4865 u32::from(desired_vcpus) 4866 ); 4867 4868 // Resize the VM back up to 4 4869 let desired_vcpus = 4; 4870 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4871 4872 guest 4873 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4874 .unwrap(); 4875 guest 4876 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4877 .unwrap(); 4878 thread::sleep(std::time::Duration::new(10, 0)); 4879 assert_eq!( 4880 guest.get_cpu_count().unwrap_or_default(), 4881 u32::from(desired_vcpus) 4882 ); 4883 }); 4884 4885 kill_child(&mut child); 4886 let output = child.wait_with_output().unwrap(); 4887 4888 handle_child_output(r, &output); 4889 } 4890 4891 #[test] 4892 fn test_memory_hotplug() { 4893 #[cfg(target_arch = "aarch64")] 4894 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4895 #[cfg(target_arch = "x86_64")] 4896 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4897 let focal = UbuntuDiskConfig::new(focal_image); 4898 let guest = Guest::new(Box::new(focal)); 4899 let api_socket = temp_api_path(&guest.tmp_dir); 4900 4901 #[cfg(target_arch = "aarch64")] 4902 let kernel_path = edk2_path(); 4903 #[cfg(target_arch = "x86_64")] 4904 let kernel_path = direct_kernel_boot_path(); 4905 4906 let mut child = GuestCommand::new(&guest) 4907 .args(["--cpus", "boot=2,max=4"]) 4908 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4909 .args(["--kernel", kernel_path.to_str().unwrap()]) 4910 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4911 .default_disks() 4912 .default_net() 4913 .args(["--balloon", "size=0"]) 4914 .args(["--api-socket", &api_socket]) 4915 .capture_output() 4916 .spawn() 4917 .unwrap(); 4918 4919 let r = std::panic::catch_unwind(|| { 4920 guest.wait_vm_boot(None).unwrap(); 4921 4922 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4923 4924 guest.enable_memory_hotplug(); 4925 4926 // Add RAM to the VM 4927 let desired_ram = 1024 << 20; 4928 resize_command(&api_socket, None, Some(desired_ram), None, None); 4929 4930 thread::sleep(std::time::Duration::new(10, 0)); 4931 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4932 4933 // Use balloon to remove RAM from the VM 4934 let desired_balloon = 512 << 20; 4935 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4936 4937 thread::sleep(std::time::Duration::new(10, 0)); 4938 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4939 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4940 4941 guest.reboot_linux(0, None); 4942 4943 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4944 4945 // Use balloon add RAM to the VM 4946 let desired_balloon = 0; 4947 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4948 4949 thread::sleep(std::time::Duration::new(10, 0)); 4950 4951 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4952 4953 guest.enable_memory_hotplug(); 4954 4955 // Add RAM to the VM 4956 let desired_ram = 2048 << 20; 4957 resize_command(&api_socket, None, Some(desired_ram), None, None); 4958 4959 thread::sleep(std::time::Duration::new(10, 0)); 4960 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4961 4962 // Remove RAM to the VM (only applies after reboot) 4963 let desired_ram = 1024 << 20; 4964 resize_command(&api_socket, None, Some(desired_ram), None, None); 4965 4966 guest.reboot_linux(1, None); 4967 4968 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4969 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4970 }); 4971 4972 kill_child(&mut child); 4973 let output = child.wait_with_output().unwrap(); 4974 4975 handle_child_output(r, &output); 4976 } 4977 4978 #[test] 4979 #[cfg(not(feature = "mshv"))] 4980 fn test_virtio_mem() { 4981 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4982 let guest = Guest::new(Box::new(focal)); 4983 let api_socket = temp_api_path(&guest.tmp_dir); 4984 4985 let kernel_path = direct_kernel_boot_path(); 4986 4987 let mut child = GuestCommand::new(&guest) 4988 .args(["--cpus", "boot=2,max=4"]) 4989 .args([ 4990 "--memory", 4991 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4992 ]) 4993 .args(["--kernel", kernel_path.to_str().unwrap()]) 4994 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4995 .default_disks() 4996 .default_net() 4997 .args(["--api-socket", &api_socket]) 4998 .capture_output() 4999 .spawn() 5000 .unwrap(); 5001 5002 let r = std::panic::catch_unwind(|| { 5003 guest.wait_vm_boot(None).unwrap(); 5004 5005 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5006 5007 guest.enable_memory_hotplug(); 5008 5009 // Add RAM to the VM 5010 let desired_ram = 1024 << 20; 5011 resize_command(&api_socket, None, Some(desired_ram), None, None); 5012 5013 thread::sleep(std::time::Duration::new(10, 0)); 5014 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5015 5016 // Add RAM to the VM 5017 let desired_ram = 2048 << 20; 5018 resize_command(&api_socket, None, Some(desired_ram), None, None); 5019 5020 thread::sleep(std::time::Duration::new(10, 0)); 5021 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 5022 5023 // Remove RAM from the VM 5024 let desired_ram = 1024 << 20; 5025 resize_command(&api_socket, None, Some(desired_ram), None, None); 5026 5027 thread::sleep(std::time::Duration::new(10, 0)); 5028 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5029 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5030 5031 guest.reboot_linux(0, None); 5032 5033 // Check the amount of memory after reboot is 1GiB 5034 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5035 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5036 5037 // Check we can still resize to 512MiB 5038 let desired_ram = 512 << 20; 5039 resize_command(&api_socket, None, Some(desired_ram), None, None); 5040 thread::sleep(std::time::Duration::new(10, 0)); 5041 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5042 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 5043 }); 5044 5045 kill_child(&mut child); 5046 let output = child.wait_with_output().unwrap(); 5047 5048 handle_child_output(r, &output); 5049 } 5050 5051 #[test] 5052 #[cfg(target_arch = "x86_64")] 5053 #[cfg(not(feature = "mshv"))] 5054 // Test both vCPU and memory resizing together 5055 fn test_resize() { 5056 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5057 let guest = Guest::new(Box::new(focal)); 5058 let api_socket = temp_api_path(&guest.tmp_dir); 5059 5060 let kernel_path = direct_kernel_boot_path(); 5061 5062 let mut child = GuestCommand::new(&guest) 5063 .args(["--cpus", "boot=2,max=4"]) 5064 .args(["--memory", "size=512M,hotplug_size=8192M"]) 5065 .args(["--kernel", kernel_path.to_str().unwrap()]) 5066 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5067 .default_disks() 5068 .default_net() 5069 .args(["--api-socket", &api_socket]) 5070 .capture_output() 5071 .spawn() 5072 .unwrap(); 5073 5074 let r = std::panic::catch_unwind(|| { 5075 guest.wait_vm_boot(None).unwrap(); 5076 5077 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 5078 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5079 5080 guest.enable_memory_hotplug(); 5081 5082 // Resize the VM 5083 let desired_vcpus = 4; 5084 let desired_ram = 1024 << 20; 5085 resize_command( 5086 &api_socket, 5087 Some(desired_vcpus), 5088 Some(desired_ram), 5089 None, 5090 None, 5091 ); 5092 5093 guest 5094 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 5095 .unwrap(); 5096 guest 5097 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 5098 .unwrap(); 5099 thread::sleep(std::time::Duration::new(10, 0)); 5100 assert_eq!( 5101 guest.get_cpu_count().unwrap_or_default(), 5102 u32::from(desired_vcpus) 5103 ); 5104 5105 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5106 }); 5107 5108 kill_child(&mut child); 5109 let output = child.wait_with_output().unwrap(); 5110 5111 handle_child_output(r, &output); 5112 } 5113 5114 #[test] 5115 fn test_memory_overhead() { 5116 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5117 let guest = Guest::new(Box::new(focal)); 5118 5119 let kernel_path = direct_kernel_boot_path(); 5120 5121 let guest_memory_size_kb = 512 * 1024; 5122 5123 let mut child = GuestCommand::new(&guest) 5124 .args(["--cpus", "boot=1"]) 5125 .args(["--memory", format!("size={guest_memory_size_kb}K").as_str()]) 5126 .args(["--kernel", kernel_path.to_str().unwrap()]) 5127 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5128 .default_net() 5129 .default_disks() 5130 .capture_output() 5131 .spawn() 5132 .unwrap(); 5133 5134 guest.wait_vm_boot(None).unwrap(); 5135 5136 let r = std::panic::catch_unwind(|| { 5137 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 5138 eprintln!("Guest memory overhead: {overhead} vs {MAXIMUM_VMM_OVERHEAD_KB}"); 5139 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 5140 }); 5141 5142 kill_child(&mut child); 5143 let output = child.wait_with_output().unwrap(); 5144 5145 handle_child_output(r, &output); 5146 } 5147 5148 #[test] 5149 #[cfg(target_arch = "x86_64")] 5150 // This test runs a guest with Landlock enabled and hotplugs a new disk. As 5151 // the path for the hotplug disk is not pre-added to Landlock rules, this 5152 // the test will result in a failure. 5153 fn test_landlock() { 5154 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5155 let guest = Guest::new(Box::new(focal)); 5156 5157 #[cfg(target_arch = "x86_64")] 5158 let kernel_path = direct_kernel_boot_path(); 5159 #[cfg(target_arch = "aarch64")] 5160 let kernel_path = edk2_path(); 5161 5162 let api_socket = temp_api_path(&guest.tmp_dir); 5163 5164 let mut child = GuestCommand::new(&guest) 5165 .args(["--api-socket", &api_socket]) 5166 .args(["--cpus", "boot=1"]) 5167 .args(["--memory", "size=512M"]) 5168 .args(["--kernel", kernel_path.to_str().unwrap()]) 5169 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5170 .args(["--landlock"]) 5171 .default_disks() 5172 .default_net() 5173 .capture_output() 5174 .spawn() 5175 .unwrap(); 5176 5177 let r = std::panic::catch_unwind(|| { 5178 guest.wait_vm_boot(None).unwrap(); 5179 5180 // Check /dev/vdc is not there 5181 assert_eq!( 5182 guest 5183 .ssh_command("lsblk | grep -c vdc.*16M || true") 5184 .unwrap() 5185 .trim() 5186 .parse::<u32>() 5187 .unwrap_or(1), 5188 0 5189 ); 5190 5191 // Now let's add the extra disk. 5192 let mut blk_file_path = dirs::home_dir().unwrap(); 5193 blk_file_path.push("workloads"); 5194 blk_file_path.push("blk.img"); 5195 // As the path to the hotplug disk is not pre-added, this remote 5196 // command will fail. 5197 assert!(!remote_command( 5198 &api_socket, 5199 "add-disk", 5200 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5201 )); 5202 }); 5203 5204 let _ = child.kill(); 5205 let output = child.wait_with_output().unwrap(); 5206 5207 handle_child_output(r, &output); 5208 } 5209 5210 fn _test_disk_hotplug(landlock_enabled: bool) { 5211 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5212 let guest = Guest::new(Box::new(focal)); 5213 5214 #[cfg(target_arch = "x86_64")] 5215 let kernel_path = direct_kernel_boot_path(); 5216 #[cfg(target_arch = "aarch64")] 5217 let kernel_path = edk2_path(); 5218 5219 let api_socket = temp_api_path(&guest.tmp_dir); 5220 5221 let mut blk_file_path = dirs::home_dir().unwrap(); 5222 blk_file_path.push("workloads"); 5223 blk_file_path.push("blk.img"); 5224 5225 let mut cmd = GuestCommand::new(&guest); 5226 if landlock_enabled { 5227 cmd.args(["--landlock"]).args([ 5228 "--landlock-rules", 5229 format!("path={:?},access=rw", blk_file_path).as_str(), 5230 ]); 5231 } 5232 5233 cmd.args(["--api-socket", &api_socket]) 5234 .args(["--cpus", "boot=1"]) 5235 .args(["--memory", "size=512M"]) 5236 .args(["--kernel", kernel_path.to_str().unwrap()]) 5237 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5238 .default_disks() 5239 .default_net() 5240 .capture_output(); 5241 5242 let mut child = cmd.spawn().unwrap(); 5243 5244 let r = std::panic::catch_unwind(|| { 5245 guest.wait_vm_boot(None).unwrap(); 5246 5247 // Check /dev/vdc is not there 5248 assert_eq!( 5249 guest 5250 .ssh_command("lsblk | grep -c vdc.*16M || true") 5251 .unwrap() 5252 .trim() 5253 .parse::<u32>() 5254 .unwrap_or(1), 5255 0 5256 ); 5257 5258 // Now let's add the extra disk. 5259 let (cmd_success, cmd_output) = remote_command_w_output( 5260 &api_socket, 5261 "add-disk", 5262 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5263 ); 5264 assert!(cmd_success); 5265 assert!(String::from_utf8_lossy(&cmd_output) 5266 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5267 5268 thread::sleep(std::time::Duration::new(10, 0)); 5269 5270 // Check that /dev/vdc exists and the block size is 16M. 5271 assert_eq!( 5272 guest 5273 .ssh_command("lsblk | grep vdc | grep -c 16M") 5274 .unwrap() 5275 .trim() 5276 .parse::<u32>() 5277 .unwrap_or_default(), 5278 1 5279 ); 5280 // And check the block device can be read. 5281 guest 5282 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5283 .unwrap(); 5284 5285 // Let's remove it the extra disk. 5286 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5287 thread::sleep(std::time::Duration::new(5, 0)); 5288 // And check /dev/vdc is not there 5289 assert_eq!( 5290 guest 5291 .ssh_command("lsblk | grep -c vdc.*16M || true") 5292 .unwrap() 5293 .trim() 5294 .parse::<u32>() 5295 .unwrap_or(1), 5296 0 5297 ); 5298 5299 // And add it back to validate unplug did work correctly. 5300 let (cmd_success, cmd_output) = remote_command_w_output( 5301 &api_socket, 5302 "add-disk", 5303 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5304 ); 5305 assert!(cmd_success); 5306 assert!(String::from_utf8_lossy(&cmd_output) 5307 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5308 5309 thread::sleep(std::time::Duration::new(10, 0)); 5310 5311 // Check that /dev/vdc exists and the block size is 16M. 5312 assert_eq!( 5313 guest 5314 .ssh_command("lsblk | grep vdc | grep -c 16M") 5315 .unwrap() 5316 .trim() 5317 .parse::<u32>() 5318 .unwrap_or_default(), 5319 1 5320 ); 5321 // And check the block device can be read. 5322 guest 5323 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5324 .unwrap(); 5325 5326 // Reboot the VM. 5327 guest.reboot_linux(0, None); 5328 5329 // Check still there after reboot 5330 assert_eq!( 5331 guest 5332 .ssh_command("lsblk | grep vdc | grep -c 16M") 5333 .unwrap() 5334 .trim() 5335 .parse::<u32>() 5336 .unwrap_or_default(), 5337 1 5338 ); 5339 5340 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5341 5342 thread::sleep(std::time::Duration::new(20, 0)); 5343 5344 // Check device has gone away 5345 assert_eq!( 5346 guest 5347 .ssh_command("lsblk | grep -c vdc.*16M || true") 5348 .unwrap() 5349 .trim() 5350 .parse::<u32>() 5351 .unwrap_or(1), 5352 0 5353 ); 5354 5355 guest.reboot_linux(1, None); 5356 5357 // Check device still absent 5358 assert_eq!( 5359 guest 5360 .ssh_command("lsblk | grep -c vdc.*16M || true") 5361 .unwrap() 5362 .trim() 5363 .parse::<u32>() 5364 .unwrap_or(1), 5365 0 5366 ); 5367 }); 5368 5369 kill_child(&mut child); 5370 let output = child.wait_with_output().unwrap(); 5371 5372 handle_child_output(r, &output); 5373 } 5374 5375 #[test] 5376 fn test_disk_hotplug() { 5377 _test_disk_hotplug(false) 5378 } 5379 5380 #[test] 5381 #[cfg(target_arch = "x86_64")] 5382 fn test_disk_hotplug_with_landlock() { 5383 _test_disk_hotplug(true) 5384 } 5385 5386 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 5387 const LOOP_CONFIGURE: u64 = 0x4c0a; 5388 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 5389 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 5390 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 5391 5392 #[repr(C)] 5393 struct LoopInfo64 { 5394 lo_device: u64, 5395 lo_inode: u64, 5396 lo_rdevice: u64, 5397 lo_offset: u64, 5398 lo_sizelimit: u64, 5399 lo_number: u32, 5400 lo_encrypt_type: u32, 5401 lo_encrypt_key_size: u32, 5402 lo_flags: u32, 5403 lo_file_name: [u8; 64], 5404 lo_crypt_name: [u8; 64], 5405 lo_encrypt_key: [u8; 32], 5406 lo_init: [u64; 2], 5407 } 5408 5409 impl Default for LoopInfo64 { 5410 fn default() -> Self { 5411 LoopInfo64 { 5412 lo_device: 0, 5413 lo_inode: 0, 5414 lo_rdevice: 0, 5415 lo_offset: 0, 5416 lo_sizelimit: 0, 5417 lo_number: 0, 5418 lo_encrypt_type: 0, 5419 lo_encrypt_key_size: 0, 5420 lo_flags: 0, 5421 lo_file_name: [0; 64], 5422 lo_crypt_name: [0; 64], 5423 lo_encrypt_key: [0; 32], 5424 lo_init: [0; 2], 5425 } 5426 } 5427 } 5428 5429 #[derive(Default)] 5430 #[repr(C)] 5431 struct LoopConfig { 5432 fd: u32, 5433 block_size: u32, 5434 info: LoopInfo64, 5435 _reserved: [u64; 8], 5436 } 5437 5438 // Open loop-control device 5439 let loop_ctl_file = OpenOptions::new() 5440 .read(true) 5441 .write(true) 5442 .open(LOOP_CTL_PATH) 5443 .unwrap(); 5444 5445 // Request a free loop device 5446 let loop_device_number = 5447 unsafe { libc::ioctl(loop_ctl_file.as_raw_fd(), LOOP_CTL_GET_FREE as _) }; 5448 5449 if loop_device_number < 0 { 5450 panic!("Couldn't find a free loop device"); 5451 } 5452 5453 // Create loop device path 5454 let loop_device_path = format!("{LOOP_DEVICE_PREFIX}{loop_device_number}"); 5455 5456 // Open loop device 5457 let loop_device_file = OpenOptions::new() 5458 .read(true) 5459 .write(true) 5460 .open(&loop_device_path) 5461 .unwrap(); 5462 5463 // Open backing file 5464 let backing_file = OpenOptions::new() 5465 .read(true) 5466 .write(true) 5467 .open(backing_file_path) 5468 .unwrap(); 5469 5470 let loop_config = LoopConfig { 5471 fd: backing_file.as_raw_fd() as u32, 5472 block_size, 5473 ..Default::default() 5474 }; 5475 5476 for i in 0..num_retries { 5477 let ret = unsafe { 5478 libc::ioctl( 5479 loop_device_file.as_raw_fd(), 5480 LOOP_CONFIGURE as _, 5481 &loop_config, 5482 ) 5483 }; 5484 if ret != 0 { 5485 if i < num_retries - 1 { 5486 println!( 5487 "Iteration {}: Failed to configure the loop device {}: {}", 5488 i, 5489 loop_device_path, 5490 std::io::Error::last_os_error() 5491 ); 5492 } else { 5493 panic!( 5494 "Failed {} times trying to configure the loop device {}: {}", 5495 num_retries, 5496 loop_device_path, 5497 std::io::Error::last_os_error() 5498 ); 5499 } 5500 } else { 5501 break; 5502 } 5503 5504 // Wait for a bit before retrying 5505 thread::sleep(std::time::Duration::new(5, 0)); 5506 } 5507 5508 loop_device_path 5509 } 5510 5511 #[test] 5512 fn test_virtio_block_topology() { 5513 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5514 let guest = Guest::new(Box::new(focal)); 5515 5516 let kernel_path = direct_kernel_boot_path(); 5517 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5518 5519 let output = exec_host_command_output( 5520 format!( 5521 "qemu-img create -f raw {} 16M", 5522 test_disk_path.to_str().unwrap() 5523 ) 5524 .as_str(), 5525 ); 5526 if !output.status.success() { 5527 let stdout = String::from_utf8_lossy(&output.stdout); 5528 let stderr = String::from_utf8_lossy(&output.stderr); 5529 panic!("qemu-img command failed\nstdout\n{stdout}\nstderr\n{stderr}"); 5530 } 5531 5532 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5533 5534 let mut child = GuestCommand::new(&guest) 5535 .args(["--cpus", "boot=1"]) 5536 .args(["--memory", "size=512M"]) 5537 .args(["--kernel", kernel_path.to_str().unwrap()]) 5538 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5539 .args([ 5540 "--disk", 5541 format!( 5542 "path={}", 5543 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5544 ) 5545 .as_str(), 5546 format!( 5547 "path={}", 5548 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5549 ) 5550 .as_str(), 5551 format!("path={}", &loop_dev).as_str(), 5552 ]) 5553 .default_net() 5554 .capture_output() 5555 .spawn() 5556 .unwrap(); 5557 5558 let r = std::panic::catch_unwind(|| { 5559 guest.wait_vm_boot(None).unwrap(); 5560 5561 // MIN-IO column 5562 assert_eq!( 5563 guest 5564 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5565 .unwrap() 5566 .trim() 5567 .parse::<u32>() 5568 .unwrap_or_default(), 5569 4096 5570 ); 5571 // PHY-SEC column 5572 assert_eq!( 5573 guest 5574 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5575 .unwrap() 5576 .trim() 5577 .parse::<u32>() 5578 .unwrap_or_default(), 5579 4096 5580 ); 5581 // LOG-SEC column 5582 assert_eq!( 5583 guest 5584 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5585 .unwrap() 5586 .trim() 5587 .parse::<u32>() 5588 .unwrap_or_default(), 5589 4096 5590 ); 5591 }); 5592 5593 kill_child(&mut child); 5594 let output = child.wait_with_output().unwrap(); 5595 5596 handle_child_output(r, &output); 5597 5598 Command::new("losetup") 5599 .args(["-d", &loop_dev]) 5600 .output() 5601 .expect("loop device not found"); 5602 } 5603 5604 #[test] 5605 fn test_virtio_balloon_deflate_on_oom() { 5606 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5607 let guest = Guest::new(Box::new(focal)); 5608 5609 let kernel_path = direct_kernel_boot_path(); 5610 5611 let api_socket = temp_api_path(&guest.tmp_dir); 5612 5613 //Let's start a 4G guest with balloon occupied 2G memory 5614 let mut child = GuestCommand::new(&guest) 5615 .args(["--api-socket", &api_socket]) 5616 .args(["--cpus", "boot=1"]) 5617 .args(["--memory", "size=4G"]) 5618 .args(["--kernel", kernel_path.to_str().unwrap()]) 5619 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5620 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5621 .default_disks() 5622 .default_net() 5623 .capture_output() 5624 .spawn() 5625 .unwrap(); 5626 5627 let r = std::panic::catch_unwind(|| { 5628 guest.wait_vm_boot(None).unwrap(); 5629 5630 // Wait for balloon memory's initialization and check its size. 5631 // The virtio-balloon driver might take a few seconds to report the 5632 // balloon effective size back to the VMM. 5633 thread::sleep(std::time::Duration::new(20, 0)); 5634 5635 let orig_balloon = balloon_size(&api_socket); 5636 println!("The original balloon memory size is {orig_balloon} bytes"); 5637 assert!(orig_balloon == 2147483648); 5638 5639 // Two steps to verify if the 'deflate_on_oom' parameter works. 5640 // 1st: run a command to trigger an OOM in the guest. 5641 guest 5642 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5643 .unwrap(); 5644 5645 // Give some time for the OOM to happen in the guest and be reported 5646 // back to the host. 5647 thread::sleep(std::time::Duration::new(20, 0)); 5648 5649 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5650 let deflated_balloon = balloon_size(&api_socket); 5651 println!("After deflating, balloon memory size is {deflated_balloon} bytes"); 5652 // Verify the balloon size deflated 5653 assert!(deflated_balloon < 2147483648); 5654 }); 5655 5656 kill_child(&mut child); 5657 let output = child.wait_with_output().unwrap(); 5658 5659 handle_child_output(r, &output); 5660 } 5661 5662 #[test] 5663 #[cfg(not(feature = "mshv"))] 5664 fn test_virtio_balloon_free_page_reporting() { 5665 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5666 let guest = Guest::new(Box::new(focal)); 5667 5668 //Let's start a 4G guest with balloon occupied 2G memory 5669 let mut child = GuestCommand::new(&guest) 5670 .args(["--cpus", "boot=1"]) 5671 .args(["--memory", "size=4G"]) 5672 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5673 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5674 .args(["--balloon", "size=0,free_page_reporting=on"]) 5675 .default_disks() 5676 .default_net() 5677 .capture_output() 5678 .spawn() 5679 .unwrap(); 5680 5681 let pid = child.id(); 5682 let r = std::panic::catch_unwind(|| { 5683 guest.wait_vm_boot(None).unwrap(); 5684 5685 // Check the initial RSS is less than 1GiB 5686 let rss = process_rss_kib(pid); 5687 println!("RSS {rss} < 1048576"); 5688 assert!(rss < 1048576); 5689 5690 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5691 // seconds 5692 let guest_ip = guest.network.guest_ip.clone(); 5693 thread::spawn(move || { 5694 ssh_command_ip( 5695 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5696 &guest_ip, 5697 DEFAULT_SSH_RETRIES, 5698 DEFAULT_SSH_TIMEOUT, 5699 ) 5700 .unwrap(); 5701 }); 5702 5703 // Wait for 50 seconds to make sure the stress command is consuming 5704 // the expected amount of memory. 5705 thread::sleep(std::time::Duration::new(50, 0)); 5706 let rss = process_rss_kib(pid); 5707 println!("RSS {rss} >= 2097152"); 5708 assert!(rss >= 2097152); 5709 5710 // Wait for an extra minute to make sure the stress command has 5711 // completed and that the guest reported the free pages to the VMM 5712 // through the virtio-balloon device. We expect the RSS to be under 5713 // 2GiB. 5714 thread::sleep(std::time::Duration::new(60, 0)); 5715 let rss = process_rss_kib(pid); 5716 println!("RSS {rss} < 2097152"); 5717 assert!(rss < 2097152); 5718 }); 5719 5720 kill_child(&mut child); 5721 let output = child.wait_with_output().unwrap(); 5722 5723 handle_child_output(r, &output); 5724 } 5725 5726 #[test] 5727 fn test_pmem_hotplug() { 5728 _test_pmem_hotplug(None) 5729 } 5730 5731 #[test] 5732 fn test_pmem_multi_segment_hotplug() { 5733 _test_pmem_hotplug(Some(15)) 5734 } 5735 5736 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5737 #[cfg(target_arch = "aarch64")] 5738 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5739 #[cfg(target_arch = "x86_64")] 5740 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5741 let focal = UbuntuDiskConfig::new(focal_image); 5742 let guest = Guest::new(Box::new(focal)); 5743 5744 #[cfg(target_arch = "x86_64")] 5745 let kernel_path = direct_kernel_boot_path(); 5746 #[cfg(target_arch = "aarch64")] 5747 let kernel_path = edk2_path(); 5748 5749 let api_socket = temp_api_path(&guest.tmp_dir); 5750 5751 let mut cmd = GuestCommand::new(&guest); 5752 5753 cmd.args(["--api-socket", &api_socket]) 5754 .args(["--cpus", "boot=1"]) 5755 .args(["--memory", "size=512M"]) 5756 .args(["--kernel", kernel_path.to_str().unwrap()]) 5757 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5758 .default_disks() 5759 .default_net() 5760 .capture_output(); 5761 5762 if pci_segment.is_some() { 5763 cmd.args([ 5764 "--platform", 5765 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5766 ]); 5767 } 5768 5769 let mut child = cmd.spawn().unwrap(); 5770 5771 let r = std::panic::catch_unwind(|| { 5772 guest.wait_vm_boot(None).unwrap(); 5773 5774 // Check /dev/pmem0 is not there 5775 assert_eq!( 5776 guest 5777 .ssh_command("lsblk | grep -c pmem0 || true") 5778 .unwrap() 5779 .trim() 5780 .parse::<u32>() 5781 .unwrap_or(1), 5782 0 5783 ); 5784 5785 let pmem_temp_file = TempFile::new().unwrap(); 5786 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5787 let (cmd_success, cmd_output) = remote_command_w_output( 5788 &api_socket, 5789 "add-pmem", 5790 Some(&format!( 5791 "file={},id=test0{}", 5792 pmem_temp_file.as_path().to_str().unwrap(), 5793 if let Some(pci_segment) = pci_segment { 5794 format!(",pci_segment={pci_segment}") 5795 } else { 5796 "".to_owned() 5797 } 5798 )), 5799 ); 5800 assert!(cmd_success); 5801 if let Some(pci_segment) = pci_segment { 5802 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5803 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5804 ))); 5805 } else { 5806 assert!(String::from_utf8_lossy(&cmd_output) 5807 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5808 } 5809 5810 // Check that /dev/pmem0 exists and the block size is 128M 5811 assert_eq!( 5812 guest 5813 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5814 .unwrap() 5815 .trim() 5816 .parse::<u32>() 5817 .unwrap_or_default(), 5818 1 5819 ); 5820 5821 guest.reboot_linux(0, None); 5822 5823 // Check still there after reboot 5824 assert_eq!( 5825 guest 5826 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5827 .unwrap() 5828 .trim() 5829 .parse::<u32>() 5830 .unwrap_or_default(), 5831 1 5832 ); 5833 5834 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5835 5836 thread::sleep(std::time::Duration::new(20, 0)); 5837 5838 // Check device has gone away 5839 assert_eq!( 5840 guest 5841 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5842 .unwrap() 5843 .trim() 5844 .parse::<u32>() 5845 .unwrap_or(1), 5846 0 5847 ); 5848 5849 guest.reboot_linux(1, None); 5850 5851 // Check still absent after reboot 5852 assert_eq!( 5853 guest 5854 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5855 .unwrap() 5856 .trim() 5857 .parse::<u32>() 5858 .unwrap_or(1), 5859 0 5860 ); 5861 }); 5862 5863 kill_child(&mut child); 5864 let output = child.wait_with_output().unwrap(); 5865 5866 handle_child_output(r, &output); 5867 } 5868 5869 #[test] 5870 fn test_net_hotplug() { 5871 _test_net_hotplug(None) 5872 } 5873 5874 #[test] 5875 fn test_net_multi_segment_hotplug() { 5876 _test_net_hotplug(Some(15)) 5877 } 5878 5879 fn _test_net_hotplug(pci_segment: Option<u16>) { 5880 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5881 let guest = Guest::new(Box::new(focal)); 5882 5883 #[cfg(target_arch = "x86_64")] 5884 let kernel_path = direct_kernel_boot_path(); 5885 #[cfg(target_arch = "aarch64")] 5886 let kernel_path = edk2_path(); 5887 5888 let api_socket = temp_api_path(&guest.tmp_dir); 5889 5890 // Boot without network 5891 let mut cmd = GuestCommand::new(&guest); 5892 5893 cmd.args(["--api-socket", &api_socket]) 5894 .args(["--cpus", "boot=1"]) 5895 .args(["--memory", "size=512M"]) 5896 .args(["--kernel", kernel_path.to_str().unwrap()]) 5897 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5898 .default_disks() 5899 .capture_output(); 5900 5901 if pci_segment.is_some() { 5902 cmd.args([ 5903 "--platform", 5904 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5905 ]); 5906 } 5907 5908 let mut child = cmd.spawn().unwrap(); 5909 5910 thread::sleep(std::time::Duration::new(20, 0)); 5911 5912 let r = std::panic::catch_unwind(|| { 5913 // Add network 5914 let (cmd_success, cmd_output) = remote_command_w_output( 5915 &api_socket, 5916 "add-net", 5917 Some( 5918 format!( 5919 "{}{},id=test0", 5920 guest.default_net_string(), 5921 if let Some(pci_segment) = pci_segment { 5922 format!(",pci_segment={pci_segment}") 5923 } else { 5924 "".to_owned() 5925 } 5926 ) 5927 .as_str(), 5928 ), 5929 ); 5930 assert!(cmd_success); 5931 5932 if let Some(pci_segment) = pci_segment { 5933 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5934 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5935 ))); 5936 } else { 5937 assert!(String::from_utf8_lossy(&cmd_output) 5938 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5939 } 5940 5941 thread::sleep(std::time::Duration::new(5, 0)); 5942 5943 // 1 network interfaces + default localhost ==> 2 interfaces 5944 assert_eq!( 5945 guest 5946 .ssh_command("ip -o link | wc -l") 5947 .unwrap() 5948 .trim() 5949 .parse::<u32>() 5950 .unwrap_or_default(), 5951 2 5952 ); 5953 5954 // Remove network 5955 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5956 thread::sleep(std::time::Duration::new(5, 0)); 5957 5958 let (cmd_success, cmd_output) = remote_command_w_output( 5959 &api_socket, 5960 "add-net", 5961 Some( 5962 format!( 5963 "{}{},id=test1", 5964 guest.default_net_string(), 5965 if let Some(pci_segment) = pci_segment { 5966 format!(",pci_segment={pci_segment}") 5967 } else { 5968 "".to_owned() 5969 } 5970 ) 5971 .as_str(), 5972 ), 5973 ); 5974 assert!(cmd_success); 5975 5976 if let Some(pci_segment) = pci_segment { 5977 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5978 "{{\"id\":\"test1\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5979 ))); 5980 } else { 5981 assert!(String::from_utf8_lossy(&cmd_output) 5982 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5983 } 5984 5985 thread::sleep(std::time::Duration::new(5, 0)); 5986 5987 // 1 network interfaces + default localhost ==> 2 interfaces 5988 assert_eq!( 5989 guest 5990 .ssh_command("ip -o link | wc -l") 5991 .unwrap() 5992 .trim() 5993 .parse::<u32>() 5994 .unwrap_or_default(), 5995 2 5996 ); 5997 5998 guest.reboot_linux(0, None); 5999 6000 // Check still there after reboot 6001 // 1 network interfaces + default localhost ==> 2 interfaces 6002 assert_eq!( 6003 guest 6004 .ssh_command("ip -o link | wc -l") 6005 .unwrap() 6006 .trim() 6007 .parse::<u32>() 6008 .unwrap_or_default(), 6009 2 6010 ); 6011 }); 6012 6013 kill_child(&mut child); 6014 let output = child.wait_with_output().unwrap(); 6015 6016 handle_child_output(r, &output); 6017 } 6018 6019 #[test] 6020 fn test_initramfs() { 6021 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6022 let guest = Guest::new(Box::new(focal)); 6023 let mut workload_path = dirs::home_dir().unwrap(); 6024 workload_path.push("workloads"); 6025 6026 #[cfg(target_arch = "x86_64")] 6027 let mut kernels = vec![direct_kernel_boot_path()]; 6028 #[cfg(target_arch = "aarch64")] 6029 let kernels = [direct_kernel_boot_path()]; 6030 6031 #[cfg(target_arch = "x86_64")] 6032 { 6033 let mut pvh_kernel_path = workload_path.clone(); 6034 pvh_kernel_path.push("vmlinux"); 6035 kernels.push(pvh_kernel_path); 6036 } 6037 6038 let mut initramfs_path = workload_path; 6039 initramfs_path.push("alpine_initramfs.img"); 6040 6041 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 6042 let cmdline = format!("console=hvc0 quiet TEST_STRING={test_string}"); 6043 6044 kernels.iter().for_each(|k_path| { 6045 let mut child = GuestCommand::new(&guest) 6046 .args(["--kernel", k_path.to_str().unwrap()]) 6047 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 6048 .args(["--cmdline", &cmdline]) 6049 .capture_output() 6050 .spawn() 6051 .unwrap(); 6052 6053 thread::sleep(std::time::Duration::new(20, 0)); 6054 6055 kill_child(&mut child); 6056 let output = child.wait_with_output().unwrap(); 6057 6058 let r = std::panic::catch_unwind(|| { 6059 let s = String::from_utf8_lossy(&output.stdout); 6060 6061 assert_ne!(s.lines().position(|line| line == test_string), None); 6062 }); 6063 6064 handle_child_output(r, &output); 6065 }); 6066 } 6067 6068 #[test] 6069 fn test_counters() { 6070 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6071 let guest = Guest::new(Box::new(focal)); 6072 let api_socket = temp_api_path(&guest.tmp_dir); 6073 6074 let mut cmd = GuestCommand::new(&guest); 6075 cmd.args(["--cpus", "boot=1"]) 6076 .args(["--memory", "size=512M"]) 6077 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 6078 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6079 .default_disks() 6080 .args(["--net", guest.default_net_string().as_str()]) 6081 .args(["--api-socket", &api_socket]) 6082 .capture_output(); 6083 6084 let mut child = cmd.spawn().unwrap(); 6085 6086 let r = std::panic::catch_unwind(|| { 6087 guest.wait_vm_boot(None).unwrap(); 6088 6089 let orig_counters = get_counters(&api_socket); 6090 guest 6091 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 6092 .unwrap(); 6093 6094 let new_counters = get_counters(&api_socket); 6095 6096 // Check that all the counters have increased 6097 assert!(new_counters > orig_counters); 6098 }); 6099 6100 kill_child(&mut child); 6101 let output = child.wait_with_output().unwrap(); 6102 6103 handle_child_output(r, &output); 6104 } 6105 6106 #[test] 6107 #[cfg(feature = "guest_debug")] 6108 fn test_coredump() { 6109 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6110 let guest = Guest::new(Box::new(focal)); 6111 let api_socket = temp_api_path(&guest.tmp_dir); 6112 6113 let mut cmd = GuestCommand::new(&guest); 6114 cmd.args(["--cpus", "boot=4"]) 6115 .args(["--memory", "size=4G"]) 6116 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6117 .default_disks() 6118 .args(["--net", guest.default_net_string().as_str()]) 6119 .args(["--api-socket", &api_socket]) 6120 .capture_output(); 6121 6122 let mut child = cmd.spawn().unwrap(); 6123 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6124 6125 let r = std::panic::catch_unwind(|| { 6126 guest.wait_vm_boot(None).unwrap(); 6127 6128 assert!(remote_command(&api_socket, "pause", None)); 6129 6130 assert!(remote_command( 6131 &api_socket, 6132 "coredump", 6133 Some(format!("file://{vmcore_file}").as_str()), 6134 )); 6135 6136 // the num of CORE notes should equals to vcpu 6137 let readelf_core_num_cmd = 6138 format!("readelf --all {vmcore_file} |grep CORE |grep -v Type |wc -l"); 6139 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 6140 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 6141 6142 // the num of QEMU notes should equals to vcpu 6143 let readelf_vmm_num_cmd = format!("readelf --all {vmcore_file} |grep QEMU |wc -l"); 6144 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 6145 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 6146 }); 6147 6148 kill_child(&mut child); 6149 let output = child.wait_with_output().unwrap(); 6150 6151 handle_child_output(r, &output); 6152 } 6153 6154 #[test] 6155 #[cfg(feature = "guest_debug")] 6156 fn test_coredump_no_pause() { 6157 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6158 let guest = Guest::new(Box::new(focal)); 6159 let api_socket = temp_api_path(&guest.tmp_dir); 6160 6161 let mut cmd = GuestCommand::new(&guest); 6162 cmd.args(["--cpus", "boot=4"]) 6163 .args(["--memory", "size=4G"]) 6164 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6165 .default_disks() 6166 .args(["--net", guest.default_net_string().as_str()]) 6167 .args(["--api-socket", &api_socket]) 6168 .capture_output(); 6169 6170 let mut child = cmd.spawn().unwrap(); 6171 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6172 6173 let r = std::panic::catch_unwind(|| { 6174 guest.wait_vm_boot(None).unwrap(); 6175 6176 assert!(remote_command( 6177 &api_socket, 6178 "coredump", 6179 Some(format!("file://{vmcore_file}").as_str()), 6180 )); 6181 6182 assert_eq!(vm_state(&api_socket), "Running"); 6183 }); 6184 6185 kill_child(&mut child); 6186 let output = child.wait_with_output().unwrap(); 6187 6188 handle_child_output(r, &output); 6189 } 6190 6191 #[test] 6192 fn test_watchdog() { 6193 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6194 let guest = Guest::new(Box::new(focal)); 6195 let api_socket = temp_api_path(&guest.tmp_dir); 6196 6197 let kernel_path = direct_kernel_boot_path(); 6198 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6199 6200 let mut cmd = GuestCommand::new(&guest); 6201 cmd.args(["--cpus", "boot=1"]) 6202 .args(["--memory", "size=512M"]) 6203 .args(["--kernel", kernel_path.to_str().unwrap()]) 6204 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6205 .default_disks() 6206 .args(["--net", guest.default_net_string().as_str()]) 6207 .args(["--watchdog"]) 6208 .args(["--api-socket", &api_socket]) 6209 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6210 .capture_output(); 6211 6212 let mut child = cmd.spawn().unwrap(); 6213 6214 let r = std::panic::catch_unwind(|| { 6215 guest.wait_vm_boot(None).unwrap(); 6216 6217 let mut expected_reboot_count = 1; 6218 6219 // Enable the watchdog with a 15s timeout 6220 enable_guest_watchdog(&guest, 15); 6221 6222 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6223 assert_eq!( 6224 guest 6225 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6226 .unwrap() 6227 .trim() 6228 .parse::<u32>() 6229 .unwrap_or_default(), 6230 1 6231 ); 6232 6233 // Allow some normal time to elapse to check we don't get spurious reboots 6234 thread::sleep(std::time::Duration::new(40, 0)); 6235 // Check no reboot 6236 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6237 6238 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6239 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6240 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6241 guest.wait_vm_boot(Some(50)).unwrap(); 6242 // Check a reboot is triggered by the watchdog 6243 expected_reboot_count += 1; 6244 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6245 6246 #[cfg(target_arch = "x86_64")] 6247 { 6248 // Now pause the VM and remain offline for 30s 6249 assert!(remote_command(&api_socket, "pause", None)); 6250 let latest_events = [ 6251 &MetaEvent { 6252 event: "pausing".to_string(), 6253 device_id: None, 6254 }, 6255 &MetaEvent { 6256 event: "paused".to_string(), 6257 device_id: None, 6258 }, 6259 ]; 6260 assert!(check_latest_events_exact(&latest_events, &event_path)); 6261 assert!(remote_command(&api_socket, "resume", None)); 6262 6263 // Check no reboot 6264 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6265 } 6266 }); 6267 6268 kill_child(&mut child); 6269 let output = child.wait_with_output().unwrap(); 6270 6271 handle_child_output(r, &output); 6272 } 6273 6274 #[test] 6275 fn test_pvpanic() { 6276 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 6277 let guest = Guest::new(Box::new(jammy)); 6278 let api_socket = temp_api_path(&guest.tmp_dir); 6279 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6280 6281 let kernel_path = direct_kernel_boot_path(); 6282 6283 let mut cmd = GuestCommand::new(&guest); 6284 cmd.args(["--cpus", "boot=1"]) 6285 .args(["--memory", "size=512M"]) 6286 .args(["--kernel", kernel_path.to_str().unwrap()]) 6287 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6288 .default_disks() 6289 .args(["--net", guest.default_net_string().as_str()]) 6290 .args(["--pvpanic"]) 6291 .args(["--api-socket", &api_socket]) 6292 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6293 .capture_output(); 6294 6295 let mut child = cmd.spawn().unwrap(); 6296 6297 let r = std::panic::catch_unwind(|| { 6298 guest.wait_vm_boot(None).unwrap(); 6299 6300 // Trigger guest a panic 6301 make_guest_panic(&guest); 6302 6303 // Wait a while for guest 6304 thread::sleep(std::time::Duration::new(10, 0)); 6305 6306 let expected_sequential_events = [&MetaEvent { 6307 event: "panic".to_string(), 6308 device_id: None, 6309 }]; 6310 assert!(check_latest_events_exact( 6311 &expected_sequential_events, 6312 &event_path 6313 )); 6314 }); 6315 6316 kill_child(&mut child); 6317 let output = child.wait_with_output().unwrap(); 6318 6319 handle_child_output(r, &output); 6320 } 6321 6322 #[test] 6323 fn test_tap_from_fd() { 6324 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6325 let guest = Guest::new(Box::new(focal)); 6326 let kernel_path = direct_kernel_boot_path(); 6327 6328 // Create a TAP interface with multi-queue enabled 6329 let num_queue_pairs: usize = 2; 6330 6331 use std::str::FromStr; 6332 let taps = net_util::open_tap( 6333 Some("chtap0"), 6334 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6335 None, 6336 &mut None, 6337 None, 6338 num_queue_pairs, 6339 Some(libc::O_RDWR | libc::O_NONBLOCK), 6340 ) 6341 .unwrap(); 6342 6343 let mut child = GuestCommand::new(&guest) 6344 .args(["--cpus", &format!("boot={num_queue_pairs}")]) 6345 .args(["--memory", "size=512M"]) 6346 .args(["--kernel", kernel_path.to_str().unwrap()]) 6347 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6348 .default_disks() 6349 .args([ 6350 "--net", 6351 &format!( 6352 "fd=[{},{}],mac={},num_queues={}", 6353 taps[0].as_raw_fd(), 6354 taps[1].as_raw_fd(), 6355 guest.network.guest_mac, 6356 num_queue_pairs * 2 6357 ), 6358 ]) 6359 .capture_output() 6360 .spawn() 6361 .unwrap(); 6362 6363 let r = std::panic::catch_unwind(|| { 6364 guest.wait_vm_boot(None).unwrap(); 6365 6366 assert_eq!( 6367 guest 6368 .ssh_command("ip -o link | wc -l") 6369 .unwrap() 6370 .trim() 6371 .parse::<u32>() 6372 .unwrap_or_default(), 6373 2 6374 ); 6375 6376 guest.reboot_linux(0, None); 6377 6378 assert_eq!( 6379 guest 6380 .ssh_command("ip -o link | wc -l") 6381 .unwrap() 6382 .trim() 6383 .parse::<u32>() 6384 .unwrap_or_default(), 6385 2 6386 ); 6387 }); 6388 6389 kill_child(&mut child); 6390 let output = child.wait_with_output().unwrap(); 6391 6392 handle_child_output(r, &output); 6393 } 6394 6395 // By design, a guest VM won't be able to connect to the host 6396 // machine when using a macvtap network interface (while it can 6397 // communicate externally). As a workaround, this integration 6398 // test creates two macvtap interfaces in 'bridge' mode on the 6399 // same physical net interface, one for the guest and one for 6400 // the host. With additional setup on the IP address and the 6401 // routing table, it enables the communications between the 6402 // guest VM and the host machine. 6403 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6404 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6405 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6406 let guest = Guest::new(Box::new(focal)); 6407 let api_socket = temp_api_path(&guest.tmp_dir); 6408 6409 #[cfg(target_arch = "x86_64")] 6410 let kernel_path = direct_kernel_boot_path(); 6411 #[cfg(target_arch = "aarch64")] 6412 let kernel_path = edk2_path(); 6413 6414 let phy_net = "eth0"; 6415 6416 // Create a macvtap interface for the guest VM to use 6417 assert!(exec_host_command_status(&format!( 6418 "sudo ip link add link {phy_net} name {guest_macvtap_name} type macvtap mod bridge" 6419 )) 6420 .success()); 6421 assert!(exec_host_command_status(&format!( 6422 "sudo ip link set {} address {} up", 6423 guest_macvtap_name, guest.network.guest_mac 6424 )) 6425 .success()); 6426 assert!( 6427 exec_host_command_status(&format!("sudo ip link show {guest_macvtap_name}")).success() 6428 ); 6429 6430 let tap_index = 6431 fs::read_to_string(format!("/sys/class/net/{guest_macvtap_name}/ifindex")).unwrap(); 6432 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6433 6434 assert!(exec_host_command_status(&format!("sudo chown $UID.$UID {tap_device}")).success()); 6435 6436 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6437 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6438 assert!(tap_fd1 > 0); 6439 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6440 assert!(tap_fd2 > 0); 6441 6442 // Create a macvtap on the same physical net interface for 6443 // the host machine to use 6444 assert!(exec_host_command_status(&format!( 6445 "sudo ip link add link {phy_net} name {host_macvtap_name} type macvtap mod bridge" 6446 )) 6447 .success()); 6448 // Use default mask "255.255.255.0" 6449 assert!(exec_host_command_status(&format!( 6450 "sudo ip address add {}/24 dev {}", 6451 guest.network.host_ip, host_macvtap_name 6452 )) 6453 .success()); 6454 assert!( 6455 exec_host_command_status(&format!("sudo ip link set dev {host_macvtap_name} up")) 6456 .success() 6457 ); 6458 6459 let mut guest_command = GuestCommand::new(&guest); 6460 guest_command 6461 .args(["--cpus", "boot=2"]) 6462 .args(["--memory", "size=512M"]) 6463 .args(["--kernel", kernel_path.to_str().unwrap()]) 6464 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6465 .default_disks() 6466 .args(["--api-socket", &api_socket]); 6467 6468 let net_params = format!( 6469 "fd=[{},{}],mac={},num_queues=4", 6470 tap_fd1, tap_fd2, guest.network.guest_mac 6471 ); 6472 6473 if !hotplug { 6474 guest_command.args(["--net", &net_params]); 6475 } 6476 6477 let mut child = guest_command.capture_output().spawn().unwrap(); 6478 6479 if hotplug { 6480 // Give some time to the VMM process to listen to the API 6481 // socket. This is the only requirement to avoid the following 6482 // call to ch-remote from failing. 6483 thread::sleep(std::time::Duration::new(10, 0)); 6484 // Hotplug the virtio-net device 6485 let (cmd_success, cmd_output) = 6486 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6487 assert!(cmd_success); 6488 #[cfg(target_arch = "x86_64")] 6489 assert!(String::from_utf8_lossy(&cmd_output) 6490 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6491 #[cfg(target_arch = "aarch64")] 6492 assert!(String::from_utf8_lossy(&cmd_output) 6493 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6494 } 6495 6496 // The functional connectivity provided by the virtio-net device 6497 // gets tested through wait_vm_boot() as it expects to receive a 6498 // HTTP request, and through the SSH command as well. 6499 let r = std::panic::catch_unwind(|| { 6500 guest.wait_vm_boot(None).unwrap(); 6501 6502 assert_eq!( 6503 guest 6504 .ssh_command("ip -o link | wc -l") 6505 .unwrap() 6506 .trim() 6507 .parse::<u32>() 6508 .unwrap_or_default(), 6509 2 6510 ); 6511 6512 guest.reboot_linux(0, None); 6513 6514 assert_eq!( 6515 guest 6516 .ssh_command("ip -o link | wc -l") 6517 .unwrap() 6518 .trim() 6519 .parse::<u32>() 6520 .unwrap_or_default(), 6521 2 6522 ); 6523 }); 6524 6525 kill_child(&mut child); 6526 6527 exec_host_command_status(&format!("sudo ip link del {guest_macvtap_name}")); 6528 exec_host_command_status(&format!("sudo ip link del {host_macvtap_name}")); 6529 6530 let output = child.wait_with_output().unwrap(); 6531 6532 handle_child_output(r, &output); 6533 } 6534 6535 #[test] 6536 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6537 fn test_macvtap() { 6538 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6539 } 6540 6541 #[test] 6542 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6543 fn test_macvtap_hotplug() { 6544 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6545 } 6546 6547 #[test] 6548 #[cfg(not(feature = "mshv"))] 6549 fn test_ovs_dpdk() { 6550 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6551 let guest1 = Guest::new(Box::new(focal1)); 6552 6553 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6554 let guest2 = Guest::new(Box::new(focal2)); 6555 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6556 6557 let (mut child1, mut child2) = 6558 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6559 6560 // Create the snapshot directory 6561 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6562 6563 let r = std::panic::catch_unwind(|| { 6564 // Remove one of the two ports from the OVS bridge 6565 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6566 6567 // Spawn a new netcat listener in the first VM 6568 let guest_ip = guest1.network.guest_ip.clone(); 6569 thread::spawn(move || { 6570 ssh_command_ip( 6571 "nc -l 12345", 6572 &guest_ip, 6573 DEFAULT_SSH_RETRIES, 6574 DEFAULT_SSH_TIMEOUT, 6575 ) 6576 .unwrap(); 6577 }); 6578 6579 // Wait for the server to be listening 6580 thread::sleep(std::time::Duration::new(5, 0)); 6581 6582 // Check the connection fails this time 6583 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6584 6585 // Add the OVS port back 6586 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6587 6588 // And finally check the connection is functional again 6589 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6590 6591 // Pause the VM 6592 assert!(remote_command(&api_socket_source, "pause", None)); 6593 6594 // Take a snapshot from the VM 6595 assert!(remote_command( 6596 &api_socket_source, 6597 "snapshot", 6598 Some(format!("file://{snapshot_dir}").as_str()), 6599 )); 6600 6601 // Wait to make sure the snapshot is completed 6602 thread::sleep(std::time::Duration::new(10, 0)); 6603 }); 6604 6605 // Shutdown the source VM 6606 kill_child(&mut child2); 6607 let output = child2.wait_with_output().unwrap(); 6608 handle_child_output(r, &output); 6609 6610 // Remove the vhost-user socket file. 6611 Command::new("rm") 6612 .arg("-f") 6613 .arg("/tmp/dpdkvhostclient2") 6614 .output() 6615 .unwrap(); 6616 6617 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6618 // Restore the VM from the snapshot 6619 let mut child2 = GuestCommand::new(&guest2) 6620 .args(["--api-socket", &api_socket_restored]) 6621 .args([ 6622 "--restore", 6623 format!("source_url=file://{snapshot_dir}").as_str(), 6624 ]) 6625 .capture_output() 6626 .spawn() 6627 .unwrap(); 6628 6629 // Wait for the VM to be restored 6630 thread::sleep(std::time::Duration::new(10, 0)); 6631 6632 let r = std::panic::catch_unwind(|| { 6633 // Resume the VM 6634 assert!(remote_command(&api_socket_restored, "resume", None)); 6635 6636 // Spawn a new netcat listener in the first VM 6637 let guest_ip = guest1.network.guest_ip.clone(); 6638 thread::spawn(move || { 6639 ssh_command_ip( 6640 "nc -l 12345", 6641 &guest_ip, 6642 DEFAULT_SSH_RETRIES, 6643 DEFAULT_SSH_TIMEOUT, 6644 ) 6645 .unwrap(); 6646 }); 6647 6648 // Wait for the server to be listening 6649 thread::sleep(std::time::Duration::new(5, 0)); 6650 6651 // And check the connection is still functional after restore 6652 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6653 }); 6654 6655 kill_child(&mut child1); 6656 kill_child(&mut child2); 6657 6658 let output = child1.wait_with_output().unwrap(); 6659 child2.wait().unwrap(); 6660 6661 cleanup_ovs_dpdk(); 6662 6663 handle_child_output(r, &output); 6664 } 6665 6666 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6667 cleanup_spdk_nvme(); 6668 6669 assert!(exec_host_command_status(&format!( 6670 "mkdir -p {}", 6671 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6672 )) 6673 .success()); 6674 assert!(exec_host_command_status(&format!( 6675 "truncate {} -s 128M", 6676 nvme_dir.join("test-disk.raw").to_str().unwrap() 6677 )) 6678 .success()); 6679 assert!(exec_host_command_status(&format!( 6680 "mkfs.ext4 {}", 6681 nvme_dir.join("test-disk.raw").to_str().unwrap() 6682 )) 6683 .success()); 6684 6685 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6686 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6687 .args(["-i", "0", "-m", "0x1"]) 6688 .spawn() 6689 .unwrap(); 6690 thread::sleep(std::time::Duration::new(2, 0)); 6691 6692 assert!(exec_host_command_with_retries( 6693 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER", 6694 3, 6695 std::time::Duration::new(5, 0), 6696 )); 6697 assert!(exec_host_command_status(&format!( 6698 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6699 nvme_dir.join("test-disk.raw").to_str().unwrap() 6700 )) 6701 .success()); 6702 assert!(exec_host_command_status( 6703 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6704 ) 6705 .success()); 6706 assert!(exec_host_command_status( 6707 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6708 ) 6709 .success()); 6710 assert!(exec_host_command_status(&format!( 6711 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6712 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6713 )) 6714 .success()); 6715 } 6716 6717 fn cleanup_spdk_nvme() { 6718 exec_host_command_status("pkill -f nvmf_tgt"); 6719 } 6720 6721 #[test] 6722 fn test_vfio_user() { 6723 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6724 let jammy = UbuntuDiskConfig::new(jammy_image); 6725 let guest = Guest::new(Box::new(jammy)); 6726 6727 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6728 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6729 6730 let api_socket = temp_api_path(&guest.tmp_dir); 6731 let mut child = GuestCommand::new(&guest) 6732 .args(["--api-socket", &api_socket]) 6733 .args(["--cpus", "boot=1"]) 6734 .args(["--memory", "size=512M,shared=on,hugepages=on"]) 6735 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6736 .args(["--serial", "tty", "--console", "off"]) 6737 .default_disks() 6738 .default_net() 6739 .capture_output() 6740 .spawn() 6741 .unwrap(); 6742 6743 let r = std::panic::catch_unwind(|| { 6744 guest.wait_vm_boot(None).unwrap(); 6745 6746 // Hotplug the SPDK-NVMe device to the VM 6747 let (cmd_success, cmd_output) = remote_command_w_output( 6748 &api_socket, 6749 "add-user-device", 6750 Some(&format!( 6751 "socket={},id=vfio_user0", 6752 spdk_nvme_dir 6753 .as_path() 6754 .join("nvme-vfio-user/cntrl") 6755 .to_str() 6756 .unwrap(), 6757 )), 6758 ); 6759 assert!(cmd_success); 6760 assert!(String::from_utf8_lossy(&cmd_output) 6761 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6762 6763 thread::sleep(std::time::Duration::new(10, 0)); 6764 6765 // Check both if /dev/nvme exists and if the block size is 128M. 6766 assert_eq!( 6767 guest 6768 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6769 .unwrap() 6770 .trim() 6771 .parse::<u32>() 6772 .unwrap_or_default(), 6773 1 6774 ); 6775 6776 // Check changes persist after reboot 6777 assert_eq!( 6778 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6779 "" 6780 ); 6781 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6782 guest 6783 .ssh_command("echo test123 | sudo tee /mnt/test") 6784 .unwrap(); 6785 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6786 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6787 6788 guest.reboot_linux(0, None); 6789 assert_eq!( 6790 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6791 "" 6792 ); 6793 assert_eq!( 6794 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6795 "test123" 6796 ); 6797 }); 6798 6799 cleanup_spdk_nvme(); 6800 6801 kill_child(&mut child); 6802 let output = child.wait_with_output().unwrap(); 6803 6804 handle_child_output(r, &output); 6805 } 6806 6807 #[test] 6808 #[cfg(target_arch = "x86_64")] 6809 fn test_vdpa_block() { 6810 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6811 assert!(exec_host_command_status("lsmod | grep vdpa_sim_blk").success()); 6812 6813 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6814 let guest = Guest::new(Box::new(focal)); 6815 let api_socket = temp_api_path(&guest.tmp_dir); 6816 6817 let kernel_path = direct_kernel_boot_path(); 6818 6819 let mut child = GuestCommand::new(&guest) 6820 .args(["--cpus", "boot=2"]) 6821 .args(["--memory", "size=512M,hugepages=on"]) 6822 .args(["--kernel", kernel_path.to_str().unwrap()]) 6823 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6824 .default_disks() 6825 .default_net() 6826 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6827 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 6828 .args(["--api-socket", &api_socket]) 6829 .capture_output() 6830 .spawn() 6831 .unwrap(); 6832 6833 let r = std::panic::catch_unwind(|| { 6834 guest.wait_vm_boot(None).unwrap(); 6835 6836 // Check both if /dev/vdc exists and if the block size is 128M. 6837 assert_eq!( 6838 guest 6839 .ssh_command("lsblk | grep vdc | grep -c 128M") 6840 .unwrap() 6841 .trim() 6842 .parse::<u32>() 6843 .unwrap_or_default(), 6844 1 6845 ); 6846 6847 // Check the content of the block device after we wrote to it. 6848 // The vpda-sim-blk should let us read what we previously wrote. 6849 guest 6850 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6851 .unwrap(); 6852 assert_eq!( 6853 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6854 "foobar" 6855 ); 6856 6857 // Hotplug an extra vDPA block device behind the vIOMMU 6858 // Add a new vDPA device to the VM 6859 let (cmd_success, cmd_output) = remote_command_w_output( 6860 &api_socket, 6861 "add-vdpa", 6862 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6863 ); 6864 assert!(cmd_success); 6865 assert!(String::from_utf8_lossy(&cmd_output) 6866 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6867 6868 thread::sleep(std::time::Duration::new(10, 0)); 6869 6870 // Check IOMMU setup 6871 assert!(guest 6872 .does_device_vendor_pair_match("0x1057", "0x1af4") 6873 .unwrap_or_default()); 6874 assert_eq!( 6875 guest 6876 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6877 .unwrap() 6878 .trim(), 6879 "0001:00:01.0" 6880 ); 6881 6882 // Check both if /dev/vdd exists and if the block size is 128M. 6883 assert_eq!( 6884 guest 6885 .ssh_command("lsblk | grep vdd | grep -c 128M") 6886 .unwrap() 6887 .trim() 6888 .parse::<u32>() 6889 .unwrap_or_default(), 6890 1 6891 ); 6892 6893 // Write some content to the block device we've just plugged. 6894 guest 6895 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6896 .unwrap(); 6897 6898 // Check we can read the content back. 6899 assert_eq!( 6900 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6901 "foobar" 6902 ); 6903 6904 // Unplug the device 6905 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6906 assert!(cmd_success); 6907 thread::sleep(std::time::Duration::new(10, 0)); 6908 6909 // Check /dev/vdd doesn't exist anymore 6910 assert_eq!( 6911 guest 6912 .ssh_command("lsblk | grep -c vdd || true") 6913 .unwrap() 6914 .trim() 6915 .parse::<u32>() 6916 .unwrap_or(1), 6917 0 6918 ); 6919 }); 6920 6921 kill_child(&mut child); 6922 let output = child.wait_with_output().unwrap(); 6923 6924 handle_child_output(r, &output); 6925 } 6926 6927 #[test] 6928 #[cfg(target_arch = "x86_64")] 6929 #[ignore = "See #5756"] 6930 fn test_vdpa_net() { 6931 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6932 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6933 return; 6934 } 6935 6936 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6937 let guest = Guest::new(Box::new(focal)); 6938 6939 let kernel_path = direct_kernel_boot_path(); 6940 6941 let mut child = GuestCommand::new(&guest) 6942 .args(["--cpus", "boot=2"]) 6943 .args(["--memory", "size=512M,hugepages=on"]) 6944 .args(["--kernel", kernel_path.to_str().unwrap()]) 6945 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6946 .default_disks() 6947 .default_net() 6948 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 6949 .capture_output() 6950 .spawn() 6951 .unwrap(); 6952 6953 let r = std::panic::catch_unwind(|| { 6954 guest.wait_vm_boot(None).unwrap(); 6955 6956 // Check we can find network interface related to vDPA device 6957 assert_eq!( 6958 guest 6959 .ssh_command("ip -o link | grep -c ens6") 6960 .unwrap() 6961 .trim() 6962 .parse::<u32>() 6963 .unwrap_or(0), 6964 1 6965 ); 6966 6967 guest 6968 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 6969 .unwrap(); 6970 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 6971 6972 // Check there is no packet yet on both TX/RX of the network interface 6973 assert_eq!( 6974 guest 6975 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 6976 .unwrap() 6977 .trim() 6978 .parse::<u32>() 6979 .unwrap_or(0), 6980 2 6981 ); 6982 6983 // Send 6 packets with ping command 6984 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 6985 6986 // Check we can find 6 packets on both TX/RX of the network interface 6987 assert_eq!( 6988 guest 6989 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 6990 .unwrap() 6991 .trim() 6992 .parse::<u32>() 6993 .unwrap_or(0), 6994 2 6995 ); 6996 6997 // No need to check for hotplug as we already tested it through 6998 // test_vdpa_block() 6999 }); 7000 7001 kill_child(&mut child); 7002 let output = child.wait_with_output().unwrap(); 7003 7004 handle_child_output(r, &output); 7005 } 7006 7007 #[test] 7008 #[cfg(target_arch = "x86_64")] 7009 fn test_tpm() { 7010 let focal = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7011 let guest = Guest::new(Box::new(focal)); 7012 7013 let (mut swtpm_command, swtpm_socket_path) = prepare_swtpm_daemon(&guest.tmp_dir); 7014 7015 let mut guest_cmd = GuestCommand::new(&guest); 7016 guest_cmd 7017 .args(["--cpus", "boot=1"]) 7018 .args(["--memory", "size=512M"]) 7019 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7020 .args(["--tpm", &format!("socket={swtpm_socket_path}")]) 7021 .capture_output() 7022 .default_disks() 7023 .default_net(); 7024 7025 // Start swtpm daemon 7026 let mut swtpm_child = swtpm_command.spawn().unwrap(); 7027 thread::sleep(std::time::Duration::new(10, 0)); 7028 let mut child = guest_cmd.spawn().unwrap(); 7029 let r = std::panic::catch_unwind(|| { 7030 guest.wait_vm_boot(None).unwrap(); 7031 assert_eq!( 7032 guest.ssh_command("ls /dev/tpm0").unwrap().trim(), 7033 "/dev/tpm0" 7034 ); 7035 guest.ssh_command("sudo tpm2_selftest -f").unwrap(); 7036 guest 7037 .ssh_command("echo 'hello' > /tmp/checksum_test; ") 7038 .unwrap(); 7039 guest.ssh_command("cmp <(sudo tpm2_pcrevent /tmp/checksum_test | grep sha256 | awk '{print $2}') <(sha256sum /tmp/checksum_test| awk '{print $1}')").unwrap(); 7040 }); 7041 7042 let _ = swtpm_child.kill(); 7043 let _d_out = swtpm_child.wait_with_output().unwrap(); 7044 7045 kill_child(&mut child); 7046 let output = child.wait_with_output().unwrap(); 7047 7048 handle_child_output(r, &output); 7049 } 7050 7051 #[test] 7052 #[cfg(target_arch = "x86_64")] 7053 fn test_double_tty() { 7054 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7055 let guest = Guest::new(Box::new(focal)); 7056 let mut cmd = GuestCommand::new(&guest); 7057 let api_socket = temp_api_path(&guest.tmp_dir); 7058 let tty_str: &str = "console=hvc0 earlyprintk=ttyS0 "; 7059 // linux printk module enable console log. 7060 let con_dis_str: &str = "console [hvc0] enabled"; 7061 // linux printk module disable console log. 7062 let con_enb_str: &str = "bootconsole [earlyser0] disabled"; 7063 7064 let kernel_path = direct_kernel_boot_path(); 7065 7066 cmd.args(["--cpus", "boot=1"]) 7067 .args(["--memory", "size=512M"]) 7068 .args(["--kernel", kernel_path.to_str().unwrap()]) 7069 .args([ 7070 "--cmdline", 7071 DIRECT_KERNEL_BOOT_CMDLINE 7072 .replace("console=hvc0 ", tty_str) 7073 .as_str(), 7074 ]) 7075 .capture_output() 7076 .default_disks() 7077 .default_net() 7078 .args(["--serial", "tty"]) 7079 .args(["--console", "tty"]) 7080 .args(["--api-socket", &api_socket]); 7081 7082 let mut child = cmd.spawn().unwrap(); 7083 7084 let mut r = std::panic::catch_unwind(|| { 7085 guest.wait_vm_boot(None).unwrap(); 7086 }); 7087 7088 kill_child(&mut child); 7089 let output = child.wait_with_output().unwrap(); 7090 7091 if r.is_ok() { 7092 r = std::panic::catch_unwind(|| { 7093 let s = String::from_utf8_lossy(&output.stdout); 7094 assert!(s.contains(tty_str)); 7095 assert!(s.contains(con_dis_str)); 7096 assert!(s.contains(con_enb_str)); 7097 }); 7098 } 7099 7100 handle_child_output(r, &output); 7101 } 7102 7103 #[test] 7104 #[cfg(target_arch = "x86_64")] 7105 fn test_nmi() { 7106 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7107 let guest = Guest::new(Box::new(jammy)); 7108 let api_socket = temp_api_path(&guest.tmp_dir); 7109 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7110 7111 let kernel_path = direct_kernel_boot_path(); 7112 let cmd_line = format!("{} {}", DIRECT_KERNEL_BOOT_CMDLINE, "unknown_nmi_panic=1"); 7113 7114 let mut cmd = GuestCommand::new(&guest); 7115 cmd.args(["--cpus", "boot=4"]) 7116 .args(["--memory", "size=512M"]) 7117 .args(["--kernel", kernel_path.to_str().unwrap()]) 7118 .args(["--cmdline", cmd_line.as_str()]) 7119 .default_disks() 7120 .args(["--net", guest.default_net_string().as_str()]) 7121 .args(["--pvpanic"]) 7122 .args(["--api-socket", &api_socket]) 7123 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7124 .capture_output(); 7125 7126 let mut child = cmd.spawn().unwrap(); 7127 7128 let r = std::panic::catch_unwind(|| { 7129 guest.wait_vm_boot(None).unwrap(); 7130 7131 assert!(remote_command(&api_socket, "nmi", None)); 7132 7133 // Wait a while for guest 7134 thread::sleep(std::time::Duration::new(3, 0)); 7135 7136 let expected_sequential_events = [&MetaEvent { 7137 event: "panic".to_string(), 7138 device_id: None, 7139 }]; 7140 assert!(check_latest_events_exact( 7141 &expected_sequential_events, 7142 &event_path 7143 )); 7144 }); 7145 7146 kill_child(&mut child); 7147 let output = child.wait_with_output().unwrap(); 7148 7149 handle_child_output(r, &output); 7150 } 7151 } 7152 7153 mod dbus_api { 7154 use crate::*; 7155 7156 // Start cloud-hypervisor with no VM parameters, running both the HTTP 7157 // and DBus APIs. Alternate calls to the external APIs (HTTP and DBus) 7158 // to create a VM, boot it, and verify that it can be shut down and then 7159 // booted again. 7160 #[test] 7161 fn test_api_dbus_and_http_interleaved() { 7162 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7163 let guest = Guest::new(Box::new(focal)); 7164 let dbus_api = TargetApi::new_dbus_api(&guest.tmp_dir); 7165 let http_api = TargetApi::new_http_api(&guest.tmp_dir); 7166 7167 let mut child = GuestCommand::new(&guest) 7168 .args(dbus_api.guest_args()) 7169 .args(http_api.guest_args()) 7170 .capture_output() 7171 .spawn() 7172 .unwrap(); 7173 7174 thread::sleep(std::time::Duration::new(1, 0)); 7175 7176 // Verify API servers are running 7177 assert!(dbus_api.remote_command("ping", None)); 7178 assert!(http_api.remote_command("ping", None)); 7179 7180 // Create the VM first 7181 let cpu_count: u8 = 4; 7182 let request_body = guest.api_create_body( 7183 cpu_count, 7184 direct_kernel_boot_path().to_str().unwrap(), 7185 DIRECT_KERNEL_BOOT_CMDLINE, 7186 ); 7187 7188 let temp_config_path = guest.tmp_dir.as_path().join("config"); 7189 std::fs::write(&temp_config_path, request_body).unwrap(); 7190 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 7191 7192 let r = std::panic::catch_unwind(|| { 7193 // Create the VM 7194 assert!(dbus_api.remote_command("create", Some(create_config),)); 7195 7196 // Then boot it 7197 assert!(http_api.remote_command("boot", None)); 7198 guest.wait_vm_boot(None).unwrap(); 7199 7200 // Check that the VM booted as expected 7201 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7202 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7203 7204 // Sync and shutdown without powering off to prevent filesystem 7205 // corruption. 7206 guest.ssh_command("sync").unwrap(); 7207 guest.ssh_command("sudo shutdown -H now").unwrap(); 7208 7209 // Wait for the guest to be fully shutdown 7210 thread::sleep(std::time::Duration::new(20, 0)); 7211 7212 // Then shutdown the VM 7213 assert!(dbus_api.remote_command("shutdown", None)); 7214 7215 // Then boot it again 7216 assert!(http_api.remote_command("boot", None)); 7217 guest.wait_vm_boot(None).unwrap(); 7218 7219 // Check that the VM booted as expected 7220 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7221 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7222 }); 7223 7224 kill_child(&mut child); 7225 let output = child.wait_with_output().unwrap(); 7226 7227 handle_child_output(r, &output); 7228 } 7229 7230 #[test] 7231 fn test_api_dbus_create_boot() { 7232 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7233 let guest = Guest::new(Box::new(focal)); 7234 7235 _test_api_create_boot(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7236 } 7237 7238 #[test] 7239 fn test_api_dbus_shutdown() { 7240 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7241 let guest = Guest::new(Box::new(focal)); 7242 7243 _test_api_shutdown(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7244 } 7245 7246 #[test] 7247 fn test_api_dbus_delete() { 7248 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7249 let guest = Guest::new(Box::new(focal)); 7250 7251 _test_api_delete(TargetApi::new_dbus_api(&guest.tmp_dir), guest); 7252 } 7253 7254 #[test] 7255 fn test_api_dbus_pause_resume() { 7256 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7257 let guest = Guest::new(Box::new(focal)); 7258 7259 _test_api_pause_resume(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7260 } 7261 } 7262 7263 mod common_sequential { 7264 use std::fs::remove_dir_all; 7265 7266 use crate::*; 7267 7268 #[test] 7269 #[cfg(not(feature = "mshv"))] 7270 fn test_memory_mergeable_on() { 7271 test_memory_mergeable(true) 7272 } 7273 7274 fn snapshot_and_check_events(api_socket: &str, snapshot_dir: &str, event_path: &str) { 7275 // Pause the VM 7276 assert!(remote_command(api_socket, "pause", None)); 7277 let latest_events: [&MetaEvent; 2] = [ 7278 &MetaEvent { 7279 event: "pausing".to_string(), 7280 device_id: None, 7281 }, 7282 &MetaEvent { 7283 event: "paused".to_string(), 7284 device_id: None, 7285 }, 7286 ]; 7287 // See: #5938 7288 thread::sleep(std::time::Duration::new(1, 0)); 7289 assert!(check_latest_events_exact(&latest_events, event_path)); 7290 7291 // Take a snapshot from the VM 7292 assert!(remote_command( 7293 api_socket, 7294 "snapshot", 7295 Some(format!("file://{snapshot_dir}").as_str()), 7296 )); 7297 7298 // Wait to make sure the snapshot is completed 7299 thread::sleep(std::time::Duration::new(10, 0)); 7300 7301 let latest_events = [ 7302 &MetaEvent { 7303 event: "snapshotting".to_string(), 7304 device_id: None, 7305 }, 7306 &MetaEvent { 7307 event: "snapshotted".to_string(), 7308 device_id: None, 7309 }, 7310 ]; 7311 // See: #5938 7312 thread::sleep(std::time::Duration::new(1, 0)); 7313 assert!(check_latest_events_exact(&latest_events, event_path)); 7314 } 7315 7316 // One thing to note about this test. The virtio-net device is heavily used 7317 // through each ssh command. There's no need to perform a dedicated test to 7318 // verify the migration went well for virtio-net. 7319 #[test] 7320 #[cfg(not(feature = "mshv"))] 7321 fn test_snapshot_restore_hotplug_virtiomem() { 7322 _test_snapshot_restore(true); 7323 } 7324 7325 #[test] 7326 fn test_snapshot_restore_basic() { 7327 _test_snapshot_restore(false); 7328 } 7329 7330 fn _test_snapshot_restore(use_hotplug: bool) { 7331 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7332 let guest = Guest::new(Box::new(focal)); 7333 let kernel_path = direct_kernel_boot_path(); 7334 7335 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7336 7337 let net_id = "net123"; 7338 let net_params = format!( 7339 "id={},tap=,mac={},ip={},mask=255.255.255.0", 7340 net_id, guest.network.guest_mac, guest.network.host_ip 7341 ); 7342 let mut mem_params = "size=2G"; 7343 7344 if use_hotplug { 7345 mem_params = "size=2G,hotplug_method=virtio-mem,hotplug_size=32G" 7346 } 7347 7348 let cloudinit_params = format!( 7349 "path={},iommu=on", 7350 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7351 ); 7352 7353 let socket = temp_vsock_path(&guest.tmp_dir); 7354 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7355 7356 let mut child = GuestCommand::new(&guest) 7357 .args(["--api-socket", &api_socket_source]) 7358 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7359 .args(["--cpus", "boot=4"]) 7360 .args(["--memory", mem_params]) 7361 .args(["--balloon", "size=0"]) 7362 .args(["--kernel", kernel_path.to_str().unwrap()]) 7363 .args([ 7364 "--disk", 7365 format!( 7366 "path={}", 7367 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7368 ) 7369 .as_str(), 7370 cloudinit_params.as_str(), 7371 ]) 7372 .args(["--net", net_params.as_str()]) 7373 .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) 7374 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7375 .capture_output() 7376 .spawn() 7377 .unwrap(); 7378 7379 let console_text = String::from("On a branch floating down river a cricket, singing."); 7380 // Create the snapshot directory 7381 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7382 7383 let r = std::panic::catch_unwind(|| { 7384 guest.wait_vm_boot(None).unwrap(); 7385 7386 // Check the number of vCPUs 7387 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 7388 // Check the guest RAM 7389 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 7390 if use_hotplug { 7391 // Increase guest RAM with virtio-mem 7392 resize_command( 7393 &api_socket_source, 7394 None, 7395 Some(6 << 30), 7396 None, 7397 Some(&event_path), 7398 ); 7399 thread::sleep(std::time::Duration::new(5, 0)); 7400 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7401 // Use balloon to remove RAM from the VM 7402 resize_command( 7403 &api_socket_source, 7404 None, 7405 None, 7406 Some(1 << 30), 7407 Some(&event_path), 7408 ); 7409 thread::sleep(std::time::Duration::new(5, 0)); 7410 let total_memory = guest.get_total_memory().unwrap_or_default(); 7411 assert!(total_memory > 4_800_000); 7412 assert!(total_memory < 5_760_000); 7413 } 7414 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 7415 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7416 7417 // x86_64: We check that removing and adding back the virtio-net device 7418 // does not break the snapshot/restore support for virtio-pci. 7419 // This is an important thing to test as the hotplug will 7420 // trigger a PCI BAR reprogramming, which is a good way of 7421 // checking if the stored resources are correctly restored. 7422 // Unplug the virtio-net device 7423 // AArch64: Device hotplug is currently not supported, skipping here. 7424 #[cfg(target_arch = "x86_64")] 7425 { 7426 assert!(remote_command( 7427 &api_socket_source, 7428 "remove-device", 7429 Some(net_id), 7430 )); 7431 thread::sleep(std::time::Duration::new(10, 0)); 7432 let latest_events = [&MetaEvent { 7433 event: "device-removed".to_string(), 7434 device_id: Some(net_id.to_string()), 7435 }]; 7436 // See: #5938 7437 thread::sleep(std::time::Duration::new(1, 0)); 7438 assert!(check_latest_events_exact(&latest_events, &event_path)); 7439 7440 // Plug the virtio-net device again 7441 assert!(remote_command( 7442 &api_socket_source, 7443 "add-net", 7444 Some(net_params.as_str()), 7445 )); 7446 thread::sleep(std::time::Duration::new(10, 0)); 7447 } 7448 7449 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7450 }); 7451 7452 // Shutdown the source VM and check console output 7453 kill_child(&mut child); 7454 let output = child.wait_with_output().unwrap(); 7455 handle_child_output(r, &output); 7456 7457 let r = std::panic::catch_unwind(|| { 7458 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7459 }); 7460 7461 handle_child_output(r, &output); 7462 7463 // Remove the vsock socket file. 7464 Command::new("rm") 7465 .arg("-f") 7466 .arg(socket.as_str()) 7467 .output() 7468 .unwrap(); 7469 7470 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7471 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7472 7473 // Restore the VM from the snapshot 7474 let mut child = GuestCommand::new(&guest) 7475 .args(["--api-socket", &api_socket_restored]) 7476 .args([ 7477 "--event-monitor", 7478 format!("path={event_path_restored}").as_str(), 7479 ]) 7480 .args([ 7481 "--restore", 7482 format!("source_url=file://{snapshot_dir}").as_str(), 7483 ]) 7484 .capture_output() 7485 .spawn() 7486 .unwrap(); 7487 7488 // Wait for the VM to be restored 7489 thread::sleep(std::time::Duration::new(20, 0)); 7490 let expected_events = [ 7491 &MetaEvent { 7492 event: "starting".to_string(), 7493 device_id: None, 7494 }, 7495 &MetaEvent { 7496 event: "activated".to_string(), 7497 device_id: Some("__console".to_string()), 7498 }, 7499 &MetaEvent { 7500 event: "activated".to_string(), 7501 device_id: Some("__rng".to_string()), 7502 }, 7503 &MetaEvent { 7504 event: "restoring".to_string(), 7505 device_id: None, 7506 }, 7507 ]; 7508 assert!(check_sequential_events( 7509 &expected_events, 7510 &event_path_restored 7511 )); 7512 let latest_events = [&MetaEvent { 7513 event: "restored".to_string(), 7514 device_id: None, 7515 }]; 7516 assert!(check_latest_events_exact( 7517 &latest_events, 7518 &event_path_restored 7519 )); 7520 7521 // Remove the snapshot dir 7522 let _ = remove_dir_all(snapshot_dir.as_str()); 7523 7524 let r = std::panic::catch_unwind(|| { 7525 // Resume the VM 7526 assert!(remote_command(&api_socket_restored, "resume", None)); 7527 // There is no way that we can ensure the 'write()' to the 7528 // event file is completed when the 'resume' request is 7529 // returned successfully, because the 'write()' was done 7530 // asynchronously from a different thread of Cloud 7531 // Hypervisor (e.g. the event-monitor thread). 7532 thread::sleep(std::time::Duration::new(1, 0)); 7533 let latest_events = [ 7534 &MetaEvent { 7535 event: "resuming".to_string(), 7536 device_id: None, 7537 }, 7538 &MetaEvent { 7539 event: "resumed".to_string(), 7540 device_id: None, 7541 }, 7542 ]; 7543 assert!(check_latest_events_exact( 7544 &latest_events, 7545 &event_path_restored 7546 )); 7547 7548 // Perform same checks to validate VM has been properly restored 7549 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 7550 let total_memory = guest.get_total_memory().unwrap_or_default(); 7551 if !use_hotplug { 7552 assert!(total_memory > 1_920_000); 7553 } else { 7554 assert!(total_memory > 4_800_000); 7555 assert!(total_memory < 5_760_000); 7556 // Deflate balloon to restore entire RAM to the VM 7557 resize_command(&api_socket_restored, None, None, Some(0), None); 7558 thread::sleep(std::time::Duration::new(5, 0)); 7559 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7560 // Decrease guest RAM with virtio-mem 7561 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 7562 thread::sleep(std::time::Duration::new(5, 0)); 7563 let total_memory = guest.get_total_memory().unwrap_or_default(); 7564 assert!(total_memory > 4_800_000); 7565 assert!(total_memory < 5_760_000); 7566 } 7567 7568 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7569 }); 7570 // Shutdown the target VM and check console output 7571 kill_child(&mut child); 7572 let output = child.wait_with_output().unwrap(); 7573 handle_child_output(r, &output); 7574 7575 let r = std::panic::catch_unwind(|| { 7576 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7577 }); 7578 7579 handle_child_output(r, &output); 7580 } 7581 7582 #[test] 7583 fn test_snapshot_restore_with_fd() { 7584 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7585 let guest = Guest::new(Box::new(focal)); 7586 let kernel_path = direct_kernel_boot_path(); 7587 7588 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7589 7590 let net_id = "net123"; 7591 let num_queue_pairs: usize = 2; 7592 // use a name that does not conflict with tap dev created from other tests 7593 let tap_name = "chtap999"; 7594 use std::str::FromStr; 7595 let taps = net_util::open_tap( 7596 Some(tap_name), 7597 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 7598 None, 7599 &mut None, 7600 None, 7601 num_queue_pairs, 7602 Some(libc::O_RDWR | libc::O_NONBLOCK), 7603 ) 7604 .unwrap(); 7605 let net_params = format!( 7606 "id={},fd=[{},{}],mac={},ip={},mask=255.255.255.0,num_queues={}", 7607 net_id, 7608 taps[0].as_raw_fd(), 7609 taps[1].as_raw_fd(), 7610 guest.network.guest_mac, 7611 guest.network.host_ip, 7612 num_queue_pairs * 2 7613 ); 7614 7615 let cloudinit_params = format!( 7616 "path={},iommu=on", 7617 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7618 ); 7619 7620 let n_cpu = 2; 7621 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7622 7623 let mut child = GuestCommand::new(&guest) 7624 .args(["--api-socket", &api_socket_source]) 7625 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7626 .args(["--cpus", format!("boot={}", n_cpu).as_str()]) 7627 .args(["--memory", "size=1G"]) 7628 .args(["--kernel", kernel_path.to_str().unwrap()]) 7629 .args([ 7630 "--disk", 7631 format!( 7632 "path={}", 7633 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7634 ) 7635 .as_str(), 7636 cloudinit_params.as_str(), 7637 ]) 7638 .args(["--net", net_params.as_str()]) 7639 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7640 .capture_output() 7641 .spawn() 7642 .unwrap(); 7643 7644 let console_text = String::from("On a branch floating down river a cricket, singing."); 7645 // Create the snapshot directory 7646 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7647 7648 let r = std::panic::catch_unwind(|| { 7649 guest.wait_vm_boot(None).unwrap(); 7650 7651 // close the fds after VM boots, as CH duplicates them before using 7652 for tap in taps.iter() { 7653 unsafe { libc::close(tap.as_raw_fd()) }; 7654 } 7655 7656 // Check the number of vCPUs 7657 assert_eq!(guest.get_cpu_count().unwrap_or_default(), n_cpu); 7658 // Check the guest RAM 7659 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 7660 7661 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 7662 guest.check_devices_common(None, Some(&console_text), None); 7663 7664 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7665 }); 7666 7667 // Shutdown the source VM and check console output 7668 kill_child(&mut child); 7669 let output = child.wait_with_output().unwrap(); 7670 handle_child_output(r, &output); 7671 7672 let r = std::panic::catch_unwind(|| { 7673 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7674 }); 7675 7676 handle_child_output(r, &output); 7677 7678 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7679 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7680 7681 // Restore the VM from the snapshot 7682 let mut child = GuestCommand::new(&guest) 7683 .args(["--api-socket", &api_socket_restored]) 7684 .args([ 7685 "--event-monitor", 7686 format!("path={event_path_restored}").as_str(), 7687 ]) 7688 .capture_output() 7689 .spawn() 7690 .unwrap(); 7691 thread::sleep(std::time::Duration::new(2, 0)); 7692 7693 let taps = net_util::open_tap( 7694 Some(tap_name), 7695 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 7696 None, 7697 &mut None, 7698 None, 7699 num_queue_pairs, 7700 Some(libc::O_RDWR | libc::O_NONBLOCK), 7701 ) 7702 .unwrap(); 7703 let restore_params = format!( 7704 "source_url=file://{},net_fds=[{}@[{},{}]]", 7705 snapshot_dir, 7706 net_id, 7707 taps[0].as_raw_fd(), 7708 taps[1].as_raw_fd() 7709 ); 7710 assert!(remote_command( 7711 &api_socket_restored, 7712 "restore", 7713 Some(restore_params.as_str()) 7714 )); 7715 7716 // Wait for the VM to be restored 7717 thread::sleep(std::time::Duration::new(20, 0)); 7718 7719 // close the fds as CH duplicates them before using 7720 for tap in taps.iter() { 7721 unsafe { libc::close(tap.as_raw_fd()) }; 7722 } 7723 7724 let expected_events = [ 7725 &MetaEvent { 7726 event: "starting".to_string(), 7727 device_id: None, 7728 }, 7729 &MetaEvent { 7730 event: "activated".to_string(), 7731 device_id: Some("__console".to_string()), 7732 }, 7733 &MetaEvent { 7734 event: "activated".to_string(), 7735 device_id: Some("__rng".to_string()), 7736 }, 7737 &MetaEvent { 7738 event: "restoring".to_string(), 7739 device_id: None, 7740 }, 7741 ]; 7742 assert!(check_sequential_events( 7743 &expected_events, 7744 &event_path_restored 7745 )); 7746 let latest_events = [&MetaEvent { 7747 event: "restored".to_string(), 7748 device_id: None, 7749 }]; 7750 assert!(check_latest_events_exact( 7751 &latest_events, 7752 &event_path_restored 7753 )); 7754 7755 // Remove the snapshot dir 7756 let _ = remove_dir_all(snapshot_dir.as_str()); 7757 7758 let r = std::panic::catch_unwind(|| { 7759 // Resume the VM 7760 assert!(remote_command(&api_socket_restored, "resume", None)); 7761 // There is no way that we can ensure the 'write()' to the 7762 // event file is completed when the 'resume' request is 7763 // returned successfully, because the 'write()' was done 7764 // asynchronously from a different thread of Cloud 7765 // Hypervisor (e.g. the event-monitor thread). 7766 thread::sleep(std::time::Duration::new(1, 0)); 7767 let latest_events = [ 7768 &MetaEvent { 7769 event: "resuming".to_string(), 7770 device_id: None, 7771 }, 7772 &MetaEvent { 7773 event: "resumed".to_string(), 7774 device_id: None, 7775 }, 7776 ]; 7777 assert!(check_latest_events_exact( 7778 &latest_events, 7779 &event_path_restored 7780 )); 7781 7782 // Perform same checks to validate VM has been properly restored 7783 assert_eq!(guest.get_cpu_count().unwrap_or_default(), n_cpu); 7784 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 7785 7786 guest.check_devices_common(None, Some(&console_text), None); 7787 }); 7788 // Shutdown the target VM and check console output 7789 kill_child(&mut child); 7790 let output = child.wait_with_output().unwrap(); 7791 handle_child_output(r, &output); 7792 7793 let r = std::panic::catch_unwind(|| { 7794 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7795 }); 7796 7797 handle_child_output(r, &output); 7798 } 7799 7800 #[test] 7801 #[cfg(target_arch = "x86_64")] 7802 fn test_snapshot_restore_pvpanic() { 7803 _test_snapshot_restore_devices(true); 7804 } 7805 7806 fn _test_snapshot_restore_devices(pvpanic: bool) { 7807 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7808 let guest = Guest::new(Box::new(focal)); 7809 let kernel_path = direct_kernel_boot_path(); 7810 7811 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7812 7813 let device_params = { 7814 let mut data = vec![]; 7815 if pvpanic { 7816 data.push("--pvpanic"); 7817 } 7818 data 7819 }; 7820 7821 let socket = temp_vsock_path(&guest.tmp_dir); 7822 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7823 7824 let mut child = GuestCommand::new(&guest) 7825 .args(["--api-socket", &api_socket_source]) 7826 .args(["--event-monitor", format!("path={}", event_path).as_str()]) 7827 .args(["--cpus", "boot=2"]) 7828 .args(["--memory", "size=1G"]) 7829 .args(["--kernel", kernel_path.to_str().unwrap()]) 7830 .default_disks() 7831 .default_net() 7832 .args(["--vsock", format!("cid=3,socket={}", socket).as_str()]) 7833 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7834 .args(device_params) 7835 .capture_output() 7836 .spawn() 7837 .unwrap(); 7838 7839 let console_text = String::from("On a branch floating down river a cricket, singing."); 7840 // Create the snapshot directory 7841 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7842 7843 let r = std::panic::catch_unwind(|| { 7844 guest.wait_vm_boot(None).unwrap(); 7845 7846 // Check the number of vCPUs 7847 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 7848 7849 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7850 }); 7851 7852 // Shutdown the source VM and check console output 7853 kill_child(&mut child); 7854 let output = child.wait_with_output().unwrap(); 7855 handle_child_output(r, &output); 7856 7857 // Remove the vsock socket file. 7858 Command::new("rm") 7859 .arg("-f") 7860 .arg(socket.as_str()) 7861 .output() 7862 .unwrap(); 7863 7864 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7865 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7866 7867 // Restore the VM from the snapshot 7868 let mut child = GuestCommand::new(&guest) 7869 .args(["--api-socket", &api_socket_restored]) 7870 .args([ 7871 "--event-monitor", 7872 format!("path={event_path_restored}").as_str(), 7873 ]) 7874 .args([ 7875 "--restore", 7876 format!("source_url=file://{snapshot_dir}").as_str(), 7877 ]) 7878 .capture_output() 7879 .spawn() 7880 .unwrap(); 7881 7882 // Wait for the VM to be restored 7883 thread::sleep(std::time::Duration::new(20, 0)); 7884 7885 let latest_events = [&MetaEvent { 7886 event: "restored".to_string(), 7887 device_id: None, 7888 }]; 7889 assert!(check_latest_events_exact( 7890 &latest_events, 7891 &event_path_restored 7892 )); 7893 7894 // Remove the snapshot dir 7895 let _ = remove_dir_all(snapshot_dir.as_str()); 7896 7897 let r = std::panic::catch_unwind(|| { 7898 // Resume the VM 7899 assert!(remote_command(&api_socket_restored, "resume", None)); 7900 // There is no way that we can ensure the 'write()' to the 7901 // event file is completed when the 'resume' request is 7902 // returned successfully, because the 'write()' was done 7903 // asynchronously from a different thread of Cloud 7904 // Hypervisor (e.g. the event-monitor thread). 7905 thread::sleep(std::time::Duration::new(1, 0)); 7906 let latest_events = [ 7907 &MetaEvent { 7908 event: "resuming".to_string(), 7909 device_id: None, 7910 }, 7911 &MetaEvent { 7912 event: "resumed".to_string(), 7913 device_id: None, 7914 }, 7915 ]; 7916 assert!(check_latest_events_exact( 7917 &latest_events, 7918 &event_path_restored 7919 )); 7920 7921 // Check the number of vCPUs 7922 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 7923 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7924 7925 if pvpanic { 7926 // Trigger guest a panic 7927 make_guest_panic(&guest); 7928 // Wait a while for guest 7929 thread::sleep(std::time::Duration::new(10, 0)); 7930 7931 let expected_sequential_events = [&MetaEvent { 7932 event: "panic".to_string(), 7933 device_id: None, 7934 }]; 7935 assert!(check_latest_events_exact( 7936 &expected_sequential_events, 7937 &event_path_restored 7938 )); 7939 } 7940 }); 7941 // Shutdown the target VM and check console output 7942 kill_child(&mut child); 7943 let output = child.wait_with_output().unwrap(); 7944 handle_child_output(r, &output); 7945 7946 let r = std::panic::catch_unwind(|| { 7947 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7948 }); 7949 7950 handle_child_output(r, &output); 7951 } 7952 } 7953 7954 mod windows { 7955 use once_cell::sync::Lazy; 7956 7957 use crate::*; 7958 7959 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 7960 7961 struct WindowsGuest { 7962 guest: Guest, 7963 auth: PasswordAuth, 7964 } 7965 7966 trait FsType { 7967 const FS_FAT: u8; 7968 const FS_NTFS: u8; 7969 } 7970 impl FsType for WindowsGuest { 7971 const FS_FAT: u8 = 0; 7972 const FS_NTFS: u8 = 1; 7973 } 7974 7975 impl WindowsGuest { 7976 fn new() -> Self { 7977 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 7978 let guest = Guest::new(Box::new(disk)); 7979 let auth = PasswordAuth { 7980 username: String::from("administrator"), 7981 password: String::from("Admin123"), 7982 }; 7983 7984 WindowsGuest { guest, auth } 7985 } 7986 7987 fn guest(&self) -> &Guest { 7988 &self.guest 7989 } 7990 7991 fn ssh_cmd(&self, cmd: &str) -> String { 7992 ssh_command_ip_with_auth( 7993 cmd, 7994 &self.auth, 7995 &self.guest.network.guest_ip, 7996 DEFAULT_SSH_RETRIES, 7997 DEFAULT_SSH_TIMEOUT, 7998 ) 7999 .unwrap() 8000 } 8001 8002 fn cpu_count(&self) -> u8 { 8003 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 8004 .trim() 8005 .parse::<u8>() 8006 .unwrap_or(0) 8007 } 8008 8009 fn ram_size(&self) -> usize { 8010 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 8011 .trim() 8012 .parse::<usize>() 8013 .unwrap_or(0) 8014 } 8015 8016 fn netdev_count(&self) -> u8 { 8017 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 8018 .trim() 8019 .parse::<u8>() 8020 .unwrap_or(0) 8021 } 8022 8023 fn disk_count(&self) -> u8 { 8024 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 8025 .trim() 8026 .parse::<u8>() 8027 .unwrap_or(0) 8028 } 8029 8030 fn reboot(&self) { 8031 let _ = self.ssh_cmd("shutdown /r /t 0"); 8032 } 8033 8034 fn shutdown(&self) { 8035 let _ = self.ssh_cmd("shutdown /s /t 0"); 8036 } 8037 8038 fn run_dnsmasq(&self) -> std::process::Child { 8039 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 8040 let dhcp_host = format!( 8041 "--dhcp-host={},{}", 8042 self.guest.network.guest_mac, self.guest.network.guest_ip 8043 ); 8044 let dhcp_range = format!( 8045 "--dhcp-range=eth,{},{}", 8046 self.guest.network.guest_ip, self.guest.network.guest_ip 8047 ); 8048 8049 Command::new("dnsmasq") 8050 .arg("--no-daemon") 8051 .arg("--log-queries") 8052 .arg(listen_address.as_str()) 8053 .arg("--except-interface=lo") 8054 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 8055 .arg("--conf-file=/dev/null") 8056 .arg(dhcp_host.as_str()) 8057 .arg(dhcp_range.as_str()) 8058 .spawn() 8059 .unwrap() 8060 } 8061 8062 // TODO Cleanup image file explicitly after test, if there's some space issues. 8063 fn disk_new(&self, fs: u8, sz: usize) -> String { 8064 let mut guard = NEXT_DISK_ID.lock().unwrap(); 8065 let id = *guard; 8066 *guard = id + 1; 8067 8068 let img = PathBuf::from(format!("/tmp/test-hotplug-{id}.raw")); 8069 let _ = fs::remove_file(&img); 8070 8071 // Create an image file 8072 let out = Command::new("qemu-img") 8073 .args([ 8074 "create", 8075 "-f", 8076 "raw", 8077 img.to_str().unwrap(), 8078 format!("{sz}m").as_str(), 8079 ]) 8080 .output() 8081 .expect("qemu-img command failed") 8082 .stdout; 8083 println!("{out:?}"); 8084 8085 // Associate image to a loop device 8086 let out = Command::new("losetup") 8087 .args(["--show", "-f", img.to_str().unwrap()]) 8088 .output() 8089 .expect("failed to create loop device") 8090 .stdout; 8091 let _tmp = String::from_utf8_lossy(&out); 8092 let loop_dev = _tmp.trim(); 8093 println!("{out:?}"); 8094 8095 // Create a partition table 8096 // echo 'type=7' | sudo sfdisk "${LOOP}" 8097 let mut child = Command::new("sfdisk") 8098 .args([loop_dev]) 8099 .stdin(Stdio::piped()) 8100 .spawn() 8101 .unwrap(); 8102 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 8103 stdin 8104 .write_all("type=7".as_bytes()) 8105 .expect("failed to write stdin"); 8106 let out = child.wait_with_output().expect("sfdisk failed").stdout; 8107 println!("{out:?}"); 8108 8109 // Disengage the loop device 8110 let out = Command::new("losetup") 8111 .args(["-d", loop_dev]) 8112 .output() 8113 .expect("loop device not found") 8114 .stdout; 8115 println!("{out:?}"); 8116 8117 // Re-associate loop device pointing to the partition only 8118 let out = Command::new("losetup") 8119 .args([ 8120 "--show", 8121 "--offset", 8122 (512 * 2048).to_string().as_str(), 8123 "-f", 8124 img.to_str().unwrap(), 8125 ]) 8126 .output() 8127 .expect("failed to create loop device") 8128 .stdout; 8129 let _tmp = String::from_utf8_lossy(&out); 8130 let loop_dev = _tmp.trim(); 8131 println!("{out:?}"); 8132 8133 // Create filesystem. 8134 let fs_cmd = match fs { 8135 WindowsGuest::FS_FAT => "mkfs.msdos", 8136 WindowsGuest::FS_NTFS => "mkfs.ntfs", 8137 _ => panic!("Unknown filesystem type '{fs}'"), 8138 }; 8139 let out = Command::new(fs_cmd) 8140 .args([&loop_dev]) 8141 .output() 8142 .unwrap_or_else(|_| panic!("{fs_cmd} failed")) 8143 .stdout; 8144 println!("{out:?}"); 8145 8146 // Disengage the loop device 8147 let out = Command::new("losetup") 8148 .args(["-d", loop_dev]) 8149 .output() 8150 .unwrap_or_else(|_| panic!("loop device '{loop_dev}' not found")) 8151 .stdout; 8152 println!("{out:?}"); 8153 8154 img.to_str().unwrap().to_string() 8155 } 8156 8157 fn disks_set_rw(&self) { 8158 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 8159 } 8160 8161 fn disks_online(&self) { 8162 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 8163 } 8164 8165 fn disk_file_put(&self, fname: &str, data: &str) { 8166 let _ = self.ssh_cmd(&format!( 8167 "powershell -Command \"'{data}' | Set-Content -Path {fname}\"" 8168 )); 8169 } 8170 8171 fn disk_file_read(&self, fname: &str) -> String { 8172 self.ssh_cmd(&format!( 8173 "powershell -Command \"Get-Content -Path {fname}\"" 8174 )) 8175 } 8176 8177 fn wait_for_boot(&self) -> bool { 8178 let cmd = "dir /b c:\\ | find \"Windows\""; 8179 let tmo_max = 180; 8180 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 8181 // interval must be small. 8182 let tmo_int = 2; 8183 let out = ssh_command_ip_with_auth( 8184 cmd, 8185 &self.auth, 8186 &self.guest.network.guest_ip, 8187 { 8188 let mut ret = 1; 8189 let mut tmo_acc = 0; 8190 loop { 8191 tmo_acc += tmo_int * ret; 8192 if tmo_acc >= tmo_max { 8193 break; 8194 } 8195 ret += 1; 8196 } 8197 ret 8198 }, 8199 tmo_int, 8200 ) 8201 .unwrap(); 8202 8203 if "Windows" == out.trim() { 8204 return true; 8205 } 8206 8207 false 8208 } 8209 } 8210 8211 fn vcpu_threads_count(pid: u32) -> u8 { 8212 // ps -T -p 12345 | grep vcpu | wc -l 8213 let out = Command::new("ps") 8214 .args(["-T", "-p", format!("{pid}").as_str()]) 8215 .output() 8216 .expect("ps command failed") 8217 .stdout; 8218 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 8219 } 8220 8221 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 8222 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 8223 let out = Command::new("ps") 8224 .args(["-T", "-p", format!("{pid}").as_str()]) 8225 .output() 8226 .expect("ps command failed") 8227 .stdout; 8228 let mut n = 0; 8229 String::from_utf8_lossy(&out) 8230 .split_whitespace() 8231 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 8232 n 8233 } 8234 8235 fn disk_ctrl_threads_count(pid: u32) -> u8 { 8236 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 8237 let out = Command::new("ps") 8238 .args(["-T", "-p", format!("{pid}").as_str()]) 8239 .output() 8240 .expect("ps command failed") 8241 .stdout; 8242 let mut n = 0; 8243 String::from_utf8_lossy(&out) 8244 .split_whitespace() 8245 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 8246 n 8247 } 8248 8249 #[test] 8250 fn test_windows_guest() { 8251 let windows_guest = WindowsGuest::new(); 8252 8253 let mut child = GuestCommand::new(windows_guest.guest()) 8254 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8255 .args(["--memory", "size=4G"]) 8256 .args(["--kernel", edk2_path().to_str().unwrap()]) 8257 .args(["--serial", "tty"]) 8258 .args(["--console", "off"]) 8259 .default_disks() 8260 .default_net() 8261 .capture_output() 8262 .spawn() 8263 .unwrap(); 8264 8265 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8266 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8267 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8268 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8269 8270 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8271 8272 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8273 8274 let r = std::panic::catch_unwind(|| { 8275 // Wait to make sure Windows boots up 8276 assert!(windows_guest.wait_for_boot()); 8277 8278 windows_guest.shutdown(); 8279 }); 8280 8281 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8282 let _ = child.kill(); 8283 let output = child.wait_with_output().unwrap(); 8284 8285 let _ = child_dnsmasq.kill(); 8286 let _ = child_dnsmasq.wait(); 8287 8288 handle_child_output(r, &output); 8289 } 8290 8291 #[test] 8292 fn test_windows_guest_multiple_queues() { 8293 let windows_guest = WindowsGuest::new(); 8294 8295 let mut ovmf_path = dirs::home_dir().unwrap(); 8296 ovmf_path.push("workloads"); 8297 ovmf_path.push(OVMF_NAME); 8298 8299 let mut child = GuestCommand::new(windows_guest.guest()) 8300 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 8301 .args(["--memory", "size=4G"]) 8302 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8303 .args(["--serial", "tty"]) 8304 .args(["--console", "off"]) 8305 .args([ 8306 "--disk", 8307 format!( 8308 "path={},num_queues=4", 8309 windows_guest 8310 .guest() 8311 .disk_config 8312 .disk(DiskType::OperatingSystem) 8313 .unwrap() 8314 ) 8315 .as_str(), 8316 ]) 8317 .args([ 8318 "--net", 8319 format!( 8320 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 8321 windows_guest.guest().network.guest_mac, 8322 windows_guest.guest().network.host_ip 8323 ) 8324 .as_str(), 8325 ]) 8326 .capture_output() 8327 .spawn() 8328 .unwrap(); 8329 8330 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8331 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8332 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8333 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8334 8335 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8336 8337 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8338 8339 let r = std::panic::catch_unwind(|| { 8340 // Wait to make sure Windows boots up 8341 assert!(windows_guest.wait_for_boot()); 8342 8343 windows_guest.shutdown(); 8344 }); 8345 8346 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8347 let _ = child.kill(); 8348 let output = child.wait_with_output().unwrap(); 8349 8350 let _ = child_dnsmasq.kill(); 8351 let _ = child_dnsmasq.wait(); 8352 8353 handle_child_output(r, &output); 8354 } 8355 8356 #[test] 8357 #[cfg(not(feature = "mshv"))] 8358 #[ignore = "See #4327"] 8359 fn test_windows_guest_snapshot_restore() { 8360 let windows_guest = WindowsGuest::new(); 8361 8362 let mut ovmf_path = dirs::home_dir().unwrap(); 8363 ovmf_path.push("workloads"); 8364 ovmf_path.push(OVMF_NAME); 8365 8366 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8367 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 8368 8369 let mut child = GuestCommand::new(windows_guest.guest()) 8370 .args(["--api-socket", &api_socket_source]) 8371 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8372 .args(["--memory", "size=4G"]) 8373 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8374 .args(["--serial", "tty"]) 8375 .args(["--console", "off"]) 8376 .default_disks() 8377 .default_net() 8378 .capture_output() 8379 .spawn() 8380 .unwrap(); 8381 8382 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8383 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8384 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8385 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8386 8387 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8388 8389 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8390 8391 // Wait to make sure Windows boots up 8392 assert!(windows_guest.wait_for_boot()); 8393 8394 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 8395 8396 // Pause the VM 8397 assert!(remote_command(&api_socket_source, "pause", None)); 8398 8399 // Take a snapshot from the VM 8400 assert!(remote_command( 8401 &api_socket_source, 8402 "snapshot", 8403 Some(format!("file://{snapshot_dir}").as_str()), 8404 )); 8405 8406 // Wait to make sure the snapshot is completed 8407 thread::sleep(std::time::Duration::new(30, 0)); 8408 8409 let _ = child.kill(); 8410 child.wait().unwrap(); 8411 8412 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 8413 8414 // Restore the VM from the snapshot 8415 let mut child = GuestCommand::new(windows_guest.guest()) 8416 .args(["--api-socket", &api_socket_restored]) 8417 .args([ 8418 "--restore", 8419 format!("source_url=file://{snapshot_dir}").as_str(), 8420 ]) 8421 .capture_output() 8422 .spawn() 8423 .unwrap(); 8424 8425 // Wait for the VM to be restored 8426 thread::sleep(std::time::Duration::new(20, 0)); 8427 8428 let r = std::panic::catch_unwind(|| { 8429 // Resume the VM 8430 assert!(remote_command(&api_socket_restored, "resume", None)); 8431 8432 windows_guest.shutdown(); 8433 }); 8434 8435 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8436 let _ = child.kill(); 8437 let output = child.wait_with_output().unwrap(); 8438 8439 let _ = child_dnsmasq.kill(); 8440 let _ = child_dnsmasq.wait(); 8441 8442 handle_child_output(r, &output); 8443 } 8444 8445 #[test] 8446 #[cfg(not(feature = "mshv"))] 8447 #[cfg(not(target_arch = "aarch64"))] 8448 fn test_windows_guest_cpu_hotplug() { 8449 let windows_guest = WindowsGuest::new(); 8450 8451 let mut ovmf_path = dirs::home_dir().unwrap(); 8452 ovmf_path.push("workloads"); 8453 ovmf_path.push(OVMF_NAME); 8454 8455 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8456 let api_socket = temp_api_path(&tmp_dir); 8457 8458 let mut child = GuestCommand::new(windows_guest.guest()) 8459 .args(["--api-socket", &api_socket]) 8460 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 8461 .args(["--memory", "size=4G"]) 8462 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8463 .args(["--serial", "tty"]) 8464 .args(["--console", "off"]) 8465 .default_disks() 8466 .default_net() 8467 .capture_output() 8468 .spawn() 8469 .unwrap(); 8470 8471 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8472 8473 let r = std::panic::catch_unwind(|| { 8474 // Wait to make sure Windows boots up 8475 assert!(windows_guest.wait_for_boot()); 8476 8477 let vcpu_num = 2; 8478 // Check the initial number of CPUs the guest sees 8479 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8480 // Check the initial number of vcpu threads in the CH process 8481 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8482 8483 let vcpu_num = 6; 8484 // Hotplug some CPUs 8485 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8486 // Wait to make sure CPUs are added 8487 thread::sleep(std::time::Duration::new(10, 0)); 8488 // Check the guest sees the correct number 8489 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8490 // Check the CH process has the correct number of vcpu threads 8491 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8492 8493 let vcpu_num = 4; 8494 // Remove some CPUs. Note that Windows doesn't support hot-remove. 8495 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8496 // Wait to make sure CPUs are removed 8497 thread::sleep(std::time::Duration::new(10, 0)); 8498 // Reboot to let Windows catch up 8499 windows_guest.reboot(); 8500 // Wait to make sure Windows completely rebooted 8501 thread::sleep(std::time::Duration::new(60, 0)); 8502 // Check the guest sees the correct number 8503 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8504 // Check the CH process has the correct number of vcpu threads 8505 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8506 8507 windows_guest.shutdown(); 8508 }); 8509 8510 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8511 let _ = child.kill(); 8512 let output = child.wait_with_output().unwrap(); 8513 8514 let _ = child_dnsmasq.kill(); 8515 let _ = child_dnsmasq.wait(); 8516 8517 handle_child_output(r, &output); 8518 } 8519 8520 #[test] 8521 #[cfg(not(feature = "mshv"))] 8522 #[cfg(not(target_arch = "aarch64"))] 8523 fn test_windows_guest_ram_hotplug() { 8524 let windows_guest = WindowsGuest::new(); 8525 8526 let mut ovmf_path = dirs::home_dir().unwrap(); 8527 ovmf_path.push("workloads"); 8528 ovmf_path.push(OVMF_NAME); 8529 8530 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8531 let api_socket = temp_api_path(&tmp_dir); 8532 8533 let mut child = GuestCommand::new(windows_guest.guest()) 8534 .args(["--api-socket", &api_socket]) 8535 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8536 .args(["--memory", "size=2G,hotplug_size=5G"]) 8537 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8538 .args(["--serial", "tty"]) 8539 .args(["--console", "off"]) 8540 .default_disks() 8541 .default_net() 8542 .capture_output() 8543 .spawn() 8544 .unwrap(); 8545 8546 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8547 8548 let r = std::panic::catch_unwind(|| { 8549 // Wait to make sure Windows boots up 8550 assert!(windows_guest.wait_for_boot()); 8551 8552 let ram_size = 2 * 1024 * 1024 * 1024; 8553 // Check the initial number of RAM the guest sees 8554 let current_ram_size = windows_guest.ram_size(); 8555 // This size seems to be reserved by the system and thus the 8556 // reported amount differs by this constant value. 8557 let reserved_ram_size = ram_size - current_ram_size; 8558 // Verify that there's not more than 4mb constant diff wasted 8559 // by the reserved ram. 8560 assert!(reserved_ram_size < 4 * 1024 * 1024); 8561 8562 let ram_size = 4 * 1024 * 1024 * 1024; 8563 // Hotplug some RAM 8564 resize_command(&api_socket, None, Some(ram_size), None, None); 8565 // Wait to make sure RAM has been added 8566 thread::sleep(std::time::Duration::new(10, 0)); 8567 // Check the guest sees the correct number 8568 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8569 8570 let ram_size = 3 * 1024 * 1024 * 1024; 8571 // Unplug some RAM. Note that hot-remove most likely won't work. 8572 resize_command(&api_socket, None, Some(ram_size), None, None); 8573 // Wait to make sure RAM has been added 8574 thread::sleep(std::time::Duration::new(10, 0)); 8575 // Reboot to let Windows catch up 8576 windows_guest.reboot(); 8577 // Wait to make sure guest completely rebooted 8578 thread::sleep(std::time::Duration::new(60, 0)); 8579 // Check the guest sees the correct number 8580 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8581 8582 windows_guest.shutdown(); 8583 }); 8584 8585 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8586 let _ = child.kill(); 8587 let output = child.wait_with_output().unwrap(); 8588 8589 let _ = child_dnsmasq.kill(); 8590 let _ = child_dnsmasq.wait(); 8591 8592 handle_child_output(r, &output); 8593 } 8594 8595 #[test] 8596 #[cfg(not(feature = "mshv"))] 8597 fn test_windows_guest_netdev_hotplug() { 8598 let windows_guest = WindowsGuest::new(); 8599 8600 let mut ovmf_path = dirs::home_dir().unwrap(); 8601 ovmf_path.push("workloads"); 8602 ovmf_path.push(OVMF_NAME); 8603 8604 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8605 let api_socket = temp_api_path(&tmp_dir); 8606 8607 let mut child = GuestCommand::new(windows_guest.guest()) 8608 .args(["--api-socket", &api_socket]) 8609 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8610 .args(["--memory", "size=4G"]) 8611 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8612 .args(["--serial", "tty"]) 8613 .args(["--console", "off"]) 8614 .default_disks() 8615 .default_net() 8616 .capture_output() 8617 .spawn() 8618 .unwrap(); 8619 8620 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8621 8622 let r = std::panic::catch_unwind(|| { 8623 // Wait to make sure Windows boots up 8624 assert!(windows_guest.wait_for_boot()); 8625 8626 // Initially present network device 8627 let netdev_num = 1; 8628 assert_eq!(windows_guest.netdev_count(), netdev_num); 8629 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8630 8631 // Hotplug network device 8632 let (cmd_success, cmd_output) = remote_command_w_output( 8633 &api_socket, 8634 "add-net", 8635 Some(windows_guest.guest().default_net_string().as_str()), 8636 ); 8637 assert!(cmd_success); 8638 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 8639 thread::sleep(std::time::Duration::new(5, 0)); 8640 // Verify the device is on the system 8641 let netdev_num = 2; 8642 assert_eq!(windows_guest.netdev_count(), netdev_num); 8643 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8644 8645 // Remove network device 8646 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 8647 assert!(cmd_success); 8648 thread::sleep(std::time::Duration::new(5, 0)); 8649 // Verify the device has been removed 8650 let netdev_num = 1; 8651 assert_eq!(windows_guest.netdev_count(), netdev_num); 8652 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8653 8654 windows_guest.shutdown(); 8655 }); 8656 8657 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8658 let _ = child.kill(); 8659 let output = child.wait_with_output().unwrap(); 8660 8661 let _ = child_dnsmasq.kill(); 8662 let _ = child_dnsmasq.wait(); 8663 8664 handle_child_output(r, &output); 8665 } 8666 8667 #[test] 8668 #[ignore = "See #6037"] 8669 #[cfg(not(feature = "mshv"))] 8670 #[cfg(not(target_arch = "aarch64"))] 8671 fn test_windows_guest_disk_hotplug() { 8672 let windows_guest = WindowsGuest::new(); 8673 8674 let mut ovmf_path = dirs::home_dir().unwrap(); 8675 ovmf_path.push("workloads"); 8676 ovmf_path.push(OVMF_NAME); 8677 8678 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8679 let api_socket = temp_api_path(&tmp_dir); 8680 8681 let mut child = GuestCommand::new(windows_guest.guest()) 8682 .args(["--api-socket", &api_socket]) 8683 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8684 .args(["--memory", "size=4G"]) 8685 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8686 .args(["--serial", "tty"]) 8687 .args(["--console", "off"]) 8688 .default_disks() 8689 .default_net() 8690 .capture_output() 8691 .spawn() 8692 .unwrap(); 8693 8694 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8695 8696 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 8697 8698 let r = std::panic::catch_unwind(|| { 8699 // Wait to make sure Windows boots up 8700 assert!(windows_guest.wait_for_boot()); 8701 8702 // Initially present disk device 8703 let disk_num = 1; 8704 assert_eq!(windows_guest.disk_count(), disk_num); 8705 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8706 8707 // Hotplug disk device 8708 let (cmd_success, cmd_output) = remote_command_w_output( 8709 &api_socket, 8710 "add-disk", 8711 Some(format!("path={disk},readonly=off").as_str()), 8712 ); 8713 assert!(cmd_success); 8714 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 8715 thread::sleep(std::time::Duration::new(5, 0)); 8716 // Online disk device 8717 windows_guest.disks_set_rw(); 8718 windows_guest.disks_online(); 8719 // Verify the device is on the system 8720 let disk_num = 2; 8721 assert_eq!(windows_guest.disk_count(), disk_num); 8722 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8723 8724 let data = "hello"; 8725 let fname = "d:\\world"; 8726 windows_guest.disk_file_put(fname, data); 8727 8728 // Unmount disk device 8729 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 8730 assert!(cmd_success); 8731 thread::sleep(std::time::Duration::new(5, 0)); 8732 // Verify the device has been removed 8733 let disk_num = 1; 8734 assert_eq!(windows_guest.disk_count(), disk_num); 8735 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8736 8737 // Remount and check the file exists with the expected contents 8738 let (cmd_success, _cmd_output) = remote_command_w_output( 8739 &api_socket, 8740 "add-disk", 8741 Some(format!("path={disk},readonly=off").as_str()), 8742 ); 8743 assert!(cmd_success); 8744 thread::sleep(std::time::Duration::new(5, 0)); 8745 let out = windows_guest.disk_file_read(fname); 8746 assert_eq!(data, out.trim()); 8747 8748 // Intentionally no unmount, it'll happen at shutdown. 8749 8750 windows_guest.shutdown(); 8751 }); 8752 8753 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8754 let _ = child.kill(); 8755 let output = child.wait_with_output().unwrap(); 8756 8757 let _ = child_dnsmasq.kill(); 8758 let _ = child_dnsmasq.wait(); 8759 8760 handle_child_output(r, &output); 8761 } 8762 8763 #[test] 8764 #[ignore = "See #6037"] 8765 #[cfg(not(feature = "mshv"))] 8766 #[cfg(not(target_arch = "aarch64"))] 8767 fn test_windows_guest_disk_hotplug_multi() { 8768 let windows_guest = WindowsGuest::new(); 8769 8770 let mut ovmf_path = dirs::home_dir().unwrap(); 8771 ovmf_path.push("workloads"); 8772 ovmf_path.push(OVMF_NAME); 8773 8774 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8775 let api_socket = temp_api_path(&tmp_dir); 8776 8777 let mut child = GuestCommand::new(windows_guest.guest()) 8778 .args(["--api-socket", &api_socket]) 8779 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8780 .args(["--memory", "size=2G"]) 8781 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8782 .args(["--serial", "tty"]) 8783 .args(["--console", "off"]) 8784 .default_disks() 8785 .default_net() 8786 .capture_output() 8787 .spawn() 8788 .unwrap(); 8789 8790 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8791 8792 // Predefined data to used at various test stages 8793 let disk_test_data: [[String; 4]; 2] = [ 8794 [ 8795 "_disk2".to_string(), 8796 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 8797 "d:\\world".to_string(), 8798 "hello".to_string(), 8799 ], 8800 [ 8801 "_disk3".to_string(), 8802 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 8803 "e:\\hello".to_string(), 8804 "world".to_string(), 8805 ], 8806 ]; 8807 8808 let r = std::panic::catch_unwind(|| { 8809 // Wait to make sure Windows boots up 8810 assert!(windows_guest.wait_for_boot()); 8811 8812 // Initially present disk device 8813 let disk_num = 1; 8814 assert_eq!(windows_guest.disk_count(), disk_num); 8815 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8816 8817 for it in &disk_test_data { 8818 let disk_id = it[0].as_str(); 8819 let disk = it[1].as_str(); 8820 // Hotplug disk device 8821 let (cmd_success, cmd_output) = remote_command_w_output( 8822 &api_socket, 8823 "add-disk", 8824 Some(format!("path={disk},readonly=off").as_str()), 8825 ); 8826 assert!(cmd_success); 8827 assert!(String::from_utf8_lossy(&cmd_output) 8828 .contains(format!("\"id\":\"{disk_id}\"").as_str())); 8829 thread::sleep(std::time::Duration::new(5, 0)); 8830 // Online disk devices 8831 windows_guest.disks_set_rw(); 8832 windows_guest.disks_online(); 8833 } 8834 // Verify the devices are on the system 8835 let disk_num = (disk_test_data.len() + 1) as u8; 8836 assert_eq!(windows_guest.disk_count(), disk_num); 8837 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8838 8839 // Put test data 8840 for it in &disk_test_data { 8841 let fname = it[2].as_str(); 8842 let data = it[3].as_str(); 8843 windows_guest.disk_file_put(fname, data); 8844 } 8845 8846 // Unmount disk devices 8847 for it in &disk_test_data { 8848 let disk_id = it[0].as_str(); 8849 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 8850 assert!(cmd_success); 8851 thread::sleep(std::time::Duration::new(5, 0)); 8852 } 8853 8854 // Verify the devices have been removed 8855 let disk_num = 1; 8856 assert_eq!(windows_guest.disk_count(), disk_num); 8857 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8858 8859 // Remount 8860 for it in &disk_test_data { 8861 let disk = it[1].as_str(); 8862 let (cmd_success, _cmd_output) = remote_command_w_output( 8863 &api_socket, 8864 "add-disk", 8865 Some(format!("path={disk},readonly=off").as_str()), 8866 ); 8867 assert!(cmd_success); 8868 thread::sleep(std::time::Duration::new(5, 0)); 8869 } 8870 8871 // Check the files exists with the expected contents 8872 for it in &disk_test_data { 8873 let fname = it[2].as_str(); 8874 let data = it[3].as_str(); 8875 let out = windows_guest.disk_file_read(fname); 8876 assert_eq!(data, out.trim()); 8877 } 8878 8879 // Intentionally no unmount, it'll happen at shutdown. 8880 8881 windows_guest.shutdown(); 8882 }); 8883 8884 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8885 let _ = child.kill(); 8886 let output = child.wait_with_output().unwrap(); 8887 8888 let _ = child_dnsmasq.kill(); 8889 let _ = child_dnsmasq.wait(); 8890 8891 handle_child_output(r, &output); 8892 } 8893 8894 #[test] 8895 #[cfg(not(feature = "mshv"))] 8896 #[cfg(not(target_arch = "aarch64"))] 8897 fn test_windows_guest_netdev_multi() { 8898 let windows_guest = WindowsGuest::new(); 8899 8900 let mut ovmf_path = dirs::home_dir().unwrap(); 8901 ovmf_path.push("workloads"); 8902 ovmf_path.push(OVMF_NAME); 8903 8904 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8905 let api_socket = temp_api_path(&tmp_dir); 8906 8907 let mut child = GuestCommand::new(windows_guest.guest()) 8908 .args(["--api-socket", &api_socket]) 8909 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8910 .args(["--memory", "size=4G"]) 8911 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8912 .args(["--serial", "tty"]) 8913 .args(["--console", "off"]) 8914 .default_disks() 8915 // The multi net dev config is borrowed from test_multiple_network_interfaces 8916 .args([ 8917 "--net", 8918 windows_guest.guest().default_net_string().as_str(), 8919 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 8920 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 8921 ]) 8922 .capture_output() 8923 .spawn() 8924 .unwrap(); 8925 8926 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8927 8928 let r = std::panic::catch_unwind(|| { 8929 // Wait to make sure Windows boots up 8930 assert!(windows_guest.wait_for_boot()); 8931 8932 let netdev_num = 3; 8933 assert_eq!(windows_guest.netdev_count(), netdev_num); 8934 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8935 8936 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 8937 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 8938 8939 windows_guest.shutdown(); 8940 }); 8941 8942 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8943 let _ = child.kill(); 8944 let output = child.wait_with_output().unwrap(); 8945 8946 let _ = child_dnsmasq.kill(); 8947 let _ = child_dnsmasq.wait(); 8948 8949 handle_child_output(r, &output); 8950 } 8951 } 8952 8953 #[cfg(target_arch = "x86_64")] 8954 mod sgx { 8955 use crate::*; 8956 8957 #[test] 8958 fn test_sgx() { 8959 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 8960 let jammy = UbuntuDiskConfig::new(jammy_image); 8961 let guest = Guest::new(Box::new(jammy)); 8962 8963 let mut child = GuestCommand::new(&guest) 8964 .args(["--cpus", "boot=1"]) 8965 .args(["--memory", "size=512M"]) 8966 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8967 .default_disks() 8968 .default_net() 8969 .args(["--sgx-epc", "id=epc0,size=64M"]) 8970 .capture_output() 8971 .spawn() 8972 .unwrap(); 8973 8974 let r = std::panic::catch_unwind(|| { 8975 guest.wait_vm_boot(None).unwrap(); 8976 8977 // Check if SGX is correctly detected in the guest. 8978 guest.check_sgx_support().unwrap(); 8979 8980 // Validate the SGX EPC section is 64MiB. 8981 assert_eq!( 8982 guest 8983 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 8984 .unwrap() 8985 .trim(), 8986 "0x0000000004000000" 8987 ); 8988 }); 8989 8990 let _ = child.kill(); 8991 let output = child.wait_with_output().unwrap(); 8992 8993 handle_child_output(r, &output); 8994 } 8995 } 8996 8997 #[cfg(target_arch = "x86_64")] 8998 mod vfio { 8999 use crate::*; 9000 9001 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 9002 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 9003 let guest = Guest::new(Box::new(jammy)); 9004 let api_socket = temp_api_path(&guest.tmp_dir); 9005 9006 let mut child = GuestCommand::new(&guest) 9007 .args(["--cpus", "boot=4"]) 9008 .args([ 9009 "--memory", 9010 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(), 9011 ]) 9012 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9013 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 9014 .args(["--api-socket", &api_socket]) 9015 .default_disks() 9016 .default_net() 9017 .capture_output() 9018 .spawn() 9019 .unwrap(); 9020 9021 let r = std::panic::catch_unwind(|| { 9022 guest.wait_vm_boot(None).unwrap(); 9023 9024 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9025 9026 guest.enable_memory_hotplug(); 9027 9028 // Add RAM to the VM 9029 let desired_ram = 6 << 30; 9030 resize_command(&api_socket, None, Some(desired_ram), None, None); 9031 thread::sleep(std::time::Duration::new(30, 0)); 9032 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9033 9034 // Check the VFIO device works when RAM is increased to 6GiB 9035 guest.check_nvidia_gpu(); 9036 }); 9037 9038 let _ = child.kill(); 9039 let output = child.wait_with_output().unwrap(); 9040 9041 handle_child_output(r, &output); 9042 } 9043 9044 #[test] 9045 fn test_nvidia_card_memory_hotplug_acpi() { 9046 test_nvidia_card_memory_hotplug("acpi") 9047 } 9048 9049 #[test] 9050 fn test_nvidia_card_memory_hotplug_virtio_mem() { 9051 test_nvidia_card_memory_hotplug("virtio-mem") 9052 } 9053 9054 #[test] 9055 fn test_nvidia_card_pci_hotplug() { 9056 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 9057 let guest = Guest::new(Box::new(jammy)); 9058 let api_socket = temp_api_path(&guest.tmp_dir); 9059 9060 let mut child = GuestCommand::new(&guest) 9061 .args(["--cpus", "boot=4"]) 9062 .args(["--memory", "size=4G"]) 9063 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9064 .args(["--api-socket", &api_socket]) 9065 .default_disks() 9066 .default_net() 9067 .capture_output() 9068 .spawn() 9069 .unwrap(); 9070 9071 let r = std::panic::catch_unwind(|| { 9072 guest.wait_vm_boot(None).unwrap(); 9073 9074 // Hotplug the card to the VM 9075 let (cmd_success, cmd_output) = remote_command_w_output( 9076 &api_socket, 9077 "add-device", 9078 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 9079 ); 9080 assert!(cmd_success); 9081 assert!(String::from_utf8_lossy(&cmd_output) 9082 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 9083 9084 thread::sleep(std::time::Duration::new(10, 0)); 9085 9086 // Check the VFIO device works after hotplug 9087 guest.check_nvidia_gpu(); 9088 }); 9089 9090 let _ = child.kill(); 9091 let output = child.wait_with_output().unwrap(); 9092 9093 handle_child_output(r, &output); 9094 } 9095 9096 #[test] 9097 fn test_nvidia_card_reboot() { 9098 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 9099 let guest = Guest::new(Box::new(jammy)); 9100 let api_socket = temp_api_path(&guest.tmp_dir); 9101 9102 let mut child = GuestCommand::new(&guest) 9103 .args(["--cpus", "boot=4"]) 9104 .args(["--memory", "size=4G"]) 9105 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9106 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 9107 .args(["--api-socket", &api_socket]) 9108 .default_disks() 9109 .default_net() 9110 .capture_output() 9111 .spawn() 9112 .unwrap(); 9113 9114 let r = std::panic::catch_unwind(|| { 9115 guest.wait_vm_boot(None).unwrap(); 9116 9117 // Check the VFIO device works after boot 9118 guest.check_nvidia_gpu(); 9119 9120 guest.reboot_linux(0, None); 9121 9122 // Check the VFIO device works after reboot 9123 guest.check_nvidia_gpu(); 9124 }); 9125 9126 let _ = child.kill(); 9127 let output = child.wait_with_output().unwrap(); 9128 9129 handle_child_output(r, &output); 9130 } 9131 } 9132 9133 mod live_migration { 9134 use crate::*; 9135 9136 fn start_live_migration( 9137 migration_socket: &str, 9138 src_api_socket: &str, 9139 dest_api_socket: &str, 9140 local: bool, 9141 ) -> bool { 9142 // Start to receive migration from the destination VM 9143 let mut receive_migration = Command::new(clh_command("ch-remote")) 9144 .args([ 9145 &format!("--api-socket={dest_api_socket}"), 9146 "receive-migration", 9147 &format! {"unix:{migration_socket}"}, 9148 ]) 9149 .stderr(Stdio::piped()) 9150 .stdout(Stdio::piped()) 9151 .spawn() 9152 .unwrap(); 9153 // Give it '1s' to make sure the 'migration_socket' file is properly created 9154 thread::sleep(std::time::Duration::new(1, 0)); 9155 // Start to send migration from the source VM 9156 9157 let mut args = [ 9158 format!("--api-socket={}", &src_api_socket), 9159 "send-migration".to_string(), 9160 format! {"unix:{migration_socket}"}, 9161 ] 9162 .to_vec(); 9163 9164 if local { 9165 args.insert(2, "--local".to_string()); 9166 } 9167 9168 let mut send_migration = Command::new(clh_command("ch-remote")) 9169 .args(&args) 9170 .stderr(Stdio::piped()) 9171 .stdout(Stdio::piped()) 9172 .spawn() 9173 .unwrap(); 9174 9175 // The 'send-migration' command should be executed successfully within the given timeout 9176 let send_success = if let Some(status) = send_migration 9177 .wait_timeout(std::time::Duration::from_secs(30)) 9178 .unwrap() 9179 { 9180 status.success() 9181 } else { 9182 false 9183 }; 9184 9185 if !send_success { 9186 let _ = send_migration.kill(); 9187 let output = send_migration.wait_with_output().unwrap(); 9188 eprintln!( 9189 "\n\n==== Start 'send_migration' output ==== \ 9190 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 9191 \n\n==== End 'send_migration' output ====\n\n", 9192 String::from_utf8_lossy(&output.stdout), 9193 String::from_utf8_lossy(&output.stderr) 9194 ); 9195 } 9196 9197 // The 'receive-migration' command should be executed successfully within the given timeout 9198 let receive_success = if let Some(status) = receive_migration 9199 .wait_timeout(std::time::Duration::from_secs(30)) 9200 .unwrap() 9201 { 9202 status.success() 9203 } else { 9204 false 9205 }; 9206 9207 if !receive_success { 9208 let _ = receive_migration.kill(); 9209 let output = receive_migration.wait_with_output().unwrap(); 9210 eprintln!( 9211 "\n\n==== Start 'receive_migration' output ==== \ 9212 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 9213 \n\n==== End 'receive_migration' output ====\n\n", 9214 String::from_utf8_lossy(&output.stdout), 9215 String::from_utf8_lossy(&output.stderr) 9216 ); 9217 } 9218 9219 send_success && receive_success 9220 } 9221 9222 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 9223 let mut src_vm = src_vm; 9224 let mut dest_vm = dest_vm; 9225 9226 let _ = src_vm.kill(); 9227 let src_output = src_vm.wait_with_output().unwrap(); 9228 eprintln!( 9229 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 9230 String::from_utf8_lossy(&src_output.stdout) 9231 ); 9232 eprintln!( 9233 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 9234 String::from_utf8_lossy(&src_output.stderr) 9235 ); 9236 let _ = dest_vm.kill(); 9237 let dest_output = dest_vm.wait_with_output().unwrap(); 9238 eprintln!( 9239 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 9240 String::from_utf8_lossy(&dest_output.stdout) 9241 ); 9242 eprintln!( 9243 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 9244 String::from_utf8_lossy(&dest_output.stderr) 9245 ); 9246 9247 if let Some(ovs_vm) = ovs_vm { 9248 let mut ovs_vm = ovs_vm; 9249 let _ = ovs_vm.kill(); 9250 let ovs_output = ovs_vm.wait_with_output().unwrap(); 9251 eprintln!( 9252 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 9253 String::from_utf8_lossy(&ovs_output.stdout) 9254 ); 9255 eprintln!( 9256 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 9257 String::from_utf8_lossy(&ovs_output.stderr) 9258 ); 9259 9260 cleanup_ovs_dpdk(); 9261 } 9262 9263 panic!("Test failed: {message}") 9264 } 9265 9266 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 9267 // same host. It ensures the following behaviors: 9268 // 1. The source VM is up and functional (including various virtio-devices are working properly); 9269 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 9270 // 3. The source VM terminated gracefully after live migration; 9271 // 4. The destination VM is functional (including various virtio-devices are working properly) after 9272 // live migration; 9273 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 9274 fn _test_live_migration(upgrade_test: bool, local: bool) { 9275 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9276 let guest = Guest::new(Box::new(focal)); 9277 let kernel_path = direct_kernel_boot_path(); 9278 let console_text = String::from("On a branch floating down river a cricket, singing."); 9279 let net_id = "net123"; 9280 let net_params = format!( 9281 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9282 net_id, guest.network.guest_mac, guest.network.host_ip 9283 ); 9284 9285 let memory_param: &[&str] = if local { 9286 &["--memory", "size=4G,shared=on"] 9287 } else { 9288 &["--memory", "size=4G"] 9289 }; 9290 9291 let boot_vcpus = 2; 9292 let max_vcpus = 4; 9293 9294 let pmem_temp_file = TempFile::new().unwrap(); 9295 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9296 std::process::Command::new("mkfs.ext4") 9297 .arg(pmem_temp_file.as_path()) 9298 .output() 9299 .expect("Expect creating disk image to succeed"); 9300 let pmem_path = String::from("/dev/pmem0"); 9301 9302 // Start the source VM 9303 let src_vm_path = if !upgrade_test { 9304 clh_command("cloud-hypervisor") 9305 } else { 9306 cloud_hypervisor_release_path() 9307 }; 9308 let src_api_socket = temp_api_path(&guest.tmp_dir); 9309 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9310 src_vm_cmd 9311 .args([ 9312 "--cpus", 9313 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9314 ]) 9315 .args(memory_param) 9316 .args(["--kernel", kernel_path.to_str().unwrap()]) 9317 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9318 .default_disks() 9319 .args(["--net", net_params.as_str()]) 9320 .args(["--api-socket", &src_api_socket]) 9321 .args([ 9322 "--pmem", 9323 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9324 ]); 9325 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9326 9327 // Start the destination VM 9328 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9329 dest_api_socket.push_str(".dest"); 9330 let mut dest_child = GuestCommand::new(&guest) 9331 .args(["--api-socket", &dest_api_socket]) 9332 .capture_output() 9333 .spawn() 9334 .unwrap(); 9335 9336 let r = std::panic::catch_unwind(|| { 9337 guest.wait_vm_boot(None).unwrap(); 9338 9339 // Make sure the source VM is functional 9340 // Check the number of vCPUs 9341 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9342 9343 // Check the guest RAM 9344 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9345 9346 // Check the guest virtio-devices, e.g. block, rng, console, and net 9347 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9348 9349 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9350 // to make sure that removing and adding back the virtio-net device does 9351 // not break the live-migration support for virtio-pci. 9352 #[cfg(target_arch = "x86_64")] 9353 { 9354 assert!(remote_command( 9355 &src_api_socket, 9356 "remove-device", 9357 Some(net_id), 9358 )); 9359 thread::sleep(std::time::Duration::new(10, 0)); 9360 9361 // Plug the virtio-net device again 9362 assert!(remote_command( 9363 &src_api_socket, 9364 "add-net", 9365 Some(net_params.as_str()), 9366 )); 9367 thread::sleep(std::time::Duration::new(10, 0)); 9368 } 9369 9370 // Start the live-migration 9371 let migration_socket = String::from( 9372 guest 9373 .tmp_dir 9374 .as_path() 9375 .join("live-migration.sock") 9376 .to_str() 9377 .unwrap(), 9378 ); 9379 9380 assert!( 9381 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9382 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9383 ); 9384 }); 9385 9386 // Check and report any errors occurred during the live-migration 9387 if r.is_err() { 9388 print_and_panic( 9389 src_child, 9390 dest_child, 9391 None, 9392 "Error occurred during live-migration", 9393 ); 9394 } 9395 9396 // Check the source vm has been terminated successful (give it '3s' to settle) 9397 thread::sleep(std::time::Duration::new(3, 0)); 9398 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9399 print_and_panic( 9400 src_child, 9401 dest_child, 9402 None, 9403 "source VM was not terminated successfully.", 9404 ); 9405 }; 9406 9407 // Post live-migration check to make sure the destination VM is functional 9408 let r = std::panic::catch_unwind(|| { 9409 // Perform same checks to validate VM has been properly migrated 9410 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9411 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9412 9413 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9414 }); 9415 9416 // Clean-up the destination VM and make sure it terminated correctly 9417 let _ = dest_child.kill(); 9418 let dest_output = dest_child.wait_with_output().unwrap(); 9419 handle_child_output(r, &dest_output); 9420 9421 // Check the destination VM has the expected 'console_text' from its output 9422 let r = std::panic::catch_unwind(|| { 9423 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9424 }); 9425 handle_child_output(r, &dest_output); 9426 } 9427 9428 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 9429 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9430 let guest = Guest::new(Box::new(focal)); 9431 let kernel_path = direct_kernel_boot_path(); 9432 let console_text = String::from("On a branch floating down river a cricket, singing."); 9433 let net_id = "net123"; 9434 let net_params = format!( 9435 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9436 net_id, guest.network.guest_mac, guest.network.host_ip 9437 ); 9438 9439 let memory_param: &[&str] = if local { 9440 &[ 9441 "--memory", 9442 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 9443 "--balloon", 9444 "size=0", 9445 ] 9446 } else { 9447 &[ 9448 "--memory", 9449 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 9450 "--balloon", 9451 "size=0", 9452 ] 9453 }; 9454 9455 let boot_vcpus = 2; 9456 let max_vcpus = 4; 9457 9458 let pmem_temp_file = TempFile::new().unwrap(); 9459 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9460 std::process::Command::new("mkfs.ext4") 9461 .arg(pmem_temp_file.as_path()) 9462 .output() 9463 .expect("Expect creating disk image to succeed"); 9464 let pmem_path = String::from("/dev/pmem0"); 9465 9466 // Start the source VM 9467 let src_vm_path = if !upgrade_test { 9468 clh_command("cloud-hypervisor") 9469 } else { 9470 cloud_hypervisor_release_path() 9471 }; 9472 let src_api_socket = temp_api_path(&guest.tmp_dir); 9473 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9474 src_vm_cmd 9475 .args([ 9476 "--cpus", 9477 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9478 ]) 9479 .args(memory_param) 9480 .args(["--kernel", kernel_path.to_str().unwrap()]) 9481 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9482 .default_disks() 9483 .args(["--net", net_params.as_str()]) 9484 .args(["--api-socket", &src_api_socket]) 9485 .args([ 9486 "--pmem", 9487 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9488 ]); 9489 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9490 9491 // Start the destination VM 9492 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9493 dest_api_socket.push_str(".dest"); 9494 let mut dest_child = GuestCommand::new(&guest) 9495 .args(["--api-socket", &dest_api_socket]) 9496 .capture_output() 9497 .spawn() 9498 .unwrap(); 9499 9500 let r = std::panic::catch_unwind(|| { 9501 guest.wait_vm_boot(None).unwrap(); 9502 9503 // Make sure the source VM is functional 9504 // Check the number of vCPUs 9505 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9506 9507 // Check the guest RAM 9508 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9509 // Increase the guest RAM 9510 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 9511 thread::sleep(std::time::Duration::new(5, 0)); 9512 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9513 // Use balloon to remove RAM from the VM 9514 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 9515 thread::sleep(std::time::Duration::new(5, 0)); 9516 let total_memory = guest.get_total_memory().unwrap_or_default(); 9517 assert!(total_memory > 4_800_000); 9518 assert!(total_memory < 5_760_000); 9519 9520 // Check the guest virtio-devices, e.g. block, rng, console, and net 9521 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9522 9523 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9524 // to make sure that removing and adding back the virtio-net device does 9525 // not break the live-migration support for virtio-pci. 9526 #[cfg(target_arch = "x86_64")] 9527 { 9528 assert!(remote_command( 9529 &src_api_socket, 9530 "remove-device", 9531 Some(net_id), 9532 )); 9533 thread::sleep(std::time::Duration::new(10, 0)); 9534 9535 // Plug the virtio-net device again 9536 assert!(remote_command( 9537 &src_api_socket, 9538 "add-net", 9539 Some(net_params.as_str()), 9540 )); 9541 thread::sleep(std::time::Duration::new(10, 0)); 9542 } 9543 9544 // Start the live-migration 9545 let migration_socket = String::from( 9546 guest 9547 .tmp_dir 9548 .as_path() 9549 .join("live-migration.sock") 9550 .to_str() 9551 .unwrap(), 9552 ); 9553 9554 assert!( 9555 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9556 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9557 ); 9558 }); 9559 9560 // Check and report any errors occurred during the live-migration 9561 if r.is_err() { 9562 print_and_panic( 9563 src_child, 9564 dest_child, 9565 None, 9566 "Error occurred during live-migration", 9567 ); 9568 } 9569 9570 // Check the source vm has been terminated successful (give it '3s' to settle) 9571 thread::sleep(std::time::Duration::new(3, 0)); 9572 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9573 print_and_panic( 9574 src_child, 9575 dest_child, 9576 None, 9577 "source VM was not terminated successfully.", 9578 ); 9579 }; 9580 9581 // Post live-migration check to make sure the destination VM is functional 9582 let r = std::panic::catch_unwind(|| { 9583 // Perform same checks to validate VM has been properly migrated 9584 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9585 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9586 9587 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9588 9589 // Perform checks on guest RAM using balloon 9590 let total_memory = guest.get_total_memory().unwrap_or_default(); 9591 assert!(total_memory > 4_800_000); 9592 assert!(total_memory < 5_760_000); 9593 // Deflate balloon to restore entire RAM to the VM 9594 resize_command(&dest_api_socket, None, None, Some(0), None); 9595 thread::sleep(std::time::Duration::new(5, 0)); 9596 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9597 // Decrease guest RAM with virtio-mem 9598 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 9599 thread::sleep(std::time::Duration::new(5, 0)); 9600 let total_memory = guest.get_total_memory().unwrap_or_default(); 9601 assert!(total_memory > 4_800_000); 9602 assert!(total_memory < 5_760_000); 9603 }); 9604 9605 // Clean-up the destination VM and make sure it terminated correctly 9606 let _ = dest_child.kill(); 9607 let dest_output = dest_child.wait_with_output().unwrap(); 9608 handle_child_output(r, &dest_output); 9609 9610 // Check the destination VM has the expected 'console_text' from its output 9611 let r = std::panic::catch_unwind(|| { 9612 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9613 }); 9614 handle_child_output(r, &dest_output); 9615 } 9616 9617 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 9618 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9619 let guest = Guest::new(Box::new(focal)); 9620 let kernel_path = direct_kernel_boot_path(); 9621 let console_text = String::from("On a branch floating down river a cricket, singing."); 9622 let net_id = "net123"; 9623 let net_params = format!( 9624 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9625 net_id, guest.network.guest_mac, guest.network.host_ip 9626 ); 9627 9628 let memory_param: &[&str] = if local { 9629 &[ 9630 "--memory", 9631 "size=0,hotplug_method=virtio-mem,shared=on", 9632 "--memory-zone", 9633 "id=mem0,size=1G,hotplug_size=4G,shared=on", 9634 "id=mem1,size=1G,hotplug_size=4G,shared=on", 9635 "id=mem2,size=2G,hotplug_size=4G,shared=on", 9636 "--numa", 9637 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9638 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9639 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9640 ] 9641 } else { 9642 &[ 9643 "--memory", 9644 "size=0,hotplug_method=virtio-mem", 9645 "--memory-zone", 9646 "id=mem0,size=1G,hotplug_size=4G", 9647 "id=mem1,size=1G,hotplug_size=4G", 9648 "id=mem2,size=2G,hotplug_size=4G", 9649 "--numa", 9650 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9651 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9652 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9653 ] 9654 }; 9655 9656 let boot_vcpus = 6; 9657 let max_vcpus = 12; 9658 9659 let pmem_temp_file = TempFile::new().unwrap(); 9660 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9661 std::process::Command::new("mkfs.ext4") 9662 .arg(pmem_temp_file.as_path()) 9663 .output() 9664 .expect("Expect creating disk image to succeed"); 9665 let pmem_path = String::from("/dev/pmem0"); 9666 9667 // Start the source VM 9668 let src_vm_path = if !upgrade_test { 9669 clh_command("cloud-hypervisor") 9670 } else { 9671 cloud_hypervisor_release_path() 9672 }; 9673 let src_api_socket = temp_api_path(&guest.tmp_dir); 9674 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9675 src_vm_cmd 9676 .args([ 9677 "--cpus", 9678 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9679 ]) 9680 .args(memory_param) 9681 .args(["--kernel", kernel_path.to_str().unwrap()]) 9682 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9683 .default_disks() 9684 .args(["--net", net_params.as_str()]) 9685 .args(["--api-socket", &src_api_socket]) 9686 .args([ 9687 "--pmem", 9688 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9689 ]); 9690 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9691 9692 // Start the destination VM 9693 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9694 dest_api_socket.push_str(".dest"); 9695 let mut dest_child = GuestCommand::new(&guest) 9696 .args(["--api-socket", &dest_api_socket]) 9697 .capture_output() 9698 .spawn() 9699 .unwrap(); 9700 9701 let r = std::panic::catch_unwind(|| { 9702 guest.wait_vm_boot(None).unwrap(); 9703 9704 // Make sure the source VM is functional 9705 // Check the number of vCPUs 9706 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9707 9708 // Check the guest RAM 9709 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 9710 9711 // Check the guest virtio-devices, e.g. block, rng, console, and net 9712 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9713 9714 // Check the NUMA parameters are applied correctly and resize 9715 // each zone to test the case where we migrate a VM with the 9716 // virtio-mem regions being used. 9717 { 9718 guest.check_numa_common( 9719 Some(&[960_000, 960_000, 1_920_000]), 9720 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9721 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9722 ); 9723 9724 // AArch64 currently does not support hotplug, and therefore we only 9725 // test hotplug-related function on x86_64 here. 9726 #[cfg(target_arch = "x86_64")] 9727 { 9728 guest.enable_memory_hotplug(); 9729 9730 // Resize every memory zone and check each associated NUMA node 9731 // has been assigned the right amount of memory. 9732 resize_zone_command(&src_api_socket, "mem0", "2G"); 9733 resize_zone_command(&src_api_socket, "mem1", "2G"); 9734 resize_zone_command(&src_api_socket, "mem2", "3G"); 9735 thread::sleep(std::time::Duration::new(5, 0)); 9736 9737 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 9738 } 9739 } 9740 9741 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9742 // to make sure that removing and adding back the virtio-net device does 9743 // not break the live-migration support for virtio-pci. 9744 #[cfg(target_arch = "x86_64")] 9745 { 9746 assert!(remote_command( 9747 &src_api_socket, 9748 "remove-device", 9749 Some(net_id), 9750 )); 9751 thread::sleep(std::time::Duration::new(10, 0)); 9752 9753 // Plug the virtio-net device again 9754 assert!(remote_command( 9755 &src_api_socket, 9756 "add-net", 9757 Some(net_params.as_str()), 9758 )); 9759 thread::sleep(std::time::Duration::new(10, 0)); 9760 } 9761 9762 // Start the live-migration 9763 let migration_socket = String::from( 9764 guest 9765 .tmp_dir 9766 .as_path() 9767 .join("live-migration.sock") 9768 .to_str() 9769 .unwrap(), 9770 ); 9771 9772 assert!( 9773 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9774 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9775 ); 9776 }); 9777 9778 // Check and report any errors occurred during the live-migration 9779 if r.is_err() { 9780 print_and_panic( 9781 src_child, 9782 dest_child, 9783 None, 9784 "Error occurred during live-migration", 9785 ); 9786 } 9787 9788 // Check the source vm has been terminated successful (give it '3s' to settle) 9789 thread::sleep(std::time::Duration::new(3, 0)); 9790 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9791 print_and_panic( 9792 src_child, 9793 dest_child, 9794 None, 9795 "source VM was not terminated successfully.", 9796 ); 9797 }; 9798 9799 // Post live-migration check to make sure the destination VM is functional 9800 let r = std::panic::catch_unwind(|| { 9801 // Perform same checks to validate VM has been properly migrated 9802 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9803 #[cfg(target_arch = "x86_64")] 9804 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 9805 #[cfg(target_arch = "aarch64")] 9806 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9807 9808 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9809 9810 // Perform NUMA related checks 9811 { 9812 #[cfg(target_arch = "aarch64")] 9813 { 9814 guest.check_numa_common( 9815 Some(&[960_000, 960_000, 1_920_000]), 9816 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9817 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9818 ); 9819 } 9820 9821 // AArch64 currently does not support hotplug, and therefore we only 9822 // test hotplug-related function on x86_64 here. 9823 #[cfg(target_arch = "x86_64")] 9824 { 9825 guest.check_numa_common( 9826 Some(&[1_920_000, 1_920_000, 2_880_000]), 9827 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9828 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9829 ); 9830 9831 guest.enable_memory_hotplug(); 9832 9833 // Resize every memory zone and check each associated NUMA node 9834 // has been assigned the right amount of memory. 9835 resize_zone_command(&dest_api_socket, "mem0", "4G"); 9836 resize_zone_command(&dest_api_socket, "mem1", "4G"); 9837 resize_zone_command(&dest_api_socket, "mem2", "4G"); 9838 // Resize to the maximum amount of CPUs and check each NUMA 9839 // node has been assigned the right CPUs set. 9840 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 9841 thread::sleep(std::time::Duration::new(5, 0)); 9842 9843 guest.check_numa_common( 9844 Some(&[3_840_000, 3_840_000, 3_840_000]), 9845 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 9846 None, 9847 ); 9848 } 9849 } 9850 }); 9851 9852 // Clean-up the destination VM and make sure it terminated correctly 9853 let _ = dest_child.kill(); 9854 let dest_output = dest_child.wait_with_output().unwrap(); 9855 handle_child_output(r, &dest_output); 9856 9857 // Check the destination VM has the expected 'console_text' from its output 9858 let r = std::panic::catch_unwind(|| { 9859 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9860 }); 9861 handle_child_output(r, &dest_output); 9862 } 9863 9864 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 9865 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9866 let guest = Guest::new(Box::new(focal)); 9867 let kernel_path = direct_kernel_boot_path(); 9868 let console_text = String::from("On a branch floating down river a cricket, singing."); 9869 let net_id = "net123"; 9870 let net_params = format!( 9871 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9872 net_id, guest.network.guest_mac, guest.network.host_ip 9873 ); 9874 9875 let memory_param: &[&str] = if local { 9876 &["--memory", "size=4G,shared=on"] 9877 } else { 9878 &["--memory", "size=4G"] 9879 }; 9880 9881 let boot_vcpus = 2; 9882 let max_vcpus = 4; 9883 9884 let pmem_temp_file = TempFile::new().unwrap(); 9885 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9886 std::process::Command::new("mkfs.ext4") 9887 .arg(pmem_temp_file.as_path()) 9888 .output() 9889 .expect("Expect creating disk image to succeed"); 9890 let pmem_path = String::from("/dev/pmem0"); 9891 9892 // Start the source VM 9893 let src_vm_path = if !upgrade_test { 9894 clh_command("cloud-hypervisor") 9895 } else { 9896 cloud_hypervisor_release_path() 9897 }; 9898 let src_api_socket = temp_api_path(&guest.tmp_dir); 9899 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9900 src_vm_cmd 9901 .args([ 9902 "--cpus", 9903 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9904 ]) 9905 .args(memory_param) 9906 .args(["--kernel", kernel_path.to_str().unwrap()]) 9907 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9908 .default_disks() 9909 .args(["--net", net_params.as_str()]) 9910 .args(["--api-socket", &src_api_socket]) 9911 .args([ 9912 "--pmem", 9913 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9914 ]) 9915 .args(["--watchdog"]); 9916 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9917 9918 // Start the destination VM 9919 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9920 dest_api_socket.push_str(".dest"); 9921 let mut dest_child = GuestCommand::new(&guest) 9922 .args(["--api-socket", &dest_api_socket]) 9923 .capture_output() 9924 .spawn() 9925 .unwrap(); 9926 9927 let r = std::panic::catch_unwind(|| { 9928 guest.wait_vm_boot(None).unwrap(); 9929 9930 // Make sure the source VM is functional 9931 // Check the number of vCPUs 9932 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9933 // Check the guest RAM 9934 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9935 // Check the guest virtio-devices, e.g. block, rng, console, and net 9936 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9937 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9938 // to make sure that removing and adding back the virtio-net device does 9939 // not break the live-migration support for virtio-pci. 9940 #[cfg(target_arch = "x86_64")] 9941 { 9942 assert!(remote_command( 9943 &src_api_socket, 9944 "remove-device", 9945 Some(net_id), 9946 )); 9947 thread::sleep(std::time::Duration::new(10, 0)); 9948 9949 // Plug the virtio-net device again 9950 assert!(remote_command( 9951 &src_api_socket, 9952 "add-net", 9953 Some(net_params.as_str()), 9954 )); 9955 thread::sleep(std::time::Duration::new(10, 0)); 9956 } 9957 9958 // Enable watchdog and ensure its functional 9959 let expected_reboot_count = 1; 9960 // Enable the watchdog with a 15s timeout 9961 enable_guest_watchdog(&guest, 15); 9962 9963 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9964 assert_eq!( 9965 guest 9966 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 9967 .unwrap() 9968 .trim() 9969 .parse::<u32>() 9970 .unwrap_or_default(), 9971 1 9972 ); 9973 // Allow some normal time to elapse to check we don't get spurious reboots 9974 thread::sleep(std::time::Duration::new(40, 0)); 9975 // Check no reboot 9976 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9977 9978 // Start the live-migration 9979 let migration_socket = String::from( 9980 guest 9981 .tmp_dir 9982 .as_path() 9983 .join("live-migration.sock") 9984 .to_str() 9985 .unwrap(), 9986 ); 9987 9988 assert!( 9989 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9990 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9991 ); 9992 }); 9993 9994 // Check and report any errors occurred during the live-migration 9995 if r.is_err() { 9996 print_and_panic( 9997 src_child, 9998 dest_child, 9999 None, 10000 "Error occurred during live-migration", 10001 ); 10002 } 10003 10004 // Check the source vm has been terminated successful (give it '3s' to settle) 10005 thread::sleep(std::time::Duration::new(3, 0)); 10006 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 10007 print_and_panic( 10008 src_child, 10009 dest_child, 10010 None, 10011 "source VM was not terminated successfully.", 10012 ); 10013 }; 10014 10015 // Post live-migration check to make sure the destination VM is functional 10016 let r = std::panic::catch_unwind(|| { 10017 // Perform same checks to validate VM has been properly migrated 10018 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10019 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10020 10021 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 10022 10023 // Perform checks on watchdog 10024 let mut expected_reboot_count = 1; 10025 10026 // Allow some normal time to elapse to check we don't get spurious reboots 10027 thread::sleep(std::time::Duration::new(40, 0)); 10028 // Check no reboot 10029 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10030 10031 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 10032 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 10033 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 10034 guest.wait_vm_boot(Some(50)).unwrap(); 10035 // Check a reboot is triggered by the watchdog 10036 expected_reboot_count += 1; 10037 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10038 10039 #[cfg(target_arch = "x86_64")] 10040 { 10041 // Now pause the VM and remain offline for 30s 10042 assert!(remote_command(&dest_api_socket, "pause", None)); 10043 thread::sleep(std::time::Duration::new(30, 0)); 10044 assert!(remote_command(&dest_api_socket, "resume", None)); 10045 10046 // Check no reboot 10047 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10048 } 10049 }); 10050 10051 // Clean-up the destination VM and make sure it terminated correctly 10052 let _ = dest_child.kill(); 10053 let dest_output = dest_child.wait_with_output().unwrap(); 10054 handle_child_output(r, &dest_output); 10055 10056 // Check the destination VM has the expected 'console_text' from its output 10057 let r = std::panic::catch_unwind(|| { 10058 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 10059 }); 10060 handle_child_output(r, &dest_output); 10061 } 10062 10063 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 10064 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10065 let ovs_guest = Guest::new(Box::new(ovs_focal)); 10066 10067 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10068 let migration_guest = Guest::new(Box::new(migration_focal)); 10069 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 10070 10071 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 10072 let (mut ovs_child, mut src_child) = 10073 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 10074 10075 // Start the destination VM 10076 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 10077 dest_api_socket.push_str(".dest"); 10078 let mut dest_child = GuestCommand::new(&migration_guest) 10079 .args(["--api-socket", &dest_api_socket]) 10080 .capture_output() 10081 .spawn() 10082 .unwrap(); 10083 10084 let r = std::panic::catch_unwind(|| { 10085 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 10086 thread::sleep(std::time::Duration::new(1, 0)); 10087 10088 // Start the live-migration 10089 let migration_socket = String::from( 10090 migration_guest 10091 .tmp_dir 10092 .as_path() 10093 .join("live-migration.sock") 10094 .to_str() 10095 .unwrap(), 10096 ); 10097 10098 assert!( 10099 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 10100 "Unsuccessful command: 'send-migration' or 'receive-migration'." 10101 ); 10102 }); 10103 10104 // Check and report any errors occurred during the live-migration 10105 if r.is_err() { 10106 print_and_panic( 10107 src_child, 10108 dest_child, 10109 Some(ovs_child), 10110 "Error occurred during live-migration", 10111 ); 10112 } 10113 10114 // Check the source vm has been terminated successful (give it '3s' to settle) 10115 thread::sleep(std::time::Duration::new(3, 0)); 10116 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 10117 print_and_panic( 10118 src_child, 10119 dest_child, 10120 Some(ovs_child), 10121 "source VM was not terminated successfully.", 10122 ); 10123 }; 10124 10125 // Post live-migration check to make sure the destination VM is functional 10126 let r = std::panic::catch_unwind(|| { 10127 // Perform same checks to validate VM has been properly migrated 10128 // Spawn a new netcat listener in the OVS VM 10129 let guest_ip = ovs_guest.network.guest_ip.clone(); 10130 thread::spawn(move || { 10131 ssh_command_ip( 10132 "nc -l 12345", 10133 &guest_ip, 10134 DEFAULT_SSH_RETRIES, 10135 DEFAULT_SSH_TIMEOUT, 10136 ) 10137 .unwrap(); 10138 }); 10139 10140 // Wait for the server to be listening 10141 thread::sleep(std::time::Duration::new(5, 0)); 10142 10143 // And check the connection is still functional after live-migration 10144 migration_guest 10145 .ssh_command("nc -vz 172.100.0.1 12345") 10146 .unwrap(); 10147 }); 10148 10149 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 10150 let _ = dest_child.kill(); 10151 let _ = ovs_child.kill(); 10152 let dest_output = dest_child.wait_with_output().unwrap(); 10153 let ovs_output = ovs_child.wait_with_output().unwrap(); 10154 10155 cleanup_ovs_dpdk(); 10156 10157 handle_child_output(r, &dest_output); 10158 handle_child_output(Ok(()), &ovs_output); 10159 } 10160 10161 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 10162 // same host with Landlock enabled on both VMs. The test validates the following: 10163 // 1. The source VM is up and functional 10164 // 2. Ensure Landlock is enabled on source VM by hotplugging a disk. As the path for this 10165 // disk is not known to the source VM this step will fail. 10166 // 3. The 'send-migration' and 'receive-migration' command finished successfully; 10167 // 4. The source VM terminated gracefully after live migration; 10168 // 5. The destination VM is functional after live migration; 10169 // 6. Ensure Landlock is enabled on destination VM by hotplugging a disk. As the path for 10170 // this disk is not known to the destination VM this step will fail. 10171 fn _test_live_migration_with_landlock() { 10172 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10173 let guest = Guest::new(Box::new(focal)); 10174 let kernel_path = direct_kernel_boot_path(); 10175 let net_id = "net123"; 10176 let net_params = format!( 10177 "id={},tap=,mac={},ip={},mask=255.255.255.0", 10178 net_id, guest.network.guest_mac, guest.network.host_ip 10179 ); 10180 10181 let boot_vcpus = 2; 10182 let max_vcpus = 4; 10183 10184 let mut blk_file_path = dirs::home_dir().unwrap(); 10185 blk_file_path.push("workloads"); 10186 blk_file_path.push("blk.img"); 10187 10188 let src_api_socket = temp_api_path(&guest.tmp_dir); 10189 let mut src_child = GuestCommand::new(&guest) 10190 .args([ 10191 "--cpus", 10192 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 10193 ]) 10194 .args(["--memory", "size=4G,shared=on"]) 10195 .args(["--kernel", kernel_path.to_str().unwrap()]) 10196 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10197 .default_disks() 10198 .args(["--api-socket", &src_api_socket]) 10199 .args(["--landlock"]) 10200 .args(["--net", net_params.as_str()]) 10201 .args([ 10202 "--landlock-rules", 10203 format!("path={:?},access=rw", guest.tmp_dir.as_path()).as_str(), 10204 ]) 10205 .capture_output() 10206 .spawn() 10207 .unwrap(); 10208 10209 // Start the destination VM 10210 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 10211 dest_api_socket.push_str(".dest"); 10212 let mut dest_child = GuestCommand::new(&guest) 10213 .args(["--api-socket", &dest_api_socket]) 10214 .capture_output() 10215 .spawn() 10216 .unwrap(); 10217 10218 let r = std::panic::catch_unwind(|| { 10219 guest.wait_vm_boot(None).unwrap(); 10220 10221 // Make sure the source VM is functaionl 10222 // Check the number of vCPUs 10223 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10224 10225 // Check the guest RAM 10226 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10227 10228 // Check Landlock is enabled by hot-plugging a disk. 10229 assert!(!remote_command( 10230 &src_api_socket, 10231 "add-disk", 10232 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 10233 )); 10234 10235 // Start the live-migration 10236 let migration_socket = String::from( 10237 guest 10238 .tmp_dir 10239 .as_path() 10240 .join("live-migration.sock") 10241 .to_str() 10242 .unwrap(), 10243 ); 10244 10245 assert!( 10246 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, true), 10247 "Unsuccessful command: 'send-migration' or 'receive-migration'." 10248 ); 10249 }); 10250 10251 // Check and report any errors occurred during the live-migration 10252 if r.is_err() { 10253 print_and_panic( 10254 src_child, 10255 dest_child, 10256 None, 10257 "Error occurred during live-migration", 10258 ); 10259 } 10260 10261 // Check the source vm has been terminated successful (give it '3s' to settle) 10262 thread::sleep(std::time::Duration::new(3, 0)); 10263 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 10264 print_and_panic( 10265 src_child, 10266 dest_child, 10267 None, 10268 "source VM was not terminated successfully.", 10269 ); 10270 }; 10271 10272 // Post live-migration check to make sure the destination VM is funcational 10273 let r = std::panic::catch_unwind(|| { 10274 // Perform same checks to validate VM has been properly migrated 10275 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10276 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10277 }); 10278 10279 // Check Landlock is enabled on destination VM by hot-plugging a disk. 10280 assert!(!remote_command( 10281 &dest_api_socket, 10282 "add-disk", 10283 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 10284 )); 10285 10286 // Clean-up the destination VM and make sure it terminated correctly 10287 let _ = dest_child.kill(); 10288 let dest_output = dest_child.wait_with_output().unwrap(); 10289 handle_child_output(r, &dest_output); 10290 } 10291 10292 mod live_migration_parallel { 10293 use super::*; 10294 #[test] 10295 fn test_live_migration_basic() { 10296 _test_live_migration(false, false) 10297 } 10298 10299 #[test] 10300 fn test_live_migration_local() { 10301 _test_live_migration(false, true) 10302 } 10303 10304 #[test] 10305 fn test_live_migration_watchdog() { 10306 _test_live_migration_watchdog(false, false) 10307 } 10308 10309 #[test] 10310 fn test_live_migration_watchdog_local() { 10311 _test_live_migration_watchdog(false, true) 10312 } 10313 10314 #[test] 10315 fn test_live_upgrade_basic() { 10316 _test_live_migration(true, false) 10317 } 10318 10319 #[test] 10320 fn test_live_upgrade_local() { 10321 _test_live_migration(true, true) 10322 } 10323 10324 #[test] 10325 fn test_live_upgrade_watchdog() { 10326 _test_live_migration_watchdog(true, false) 10327 } 10328 10329 #[test] 10330 fn test_live_upgrade_watchdog_local() { 10331 _test_live_migration_watchdog(true, true) 10332 } 10333 #[test] 10334 #[cfg(target_arch = "x86_64")] 10335 fn test_live_migration_with_landlock() { 10336 _test_live_migration_with_landlock() 10337 } 10338 } 10339 10340 mod live_migration_sequential { 10341 use super::*; 10342 10343 // NUMA & balloon live migration tests are large so run sequentially 10344 10345 #[test] 10346 fn test_live_migration_balloon() { 10347 _test_live_migration_balloon(false, false) 10348 } 10349 10350 #[test] 10351 fn test_live_migration_balloon_local() { 10352 _test_live_migration_balloon(false, true) 10353 } 10354 10355 #[test] 10356 fn test_live_upgrade_balloon() { 10357 _test_live_migration_balloon(true, false) 10358 } 10359 10360 #[test] 10361 fn test_live_upgrade_balloon_local() { 10362 _test_live_migration_balloon(true, true) 10363 } 10364 10365 #[test] 10366 #[cfg(not(feature = "mshv"))] 10367 fn test_live_migration_numa() { 10368 _test_live_migration_numa(false, false) 10369 } 10370 10371 #[test] 10372 #[cfg(not(feature = "mshv"))] 10373 fn test_live_migration_numa_local() { 10374 _test_live_migration_numa(false, true) 10375 } 10376 10377 #[test] 10378 #[cfg(not(feature = "mshv"))] 10379 fn test_live_upgrade_numa() { 10380 _test_live_migration_numa(true, false) 10381 } 10382 10383 #[test] 10384 #[cfg(not(feature = "mshv"))] 10385 fn test_live_upgrade_numa_local() { 10386 _test_live_migration_numa(true, true) 10387 } 10388 10389 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 10390 #[test] 10391 #[ignore = "See #5532"] 10392 #[cfg(target_arch = "x86_64")] 10393 #[cfg(not(feature = "mshv"))] 10394 fn test_live_migration_ovs_dpdk() { 10395 _test_live_migration_ovs_dpdk(false, false); 10396 } 10397 10398 #[test] 10399 #[cfg(target_arch = "x86_64")] 10400 #[cfg(not(feature = "mshv"))] 10401 fn test_live_migration_ovs_dpdk_local() { 10402 _test_live_migration_ovs_dpdk(false, true); 10403 } 10404 10405 #[test] 10406 #[ignore = "See #5532"] 10407 #[cfg(target_arch = "x86_64")] 10408 #[cfg(not(feature = "mshv"))] 10409 fn test_live_upgrade_ovs_dpdk() { 10410 _test_live_migration_ovs_dpdk(true, false); 10411 } 10412 10413 #[test] 10414 #[ignore = "See #5532"] 10415 #[cfg(target_arch = "x86_64")] 10416 #[cfg(not(feature = "mshv"))] 10417 fn test_live_upgrade_ovs_dpdk_local() { 10418 _test_live_migration_ovs_dpdk(true, true); 10419 } 10420 } 10421 } 10422 10423 #[cfg(target_arch = "aarch64")] 10424 mod aarch64_acpi { 10425 use crate::*; 10426 10427 #[test] 10428 fn test_simple_launch_acpi() { 10429 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10430 10431 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 10432 let guest = Guest::new(disk_config); 10433 10434 let mut child = GuestCommand::new(&guest) 10435 .args(["--cpus", "boot=1"]) 10436 .args(["--memory", "size=512M"]) 10437 .args(["--kernel", edk2_path().to_str().unwrap()]) 10438 .default_disks() 10439 .default_net() 10440 .args(["--serial", "tty", "--console", "off"]) 10441 .capture_output() 10442 .spawn() 10443 .unwrap(); 10444 10445 let r = std::panic::catch_unwind(|| { 10446 guest.wait_vm_boot(Some(120)).unwrap(); 10447 10448 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 10449 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 10450 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 10451 }); 10452 10453 let _ = child.kill(); 10454 let output = child.wait_with_output().unwrap(); 10455 10456 handle_child_output(r, &output); 10457 }); 10458 } 10459 10460 #[test] 10461 fn test_guest_numa_nodes_acpi() { 10462 _test_guest_numa_nodes(true); 10463 } 10464 10465 #[test] 10466 fn test_cpu_topology_421_acpi() { 10467 test_cpu_topology(4, 2, 1, true); 10468 } 10469 10470 #[test] 10471 fn test_cpu_topology_142_acpi() { 10472 test_cpu_topology(1, 4, 2, true); 10473 } 10474 10475 #[test] 10476 fn test_cpu_topology_262_acpi() { 10477 test_cpu_topology(2, 6, 2, true); 10478 } 10479 10480 #[test] 10481 fn test_power_button_acpi() { 10482 _test_power_button(true); 10483 } 10484 10485 #[test] 10486 fn test_virtio_iommu() { 10487 _test_virtio_iommu(true) 10488 } 10489 } 10490 10491 mod rate_limiter { 10492 use super::*; 10493 10494 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 10495 // compared to given 'limit' rate. 10496 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 10497 let upper_limit = limit * (1_f64 + difference); 10498 let lower_limit = limit * (1_f64 - difference); 10499 10500 if measured > lower_limit && measured < upper_limit { 10501 return true; 10502 } 10503 10504 eprintln!( 10505 "\n\n==== Start 'check_rate_limit' failed ==== \ 10506 \n\nmeasured={measured}, , lower_limit={lower_limit}, upper_limit={upper_limit} \ 10507 \n\n==== End 'check_rate_limit' failed ====\n\n" 10508 ); 10509 10510 false 10511 } 10512 10513 fn _test_rate_limiter_net(rx: bool) { 10514 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10515 let guest = Guest::new(Box::new(focal)); 10516 10517 let test_timeout = 10; 10518 let num_queues = 2; 10519 let queue_size = 256; 10520 let bw_size = 10485760_u64; // bytes 10521 let bw_refill_time = 100; // ms 10522 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 10523 10524 let net_params = format!( 10525 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 10526 guest.network.guest_mac, 10527 guest.network.host_ip, 10528 num_queues, 10529 queue_size, 10530 bw_size, 10531 bw_refill_time, 10532 ); 10533 10534 let mut child = GuestCommand::new(&guest) 10535 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 10536 .args(["--memory", "size=4G"]) 10537 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10538 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10539 .default_disks() 10540 .args(["--net", net_params.as_str()]) 10541 .capture_output() 10542 .spawn() 10543 .unwrap(); 10544 10545 let r = std::panic::catch_unwind(|| { 10546 guest.wait_vm_boot(None).unwrap(); 10547 let measured_bps = 10548 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, true) 10549 .unwrap(); 10550 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 10551 }); 10552 10553 let _ = child.kill(); 10554 let output = child.wait_with_output().unwrap(); 10555 handle_child_output(r, &output); 10556 } 10557 10558 #[test] 10559 fn test_rate_limiter_net_rx() { 10560 _test_rate_limiter_net(true); 10561 } 10562 10563 #[test] 10564 fn test_rate_limiter_net_tx() { 10565 _test_rate_limiter_net(false); 10566 } 10567 10568 fn _test_rate_limiter_block(bandwidth: bool, num_queues: u32) { 10569 let test_timeout = 10; 10570 let fio_ops = FioOps::RandRW; 10571 10572 let bw_size = if bandwidth { 10573 10485760_u64 // bytes 10574 } else { 10575 100_u64 // I/O 10576 }; 10577 let bw_refill_time = 100; // ms 10578 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10579 10580 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10581 let guest = Guest::new(Box::new(focal)); 10582 let api_socket = temp_api_path(&guest.tmp_dir); 10583 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10584 let blk_rate_limiter_test_img = 10585 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 10586 10587 // Create the test block image 10588 assert!(exec_host_command_output(&format!( 10589 "dd if=/dev/zero of={blk_rate_limiter_test_img} bs=1M count=1024" 10590 )) 10591 .status 10592 .success()); 10593 10594 let test_blk_params = if bandwidth { 10595 format!( 10596 "path={blk_rate_limiter_test_img},num_queues={num_queues},bw_size={bw_size},bw_refill_time={bw_refill_time}" 10597 ) 10598 } else { 10599 format!( 10600 "path={blk_rate_limiter_test_img},num_queues={num_queues},ops_size={bw_size},ops_refill_time={bw_refill_time}" 10601 ) 10602 }; 10603 10604 let mut child = GuestCommand::new(&guest) 10605 .args(["--cpus", &format!("boot={num_queues}")]) 10606 .args(["--memory", "size=4G"]) 10607 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10608 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10609 .args([ 10610 "--disk", 10611 format!( 10612 "path={}", 10613 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10614 ) 10615 .as_str(), 10616 format!( 10617 "path={}", 10618 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10619 ) 10620 .as_str(), 10621 test_blk_params.as_str(), 10622 ]) 10623 .default_net() 10624 .args(["--api-socket", &api_socket]) 10625 .capture_output() 10626 .spawn() 10627 .unwrap(); 10628 10629 let r = std::panic::catch_unwind(|| { 10630 guest.wait_vm_boot(None).unwrap(); 10631 10632 let fio_command = format!( 10633 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 10634 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10635 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10636 ); 10637 let output = guest.ssh_command(&fio_command).unwrap(); 10638 10639 // Parse fio output 10640 let measured_rate = if bandwidth { 10641 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 10642 } else { 10643 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 10644 }; 10645 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10646 }); 10647 10648 let _ = child.kill(); 10649 let output = child.wait_with_output().unwrap(); 10650 handle_child_output(r, &output); 10651 } 10652 10653 fn _test_rate_limiter_group_block(bandwidth: bool, num_queues: u32, num_disks: u32) { 10654 let test_timeout = 10; 10655 let fio_ops = FioOps::RandRW; 10656 10657 let bw_size = if bandwidth { 10658 10485760_u64 // bytes 10659 } else { 10660 100_u64 // I/O 10661 }; 10662 let bw_refill_time = 100; // ms 10663 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10664 10665 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10666 let guest = Guest::new(Box::new(focal)); 10667 let api_socket = temp_api_path(&guest.tmp_dir); 10668 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10669 10670 let rate_limit_group_arg = if bandwidth { 10671 format!("id=group0,bw_size={bw_size},bw_refill_time={bw_refill_time}") 10672 } else { 10673 format!("id=group0,ops_size={bw_size},ops_refill_time={bw_refill_time}") 10674 }; 10675 10676 let mut disk_args = vec![ 10677 "--disk".to_string(), 10678 format!( 10679 "path={}", 10680 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10681 ), 10682 format!( 10683 "path={}", 10684 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10685 ), 10686 ]; 10687 10688 for i in 0..num_disks { 10689 let test_img_path = String::from( 10690 test_img_dir 10691 .as_path() 10692 .join(format!("blk{}.img", i)) 10693 .to_str() 10694 .unwrap(), 10695 ); 10696 10697 assert!(exec_host_command_output(&format!( 10698 "dd if=/dev/zero of={test_img_path} bs=1M count=1024" 10699 )) 10700 .status 10701 .success()); 10702 10703 disk_args.push(format!( 10704 "path={test_img_path},num_queues={num_queues},rate_limit_group=group0" 10705 )); 10706 } 10707 10708 let mut child = GuestCommand::new(&guest) 10709 .args(["--cpus", &format!("boot={}", num_queues * num_disks)]) 10710 .args(["--memory", "size=4G"]) 10711 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10712 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10713 .args(["--rate-limit-group", &rate_limit_group_arg]) 10714 .args(disk_args) 10715 .default_net() 10716 .args(["--api-socket", &api_socket]) 10717 .capture_output() 10718 .spawn() 10719 .unwrap(); 10720 10721 let r = std::panic::catch_unwind(|| { 10722 guest.wait_vm_boot(None).unwrap(); 10723 10724 let mut fio_command = format!( 10725 "sudo fio --name=global --output-format=json \ 10726 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10727 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10728 ); 10729 10730 // Generate additional argument for each disk: 10731 // --name=job0 --filename=/dev/vdc \ 10732 // --name=job1 --filename=/dev/vdd \ 10733 // --name=job2 --filename=/dev/vde \ 10734 // ... 10735 for i in 0..num_disks { 10736 let c: char = 'c'; 10737 let arg = format!( 10738 " --name=job{i} --filename=/dev/vd{}", 10739 char::from_u32((c as u32) + i).unwrap() 10740 ); 10741 fio_command += &arg; 10742 } 10743 let output = guest.ssh_command(&fio_command).unwrap(); 10744 10745 // Parse fio output 10746 let measured_rate = if bandwidth { 10747 parse_fio_output(&output, &fio_ops, num_queues * num_disks).unwrap() 10748 } else { 10749 parse_fio_output_iops(&output, &fio_ops, num_queues * num_disks).unwrap() 10750 }; 10751 assert!(check_rate_limit(measured_rate, limit_rate, 0.2)); 10752 }); 10753 10754 let _ = child.kill(); 10755 let output = child.wait_with_output().unwrap(); 10756 handle_child_output(r, &output); 10757 } 10758 10759 #[test] 10760 fn test_rate_limiter_block_bandwidth() { 10761 _test_rate_limiter_block(true, 1); 10762 _test_rate_limiter_block(true, 2) 10763 } 10764 10765 #[test] 10766 fn test_rate_limiter_group_block_bandwidth() { 10767 _test_rate_limiter_group_block(true, 1, 1); 10768 _test_rate_limiter_group_block(true, 2, 1); 10769 _test_rate_limiter_group_block(true, 1, 2); 10770 _test_rate_limiter_group_block(true, 2, 2); 10771 } 10772 10773 #[test] 10774 fn test_rate_limiter_block_iops() { 10775 _test_rate_limiter_block(false, 1); 10776 _test_rate_limiter_block(false, 2); 10777 } 10778 10779 #[test] 10780 fn test_rate_limiter_group_block_iops() { 10781 _test_rate_limiter_group_block(false, 1, 1); 10782 _test_rate_limiter_group_block(false, 2, 1); 10783 _test_rate_limiter_group_block(false, 1, 2); 10784 _test_rate_limiter_group_block(false, 2, 2); 10785 } 10786 } 10787