1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use std::collections::HashMap; 14 use std::io::{BufRead, Read, Seek, Write}; 15 use std::os::unix::io::AsRawFd; 16 use std::path::PathBuf; 17 use std::process::{Child, Command, Stdio}; 18 use std::string::String; 19 use std::sync::mpsc::Receiver; 20 use std::sync::{mpsc, Mutex}; 21 use std::{fs, io, thread}; 22 23 use net_util::MacAddr; 24 use test_infra::*; 25 use vmm_sys_util::tempdir::TempDir; 26 use vmm_sys_util::tempfile::TempFile; 27 use wait_timeout::ChildExt; 28 29 // Constant taken from the VMM crate. 30 const MAX_NUM_PCI_SEGMENTS: u16 = 96; 31 32 #[cfg(target_arch = "x86_64")] 33 mod x86_64 { 34 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 35 pub const JAMMY_VFIO_IMAGE_NAME: &str = 36 "jammy-server-cloudimg-amd64-custom-vfio-20241012-0.raw"; 37 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 38 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 39 "focal-server-cloudimg-amd64-custom-20210609-0-backing.qcow2"; 40 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 41 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 42 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20241017-0.raw"; 43 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2022-amd64-2.raw"; 44 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 45 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 46 } 47 48 #[cfg(target_arch = "x86_64")] 49 use x86_64::*; 50 51 #[cfg(target_arch = "aarch64")] 52 mod aarch64 { 53 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 54 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 55 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 56 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 57 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 58 "focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2"; 59 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 60 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 61 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 62 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 63 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 64 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 65 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 66 } 67 68 #[cfg(target_arch = "aarch64")] 69 use aarch64::*; 70 71 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 72 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 73 74 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 75 76 // This enum exists to make it more convenient to 77 // implement test for both D-Bus and REST APIs. 78 enum TargetApi { 79 // API socket 80 HttpApi(String), 81 // well known service name, object path 82 DBusApi(String, String), 83 } 84 85 impl TargetApi { 86 fn new_http_api(tmp_dir: &TempDir) -> Self { 87 Self::HttpApi(temp_api_path(tmp_dir)) 88 } 89 90 fn new_dbus_api(tmp_dir: &TempDir) -> Self { 91 // `tmp_dir` is in the form of "/tmp/chXXXXXX" 92 // and we take the `chXXXXXX` part as a unique identifier for the guest 93 let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); 94 95 Self::DBusApi( 96 format!("org.cloudhypervisor.{id}"), 97 format!("/org/cloudhypervisor/{id}"), 98 ) 99 } 100 101 fn guest_args(&self) -> Vec<String> { 102 match self { 103 TargetApi::HttpApi(api_socket) => { 104 vec![format!("--api-socket={}", api_socket.as_str())] 105 } 106 TargetApi::DBusApi(service_name, object_path) => { 107 vec![ 108 format!("--dbus-service-name={}", service_name.as_str()), 109 format!("--dbus-object-path={}", object_path.as_str()), 110 ] 111 } 112 } 113 } 114 115 fn remote_args(&self) -> Vec<String> { 116 // `guest_args` and `remote_args` are consistent with each other 117 self.guest_args() 118 } 119 120 fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { 121 let mut cmd = Command::new(clh_command("ch-remote")); 122 cmd.args(self.remote_args()); 123 cmd.arg(command); 124 125 if let Some(arg) = arg { 126 cmd.arg(arg); 127 } 128 129 let output = cmd.output().unwrap(); 130 if output.status.success() { 131 true 132 } else { 133 eprintln!("Error running ch-remote command: {:?}", &cmd); 134 let stderr = String::from_utf8_lossy(&output.stderr); 135 eprintln!("stderr: {stderr}"); 136 false 137 } 138 } 139 } 140 141 // Start cloud-hypervisor with no VM parameters, only the API server running. 142 // From the API: Create a VM, boot it and check that it looks as expected. 143 fn _test_api_create_boot(target_api: TargetApi, guest: Guest) { 144 let mut child = GuestCommand::new(&guest) 145 .args(target_api.guest_args()) 146 .capture_output() 147 .spawn() 148 .unwrap(); 149 150 thread::sleep(std::time::Duration::new(1, 0)); 151 152 // Verify API server is running 153 assert!(target_api.remote_command("ping", None)); 154 155 // Create the VM first 156 let cpu_count: u8 = 4; 157 let request_body = guest.api_create_body( 158 cpu_count, 159 direct_kernel_boot_path().to_str().unwrap(), 160 DIRECT_KERNEL_BOOT_CMDLINE, 161 ); 162 163 let temp_config_path = guest.tmp_dir.as_path().join("config"); 164 std::fs::write(&temp_config_path, request_body).unwrap(); 165 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 166 167 assert!(target_api.remote_command("create", Some(create_config),)); 168 169 // Then boot it 170 assert!(target_api.remote_command("boot", None)); 171 thread::sleep(std::time::Duration::new(20, 0)); 172 173 let r = std::panic::catch_unwind(|| { 174 // Check that the VM booted as expected 175 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 176 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 177 }); 178 179 kill_child(&mut child); 180 let output = child.wait_with_output().unwrap(); 181 182 handle_child_output(r, &output); 183 } 184 185 // Start cloud-hypervisor with no VM parameters, only the API server running. 186 // From the API: Create a VM, boot it and check it can be shutdown and then 187 // booted again 188 fn _test_api_shutdown(target_api: TargetApi, guest: Guest) { 189 let mut child = GuestCommand::new(&guest) 190 .args(target_api.guest_args()) 191 .capture_output() 192 .spawn() 193 .unwrap(); 194 195 thread::sleep(std::time::Duration::new(1, 0)); 196 197 // Verify API server is running 198 assert!(target_api.remote_command("ping", None)); 199 200 // Create the VM first 201 let cpu_count: u8 = 4; 202 let request_body = guest.api_create_body( 203 cpu_count, 204 direct_kernel_boot_path().to_str().unwrap(), 205 DIRECT_KERNEL_BOOT_CMDLINE, 206 ); 207 208 let temp_config_path = guest.tmp_dir.as_path().join("config"); 209 std::fs::write(&temp_config_path, request_body).unwrap(); 210 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 211 212 let r = std::panic::catch_unwind(|| { 213 assert!(target_api.remote_command("create", Some(create_config))); 214 215 // Then boot it 216 assert!(target_api.remote_command("boot", None)); 217 218 guest.wait_vm_boot(None).unwrap(); 219 220 // Check that the VM booted as expected 221 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 222 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 223 224 // Sync and shutdown without powering off to prevent filesystem 225 // corruption. 226 guest.ssh_command("sync").unwrap(); 227 guest.ssh_command("sudo shutdown -H now").unwrap(); 228 229 // Wait for the guest to be fully shutdown 230 thread::sleep(std::time::Duration::new(20, 0)); 231 232 // Then shut it down 233 assert!(target_api.remote_command("shutdown", None)); 234 235 // Then boot it again 236 assert!(target_api.remote_command("boot", None)); 237 238 guest.wait_vm_boot(None).unwrap(); 239 240 // Check that the VM booted as expected 241 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 242 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 243 }); 244 245 kill_child(&mut child); 246 let output = child.wait_with_output().unwrap(); 247 248 handle_child_output(r, &output); 249 } 250 251 // Start cloud-hypervisor with no VM parameters, only the API server running. 252 // From the API: Create a VM, boot it and check it can be deleted and then recreated 253 // booted again. 254 fn _test_api_delete(target_api: TargetApi, guest: Guest) { 255 let mut child = GuestCommand::new(&guest) 256 .args(target_api.guest_args()) 257 .capture_output() 258 .spawn() 259 .unwrap(); 260 261 thread::sleep(std::time::Duration::new(1, 0)); 262 263 // Verify API server is running 264 assert!(target_api.remote_command("ping", None)); 265 266 // Create the VM first 267 let cpu_count: u8 = 4; 268 let request_body = guest.api_create_body( 269 cpu_count, 270 direct_kernel_boot_path().to_str().unwrap(), 271 DIRECT_KERNEL_BOOT_CMDLINE, 272 ); 273 let temp_config_path = guest.tmp_dir.as_path().join("config"); 274 std::fs::write(&temp_config_path, request_body).unwrap(); 275 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 276 277 let r = std::panic::catch_unwind(|| { 278 assert!(target_api.remote_command("create", Some(create_config))); 279 280 // Then boot it 281 assert!(target_api.remote_command("boot", None)); 282 283 guest.wait_vm_boot(None).unwrap(); 284 285 // Check that the VM booted as expected 286 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 287 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 288 289 // Sync and shutdown without powering off to prevent filesystem 290 // corruption. 291 guest.ssh_command("sync").unwrap(); 292 guest.ssh_command("sudo shutdown -H now").unwrap(); 293 294 // Wait for the guest to be fully shutdown 295 thread::sleep(std::time::Duration::new(20, 0)); 296 297 // Then delete it 298 assert!(target_api.remote_command("delete", None)); 299 300 assert!(target_api.remote_command("create", Some(create_config))); 301 302 // Then boot it again 303 assert!(target_api.remote_command("boot", None)); 304 305 guest.wait_vm_boot(None).unwrap(); 306 307 // Check that the VM booted as expected 308 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 309 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 310 }); 311 312 kill_child(&mut child); 313 let output = child.wait_with_output().unwrap(); 314 315 handle_child_output(r, &output); 316 } 317 318 // Start cloud-hypervisor with no VM parameters, only the API server running. 319 // From the API: Create a VM, boot it and check that it looks as expected. 320 // Then we pause the VM, check that it's no longer available. 321 // Finally we resume the VM and check that it's available. 322 fn _test_api_pause_resume(target_api: TargetApi, guest: Guest) { 323 let mut child = GuestCommand::new(&guest) 324 .args(target_api.guest_args()) 325 .capture_output() 326 .spawn() 327 .unwrap(); 328 329 thread::sleep(std::time::Duration::new(1, 0)); 330 331 // Verify API server is running 332 assert!(target_api.remote_command("ping", None)); 333 334 // Create the VM first 335 let cpu_count: u8 = 4; 336 let request_body = guest.api_create_body( 337 cpu_count, 338 direct_kernel_boot_path().to_str().unwrap(), 339 DIRECT_KERNEL_BOOT_CMDLINE, 340 ); 341 342 let temp_config_path = guest.tmp_dir.as_path().join("config"); 343 std::fs::write(&temp_config_path, request_body).unwrap(); 344 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 345 346 assert!(target_api.remote_command("create", Some(create_config))); 347 348 // Then boot it 349 assert!(target_api.remote_command("boot", None)); 350 thread::sleep(std::time::Duration::new(20, 0)); 351 352 let r = std::panic::catch_unwind(|| { 353 // Check that the VM booted as expected 354 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 355 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 356 357 // We now pause the VM 358 assert!(target_api.remote_command("pause", None)); 359 360 // Check pausing again fails 361 assert!(!target_api.remote_command("pause", None)); 362 363 thread::sleep(std::time::Duration::new(2, 0)); 364 365 // SSH into the VM should fail 366 ssh_command_ip( 367 "grep -c processor /proc/cpuinfo", 368 &guest.network.guest_ip, 369 2, 370 5, 371 ) 372 .unwrap_err(); 373 374 // Resume the VM 375 assert!(target_api.remote_command("resume", None)); 376 377 // Check resuming again fails 378 assert!(!target_api.remote_command("resume", None)); 379 380 thread::sleep(std::time::Duration::new(2, 0)); 381 382 // Now we should be able to SSH back in and get the right number of CPUs 383 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 384 }); 385 386 kill_child(&mut child); 387 let output = child.wait_with_output().unwrap(); 388 389 handle_child_output(r, &output); 390 } 391 392 fn _test_pty_interaction(pty_path: PathBuf) { 393 let mut cf = std::fs::OpenOptions::new() 394 .write(true) 395 .read(true) 396 .open(pty_path) 397 .unwrap(); 398 399 // Some dumb sleeps but we don't want to write 400 // before the console is up and we don't want 401 // to try and write the next line before the 402 // login process is ready. 403 thread::sleep(std::time::Duration::new(5, 0)); 404 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 405 thread::sleep(std::time::Duration::new(2, 0)); 406 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 407 thread::sleep(std::time::Duration::new(2, 0)); 408 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 409 thread::sleep(std::time::Duration::new(2, 0)); 410 411 // read pty and ensure they have a login shell 412 // some fairly hacky workarounds to avoid looping 413 // forever in case the channel is blocked getting output 414 let ptyc = pty_read(cf); 415 let mut empty = 0; 416 let mut prev = String::new(); 417 loop { 418 thread::sleep(std::time::Duration::new(2, 0)); 419 match ptyc.try_recv() { 420 Ok(line) => { 421 empty = 0; 422 prev = prev + &line; 423 if prev.contains("test_pty_console") { 424 break; 425 } 426 } 427 Err(mpsc::TryRecvError::Empty) => { 428 empty += 1; 429 assert!(empty <= 5, "No login on pty"); 430 } 431 _ => { 432 panic!("No login on pty") 433 } 434 } 435 } 436 } 437 438 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 439 let mut workload_path = dirs::home_dir().unwrap(); 440 workload_path.push("workloads"); 441 442 let mut virtiofsd_path = workload_path; 443 virtiofsd_path.push("virtiofsd"); 444 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 445 446 let virtiofsd_socket_path = 447 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 448 449 // Start the daemon 450 let child = Command::new(virtiofsd_path.as_str()) 451 .args(["--shared-dir", shared_dir]) 452 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 453 .args(["--cache", "never"]) 454 .spawn() 455 .unwrap(); 456 457 thread::sleep(std::time::Duration::new(10, 0)); 458 459 (child, virtiofsd_socket_path) 460 } 461 462 fn prepare_vubd( 463 tmp_dir: &TempDir, 464 blk_img: &str, 465 num_queues: usize, 466 rdonly: bool, 467 direct: bool, 468 ) -> (std::process::Child, String) { 469 let mut workload_path = dirs::home_dir().unwrap(); 470 workload_path.push("workloads"); 471 472 let mut blk_file_path = workload_path; 473 blk_file_path.push(blk_img); 474 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 475 476 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 477 478 // Start the daemon 479 let child = Command::new(clh_command("vhost_user_block")) 480 .args([ 481 "--block-backend", 482 format!( 483 "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" 484 ) 485 .as_str(), 486 ]) 487 .spawn() 488 .unwrap(); 489 490 thread::sleep(std::time::Duration::new(10, 0)); 491 492 (child, vubd_socket_path) 493 } 494 495 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 496 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 497 } 498 499 fn temp_api_path(tmp_dir: &TempDir) -> String { 500 String::from( 501 tmp_dir 502 .as_path() 503 .join("cloud-hypervisor.sock") 504 .to_str() 505 .unwrap(), 506 ) 507 } 508 509 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 510 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 511 } 512 513 // Creates the directory and returns the path. 514 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 515 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 516 std::fs::create_dir(&snapshot_dir).unwrap(); 517 snapshot_dir 518 } 519 520 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 521 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 522 vmcore_file 523 } 524 525 // Creates the path for direct kernel boot and return the path. 526 // For x86_64, this function returns the vmlinux kernel path. 527 // For AArch64, this function returns the PE kernel path. 528 fn direct_kernel_boot_path() -> PathBuf { 529 let mut workload_path = dirs::home_dir().unwrap(); 530 workload_path.push("workloads"); 531 532 let mut kernel_path = workload_path; 533 #[cfg(target_arch = "x86_64")] 534 kernel_path.push("vmlinux"); 535 #[cfg(target_arch = "aarch64")] 536 kernel_path.push("Image"); 537 538 kernel_path 539 } 540 541 fn edk2_path() -> PathBuf { 542 let mut workload_path = dirs::home_dir().unwrap(); 543 workload_path.push("workloads"); 544 let mut edk2_path = workload_path; 545 edk2_path.push(OVMF_NAME); 546 547 edk2_path 548 } 549 550 fn cloud_hypervisor_release_path() -> String { 551 let mut workload_path = dirs::home_dir().unwrap(); 552 workload_path.push("workloads"); 553 554 let mut ch_release_path = workload_path; 555 #[cfg(target_arch = "x86_64")] 556 ch_release_path.push("cloud-hypervisor-static"); 557 #[cfg(target_arch = "aarch64")] 558 ch_release_path.push("cloud-hypervisor-static-aarch64"); 559 560 ch_release_path.into_os_string().into_string().unwrap() 561 } 562 563 fn prepare_vhost_user_net_daemon( 564 tmp_dir: &TempDir, 565 ip: &str, 566 tap: Option<&str>, 567 mtu: Option<u16>, 568 num_queues: usize, 569 client_mode: bool, 570 ) -> (std::process::Command, String) { 571 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 572 573 // Start the daemon 574 let mut net_params = format!( 575 "ip={ip},mask=255.255.255.0,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" 576 ); 577 578 if let Some(tap) = tap { 579 net_params.push_str(format!(",tap={tap}").as_str()); 580 } 581 582 if let Some(mtu) = mtu { 583 net_params.push_str(format!(",mtu={mtu}").as_str()); 584 } 585 586 let mut command = Command::new(clh_command("vhost_user_net")); 587 command.args(["--net-backend", net_params.as_str()]); 588 589 (command, vunet_socket_path) 590 } 591 592 fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { 593 let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); 594 let swtpm_socket_path = String::from( 595 tmp_dir 596 .as_path() 597 .join("swtpm") 598 .join("swtpm.sock") 599 .to_str() 600 .unwrap(), 601 ); 602 std::fs::create_dir(&swtpm_tpm_dir).unwrap(); 603 604 let mut swtpm_command = Command::new("swtpm"); 605 let swtpm_args = [ 606 "socket", 607 "--tpmstate", 608 &format!("dir={swtpm_tpm_dir}"), 609 "--ctrl", 610 &format!("type=unixio,path={swtpm_socket_path}"), 611 "--flags", 612 "startup-clear", 613 "--tpm2", 614 ]; 615 swtpm_command.args(swtpm_args); 616 617 (swtpm_command, swtpm_socket_path) 618 } 619 620 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 621 let mut cmd = Command::new(clh_command("ch-remote")); 622 cmd.args([&format!("--api-socket={api_socket}"), command]); 623 624 if let Some(arg) = arg { 625 cmd.arg(arg); 626 } 627 let output = cmd.output().unwrap(); 628 if output.status.success() { 629 true 630 } else { 631 eprintln!("Error running ch-remote command: {:?}", &cmd); 632 let stderr = String::from_utf8_lossy(&output.stderr); 633 eprintln!("stderr: {stderr}"); 634 false 635 } 636 } 637 638 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 639 let mut cmd = Command::new(clh_command("ch-remote")); 640 cmd.args([&format!("--api-socket={api_socket}"), command]); 641 642 if let Some(arg) = arg { 643 cmd.arg(arg); 644 } 645 646 let output = cmd.output().expect("Failed to launch ch-remote"); 647 648 (output.status.success(), output.stdout) 649 } 650 651 fn resize_command( 652 api_socket: &str, 653 desired_vcpus: Option<u8>, 654 desired_ram: Option<usize>, 655 desired_balloon: Option<usize>, 656 event_file: Option<&str>, 657 ) -> bool { 658 let mut cmd = Command::new(clh_command("ch-remote")); 659 cmd.args([&format!("--api-socket={api_socket}"), "resize"]); 660 661 if let Some(desired_vcpus) = desired_vcpus { 662 cmd.arg(format!("--cpus={desired_vcpus}")); 663 } 664 665 if let Some(desired_ram) = desired_ram { 666 cmd.arg(format!("--memory={desired_ram}")); 667 } 668 669 if let Some(desired_balloon) = desired_balloon { 670 cmd.arg(format!("--balloon={desired_balloon}")); 671 } 672 673 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 674 675 if let Some(event_path) = event_file { 676 let latest_events = [ 677 &MetaEvent { 678 event: "resizing".to_string(), 679 device_id: None, 680 }, 681 &MetaEvent { 682 event: "resized".to_string(), 683 device_id: None, 684 }, 685 ]; 686 // See: #5938 687 thread::sleep(std::time::Duration::new(1, 0)); 688 assert!(check_latest_events_exact(&latest_events, event_path)); 689 } 690 691 ret 692 } 693 694 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 695 let mut cmd = Command::new(clh_command("ch-remote")); 696 cmd.args([ 697 &format!("--api-socket={api_socket}"), 698 "resize-zone", 699 &format!("--id={id}"), 700 &format!("--size={desired_size}"), 701 ]); 702 703 cmd.status().expect("Failed to launch ch-remote").success() 704 } 705 706 // setup OVS-DPDK bridge and ports 707 fn setup_ovs_dpdk() { 708 // setup OVS-DPDK 709 assert!(exec_host_command_status("service openvswitch-switch start").success()); 710 assert!(exec_host_command_status("ovs-vsctl init").success()); 711 assert!( 712 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 713 .success() 714 ); 715 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 716 717 // Create OVS-DPDK bridge and ports 718 assert!(exec_host_command_status( 719 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 720 ) 721 .success()); 722 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 723 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 724 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 725 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 726 } 727 fn cleanup_ovs_dpdk() { 728 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 729 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 730 } 731 // Setup two guests and ensure they are connected through ovs-dpdk 732 fn setup_ovs_dpdk_guests( 733 guest1: &Guest, 734 guest2: &Guest, 735 api_socket: &str, 736 release_binary: bool, 737 ) -> (Child, Child) { 738 setup_ovs_dpdk(); 739 740 let clh_path = if !release_binary { 741 clh_command("cloud-hypervisor") 742 } else { 743 cloud_hypervisor_release_path() 744 }; 745 746 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 747 .args(["--cpus", "boot=2"]) 748 .args(["--memory", "size=0,shared=on"]) 749 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 750 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 751 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 752 .default_disks() 753 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 754 .capture_output() 755 .spawn() 756 .unwrap(); 757 758 #[cfg(target_arch = "x86_64")] 759 let guest_net_iface = "ens5"; 760 #[cfg(target_arch = "aarch64")] 761 let guest_net_iface = "enp0s5"; 762 763 let r = std::panic::catch_unwind(|| { 764 guest1.wait_vm_boot(None).unwrap(); 765 766 guest1 767 .ssh_command(&format!( 768 "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" 769 )) 770 .unwrap(); 771 guest1 772 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 773 .unwrap(); 774 775 let guest_ip = guest1.network.guest_ip.clone(); 776 thread::spawn(move || { 777 ssh_command_ip( 778 "nc -l 12345", 779 &guest_ip, 780 DEFAULT_SSH_RETRIES, 781 DEFAULT_SSH_TIMEOUT, 782 ) 783 .unwrap(); 784 }); 785 }); 786 if r.is_err() { 787 cleanup_ovs_dpdk(); 788 789 let _ = child1.kill(); 790 let output = child1.wait_with_output().unwrap(); 791 handle_child_output(r, &output); 792 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 793 } 794 795 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 796 .args(["--api-socket", api_socket]) 797 .args(["--cpus", "boot=2"]) 798 .args(["--memory", "size=0,shared=on"]) 799 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 800 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 801 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 802 .default_disks() 803 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 804 .capture_output() 805 .spawn() 806 .unwrap(); 807 808 let r = std::panic::catch_unwind(|| { 809 guest2.wait_vm_boot(None).unwrap(); 810 811 guest2 812 .ssh_command(&format!( 813 "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" 814 )) 815 .unwrap(); 816 guest2 817 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 818 .unwrap(); 819 820 // Check the connection works properly between the two VMs 821 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 822 }); 823 if r.is_err() { 824 cleanup_ovs_dpdk(); 825 826 let _ = child1.kill(); 827 let _ = child2.kill(); 828 let output = child2.wait_with_output().unwrap(); 829 handle_child_output(r, &output); 830 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 831 } 832 833 (child1, child2) 834 } 835 836 enum FwType { 837 Ovmf, 838 RustHypervisorFirmware, 839 } 840 841 fn fw_path(_fw_type: FwType) -> String { 842 let mut workload_path = dirs::home_dir().unwrap(); 843 workload_path.push("workloads"); 844 845 let mut fw_path = workload_path; 846 #[cfg(target_arch = "aarch64")] 847 fw_path.push("CLOUDHV_EFI.fd"); 848 #[cfg(target_arch = "x86_64")] 849 { 850 match _fw_type { 851 FwType::Ovmf => fw_path.push(OVMF_NAME), 852 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 853 } 854 } 855 856 fw_path.to_str().unwrap().to_string() 857 } 858 859 #[derive(Debug)] 860 struct MetaEvent { 861 event: String, 862 device_id: Option<String>, 863 } 864 865 impl MetaEvent { 866 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 867 let mut matched = false; 868 if v["event"].as_str().unwrap() == self.event { 869 if let Some(device_id) = &self.device_id { 870 if v["properties"]["id"].as_str().unwrap() == device_id { 871 matched = true 872 } 873 } else { 874 matched = true; 875 } 876 } 877 matched 878 } 879 } 880 881 // Parse the event_monitor file based on the format that each event 882 // is followed by a double newline 883 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 884 let content = fs::read(event_file).unwrap(); 885 let mut ret = Vec::new(); 886 for entry in String::from_utf8_lossy(&content) 887 .trim() 888 .split("\n\n") 889 .collect::<Vec<&str>>() 890 { 891 ret.push(serde_json::from_str(entry).unwrap()); 892 } 893 894 ret 895 } 896 897 // Return true if all events from the input 'expected_events' are matched sequentially 898 // with events from the 'event_file' 899 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 900 let json_events = parse_event_file(event_file); 901 let len = expected_events.len(); 902 let mut idx = 0; 903 for e in &json_events { 904 if idx == len { 905 break; 906 } 907 if expected_events[idx].match_with_json_event(e) { 908 idx += 1; 909 } 910 } 911 912 let ret = idx == len; 913 914 if !ret { 915 eprintln!( 916 "\n\n==== Start 'check_sequential_events' failed ==== \ 917 \n\nexpected_events={:?}\nactual_events={:?} \ 918 \n\n==== End 'check_sequential_events' failed ====", 919 expected_events, json_events, 920 ); 921 } 922 923 ret 924 } 925 926 // Return true if all events from the input 'expected_events' are matched exactly 927 // with events from the 'event_file' 928 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 929 let json_events = parse_event_file(event_file); 930 assert!(expected_events.len() <= json_events.len()); 931 let json_events = &json_events[..expected_events.len()]; 932 933 for (idx, e) in json_events.iter().enumerate() { 934 if !expected_events[idx].match_with_json_event(e) { 935 eprintln!( 936 "\n\n==== Start 'check_sequential_events_exact' failed ==== \ 937 \n\nexpected_events={:?}\nactual_events={:?} \ 938 \n\n==== End 'check_sequential_events_exact' failed ====", 939 expected_events, json_events, 940 ); 941 942 return false; 943 } 944 } 945 946 true 947 } 948 949 // Return true if events from the input 'latest_events' are matched exactly 950 // with the most recent events from the 'event_file' 951 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 952 let json_events = parse_event_file(event_file); 953 assert!(latest_events.len() <= json_events.len()); 954 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 955 956 for (idx, e) in json_events.iter().enumerate() { 957 if !latest_events[idx].match_with_json_event(e) { 958 eprintln!( 959 "\n\n==== Start 'check_latest_events_exact' failed ==== \ 960 \n\nexpected_events={:?}\nactual_events={:?} \ 961 \n\n==== End 'check_latest_events_exact' failed ====", 962 latest_events, json_events, 963 ); 964 965 return false; 966 } 967 } 968 969 true 970 } 971 972 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 973 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 974 let guest = Guest::new(Box::new(focal)); 975 let total_vcpus = threads_per_core * cores_per_package * packages; 976 let direct_kernel_boot_path = direct_kernel_boot_path(); 977 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 978 let fw_path = fw_path(FwType::RustHypervisorFirmware); 979 if use_fw { 980 kernel_path = fw_path.as_str(); 981 } 982 983 let mut child = GuestCommand::new(&guest) 984 .args([ 985 "--cpus", 986 &format!( 987 "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" 988 ), 989 ]) 990 .args(["--memory", "size=512M"]) 991 .args(["--kernel", kernel_path]) 992 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 993 .default_disks() 994 .default_net() 995 .capture_output() 996 .spawn() 997 .unwrap(); 998 999 let r = std::panic::catch_unwind(|| { 1000 guest.wait_vm_boot(None).unwrap(); 1001 assert_eq!( 1002 guest.get_cpu_count().unwrap_or_default(), 1003 u32::from(total_vcpus) 1004 ); 1005 assert_eq!( 1006 guest 1007 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1008 .unwrap() 1009 .trim() 1010 .parse::<u8>() 1011 .unwrap_or(0), 1012 threads_per_core 1013 ); 1014 1015 assert_eq!( 1016 guest 1017 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1018 .unwrap() 1019 .trim() 1020 .parse::<u8>() 1021 .unwrap_or(0), 1022 cores_per_package 1023 ); 1024 1025 assert_eq!( 1026 guest 1027 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1028 .unwrap() 1029 .trim() 1030 .parse::<u8>() 1031 .unwrap_or(0), 1032 packages 1033 ); 1034 1035 #[cfg(target_arch = "x86_64")] 1036 { 1037 let mut cpu_id = 0; 1038 for package_id in 0..packages { 1039 for core_id in 0..cores_per_package { 1040 for _ in 0..threads_per_core { 1041 assert_eq!( 1042 guest 1043 .ssh_command(&format!("cat /sys/devices/system/cpu/cpu{cpu_id}/topology/physical_package_id")) 1044 .unwrap() 1045 .trim() 1046 .parse::<u8>() 1047 .unwrap_or(0), 1048 package_id 1049 ); 1050 1051 assert_eq!( 1052 guest 1053 .ssh_command(&format!( 1054 "cat /sys/devices/system/cpu/cpu{cpu_id}/topology/core_id" 1055 )) 1056 .unwrap() 1057 .trim() 1058 .parse::<u8>() 1059 .unwrap_or(0), 1060 core_id 1061 ); 1062 1063 cpu_id += 1; 1064 } 1065 } 1066 } 1067 } 1068 }); 1069 1070 kill_child(&mut child); 1071 let output = child.wait_with_output().unwrap(); 1072 1073 handle_child_output(r, &output); 1074 } 1075 1076 #[allow(unused_variables)] 1077 fn _test_guest_numa_nodes(acpi: bool) { 1078 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1079 let guest = Guest::new(Box::new(focal)); 1080 let api_socket = temp_api_path(&guest.tmp_dir); 1081 #[cfg(target_arch = "x86_64")] 1082 let kernel_path = direct_kernel_boot_path(); 1083 #[cfg(target_arch = "aarch64")] 1084 let kernel_path = if acpi { 1085 edk2_path() 1086 } else { 1087 direct_kernel_boot_path() 1088 }; 1089 1090 let mut child = GuestCommand::new(&guest) 1091 .args(["--cpus", "boot=6,max=12"]) 1092 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 1093 .args([ 1094 "--memory-zone", 1095 "id=mem0,size=1G,hotplug_size=3G", 1096 "id=mem1,size=2G,hotplug_size=3G", 1097 "id=mem2,size=3G,hotplug_size=3G", 1098 ]) 1099 .args([ 1100 "--numa", 1101 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 1102 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 1103 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 1104 ]) 1105 .args(["--kernel", kernel_path.to_str().unwrap()]) 1106 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1107 .args(["--api-socket", &api_socket]) 1108 .capture_output() 1109 .default_disks() 1110 .default_net() 1111 .spawn() 1112 .unwrap(); 1113 1114 let r = std::panic::catch_unwind(|| { 1115 guest.wait_vm_boot(None).unwrap(); 1116 1117 guest.check_numa_common( 1118 Some(&[960_000, 1_920_000, 2_880_000]), 1119 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 1120 Some(&["10 15 20", "20 10 25", "25 30 10"]), 1121 ); 1122 1123 // AArch64 currently does not support hotplug, and therefore we only 1124 // test hotplug-related function on x86_64 here. 1125 #[cfg(target_arch = "x86_64")] 1126 { 1127 guest.enable_memory_hotplug(); 1128 1129 // Resize every memory zone and check each associated NUMA node 1130 // has been assigned the right amount of memory. 1131 resize_zone_command(&api_socket, "mem0", "4G"); 1132 resize_zone_command(&api_socket, "mem1", "4G"); 1133 resize_zone_command(&api_socket, "mem2", "4G"); 1134 // Resize to the maximum amount of CPUs and check each NUMA 1135 // node has been assigned the right CPUs set. 1136 resize_command(&api_socket, Some(12), None, None, None); 1137 thread::sleep(std::time::Duration::new(5, 0)); 1138 1139 guest.check_numa_common( 1140 Some(&[3_840_000, 3_840_000, 3_840_000]), 1141 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 1142 None, 1143 ); 1144 } 1145 }); 1146 1147 kill_child(&mut child); 1148 let output = child.wait_with_output().unwrap(); 1149 1150 handle_child_output(r, &output); 1151 } 1152 1153 #[allow(unused_variables)] 1154 fn _test_power_button(acpi: bool) { 1155 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1156 let guest = Guest::new(Box::new(focal)); 1157 let mut cmd = GuestCommand::new(&guest); 1158 let api_socket = temp_api_path(&guest.tmp_dir); 1159 1160 #[cfg(target_arch = "x86_64")] 1161 let kernel_path = direct_kernel_boot_path(); 1162 #[cfg(target_arch = "aarch64")] 1163 let kernel_path = if acpi { 1164 edk2_path() 1165 } else { 1166 direct_kernel_boot_path() 1167 }; 1168 1169 cmd.args(["--cpus", "boot=1"]) 1170 .args(["--memory", "size=512M"]) 1171 .args(["--kernel", kernel_path.to_str().unwrap()]) 1172 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1173 .capture_output() 1174 .default_disks() 1175 .default_net() 1176 .args(["--api-socket", &api_socket]); 1177 1178 let child = cmd.spawn().unwrap(); 1179 1180 let r = std::panic::catch_unwind(|| { 1181 guest.wait_vm_boot(None).unwrap(); 1182 assert!(remote_command(&api_socket, "power-button", None)); 1183 }); 1184 1185 let output = child.wait_with_output().unwrap(); 1186 assert!(output.status.success()); 1187 handle_child_output(r, &output); 1188 } 1189 1190 type PrepareNetDaemon = dyn Fn( 1191 &TempDir, 1192 &str, 1193 Option<&str>, 1194 Option<u16>, 1195 usize, 1196 bool, 1197 ) -> (std::process::Command, String); 1198 1199 fn test_vhost_user_net( 1200 tap: Option<&str>, 1201 num_queues: usize, 1202 prepare_daemon: &PrepareNetDaemon, 1203 generate_host_mac: bool, 1204 client_mode_daemon: bool, 1205 ) { 1206 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1207 let guest = Guest::new(Box::new(focal)); 1208 let api_socket = temp_api_path(&guest.tmp_dir); 1209 1210 let kernel_path = direct_kernel_boot_path(); 1211 1212 let host_mac = if generate_host_mac { 1213 Some(MacAddr::local_random()) 1214 } else { 1215 None 1216 }; 1217 1218 let mtu = Some(3000); 1219 1220 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 1221 &guest.tmp_dir, 1222 &guest.network.host_ip, 1223 tap, 1224 mtu, 1225 num_queues, 1226 client_mode_daemon, 1227 ); 1228 1229 let net_params = format!( 1230 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 1231 guest.network.guest_mac, 1232 vunet_socket_path, 1233 num_queues, 1234 if let Some(host_mac) = host_mac { 1235 format!(",host_mac={host_mac}") 1236 } else { 1237 "".to_owned() 1238 }, 1239 if client_mode_daemon { 1240 "server" 1241 } else { 1242 "client" 1243 }, 1244 ); 1245 1246 let mut ch_command = GuestCommand::new(&guest); 1247 ch_command 1248 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 1249 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1250 .args(["--kernel", kernel_path.to_str().unwrap()]) 1251 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1252 .default_disks() 1253 .args(["--net", net_params.as_str()]) 1254 .args(["--api-socket", &api_socket]) 1255 .capture_output(); 1256 1257 let mut daemon_child: std::process::Child; 1258 let mut child: std::process::Child; 1259 1260 if client_mode_daemon { 1261 child = ch_command.spawn().unwrap(); 1262 // Make sure the VMM is waiting for the backend to connect 1263 thread::sleep(std::time::Duration::new(10, 0)); 1264 daemon_child = daemon_command.spawn().unwrap(); 1265 } else { 1266 daemon_child = daemon_command.spawn().unwrap(); 1267 // Make sure the backend is waiting for the VMM to connect 1268 thread::sleep(std::time::Duration::new(10, 0)); 1269 child = ch_command.spawn().unwrap(); 1270 } 1271 1272 let r = std::panic::catch_unwind(|| { 1273 guest.wait_vm_boot(None).unwrap(); 1274 1275 if let Some(tap_name) = tap { 1276 let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); 1277 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1278 } 1279 1280 if let Some(host_mac) = tap { 1281 let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); 1282 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1283 } 1284 1285 #[cfg(target_arch = "aarch64")] 1286 let iface = "enp0s4"; 1287 #[cfg(target_arch = "x86_64")] 1288 let iface = "ens4"; 1289 1290 assert_eq!( 1291 guest 1292 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 1293 .unwrap() 1294 .trim(), 1295 "3000" 1296 ); 1297 1298 // 1 network interface + default localhost ==> 2 interfaces 1299 // It's important to note that this test is fully exercising the 1300 // vhost-user-net implementation and the associated backend since 1301 // it does not define any --net network interface. That means all 1302 // the ssh communication in that test happens through the network 1303 // interface backed by vhost-user-net. 1304 assert_eq!( 1305 guest 1306 .ssh_command("ip -o link | wc -l") 1307 .unwrap() 1308 .trim() 1309 .parse::<u32>() 1310 .unwrap_or_default(), 1311 2 1312 ); 1313 1314 // The following pci devices will appear on guest with PCI-MSI 1315 // interrupt vectors assigned. 1316 // 1 virtio-console with 3 vectors: config, Rx, Tx 1317 // 1 virtio-blk with 2 vectors: config, Request 1318 // 1 virtio-blk with 2 vectors: config, Request 1319 // 1 virtio-rng with 2 vectors: config, Request 1320 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1321 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1322 // Based on the above, the total vectors should 14. 1323 #[cfg(target_arch = "x86_64")] 1324 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1325 #[cfg(target_arch = "aarch64")] 1326 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1327 assert_eq!( 1328 guest 1329 .ssh_command(grep_cmd) 1330 .unwrap() 1331 .trim() 1332 .parse::<u32>() 1333 .unwrap_or_default(), 1334 10 + (num_queues as u32) 1335 ); 1336 1337 // ACPI feature is needed. 1338 #[cfg(target_arch = "x86_64")] 1339 { 1340 guest.enable_memory_hotplug(); 1341 1342 // Add RAM to the VM 1343 let desired_ram = 1024 << 20; 1344 resize_command(&api_socket, None, Some(desired_ram), None, None); 1345 1346 thread::sleep(std::time::Duration::new(10, 0)); 1347 1348 // Here by simply checking the size (through ssh), we validate 1349 // the connection is still working, which means vhost-user-net 1350 // keeps working after the resize. 1351 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1352 } 1353 }); 1354 1355 kill_child(&mut child); 1356 let output = child.wait_with_output().unwrap(); 1357 1358 thread::sleep(std::time::Duration::new(5, 0)); 1359 let _ = daemon_child.kill(); 1360 let _ = daemon_child.wait(); 1361 1362 handle_child_output(r, &output); 1363 } 1364 1365 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1366 1367 fn test_vhost_user_blk( 1368 num_queues: usize, 1369 readonly: bool, 1370 direct: bool, 1371 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1372 ) { 1373 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1374 let guest = Guest::new(Box::new(focal)); 1375 let api_socket = temp_api_path(&guest.tmp_dir); 1376 1377 let kernel_path = direct_kernel_boot_path(); 1378 1379 let (blk_params, daemon_child) = { 1380 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1381 // Start the daemon 1382 let (daemon_child, vubd_socket_path) = 1383 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1384 1385 ( 1386 format!( 1387 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1388 ), 1389 Some(daemon_child), 1390 ) 1391 }; 1392 1393 let mut child = GuestCommand::new(&guest) 1394 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1395 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1396 .args(["--kernel", kernel_path.to_str().unwrap()]) 1397 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1398 .args([ 1399 "--disk", 1400 format!( 1401 "path={}", 1402 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1403 ) 1404 .as_str(), 1405 format!( 1406 "path={}", 1407 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1408 ) 1409 .as_str(), 1410 blk_params.as_str(), 1411 ]) 1412 .default_net() 1413 .args(["--api-socket", &api_socket]) 1414 .capture_output() 1415 .spawn() 1416 .unwrap(); 1417 1418 let r = std::panic::catch_unwind(|| { 1419 guest.wait_vm_boot(None).unwrap(); 1420 1421 // Check both if /dev/vdc exists and if the block size is 16M. 1422 assert_eq!( 1423 guest 1424 .ssh_command("lsblk | grep vdc | grep -c 16M") 1425 .unwrap() 1426 .trim() 1427 .parse::<u32>() 1428 .unwrap_or_default(), 1429 1 1430 ); 1431 1432 // Check if this block is RO or RW. 1433 assert_eq!( 1434 guest 1435 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1436 .unwrap() 1437 .trim() 1438 .parse::<u32>() 1439 .unwrap_or_default(), 1440 readonly as u32 1441 ); 1442 1443 // Check if the number of queues in /sys/block/vdc/mq matches the 1444 // expected num_queues. 1445 assert_eq!( 1446 guest 1447 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1448 .unwrap() 1449 .trim() 1450 .parse::<u32>() 1451 .unwrap_or_default(), 1452 num_queues as u32 1453 ); 1454 1455 // Mount the device 1456 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1457 guest.ssh_command("mkdir mount_image").unwrap(); 1458 guest 1459 .ssh_command( 1460 format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), 1461 ) 1462 .unwrap(); 1463 1464 // Check the content of the block device. The file "foo" should 1465 // contain "bar". 1466 assert_eq!( 1467 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1468 "bar" 1469 ); 1470 1471 // ACPI feature is needed. 1472 #[cfg(target_arch = "x86_64")] 1473 { 1474 guest.enable_memory_hotplug(); 1475 1476 // Add RAM to the VM 1477 let desired_ram = 1024 << 20; 1478 resize_command(&api_socket, None, Some(desired_ram), None, None); 1479 1480 thread::sleep(std::time::Duration::new(10, 0)); 1481 1482 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1483 1484 // Check again the content of the block device after the resize 1485 // has been performed. 1486 assert_eq!( 1487 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1488 "bar" 1489 ); 1490 } 1491 1492 // Unmount the device 1493 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1494 guest.ssh_command("rm -r mount_image").unwrap(); 1495 }); 1496 1497 kill_child(&mut child); 1498 let output = child.wait_with_output().unwrap(); 1499 1500 if let Some(mut daemon_child) = daemon_child { 1501 thread::sleep(std::time::Duration::new(5, 0)); 1502 let _ = daemon_child.kill(); 1503 let _ = daemon_child.wait(); 1504 } 1505 1506 handle_child_output(r, &output); 1507 } 1508 1509 fn test_boot_from_vhost_user_blk( 1510 num_queues: usize, 1511 readonly: bool, 1512 direct: bool, 1513 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1514 ) { 1515 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1516 let guest = Guest::new(Box::new(focal)); 1517 1518 let kernel_path = direct_kernel_boot_path(); 1519 1520 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1521 1522 let (blk_boot_params, daemon_child) = { 1523 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1524 // Start the daemon 1525 let (daemon_child, vubd_socket_path) = prepare_daemon( 1526 &guest.tmp_dir, 1527 disk_path.as_str(), 1528 num_queues, 1529 readonly, 1530 direct, 1531 ); 1532 1533 ( 1534 format!( 1535 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1536 ), 1537 Some(daemon_child), 1538 ) 1539 }; 1540 1541 let mut child = GuestCommand::new(&guest) 1542 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1543 .args(["--memory", "size=512M,shared=on"]) 1544 .args(["--kernel", kernel_path.to_str().unwrap()]) 1545 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1546 .args([ 1547 "--disk", 1548 blk_boot_params.as_str(), 1549 format!( 1550 "path={}", 1551 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1552 ) 1553 .as_str(), 1554 ]) 1555 .default_net() 1556 .capture_output() 1557 .spawn() 1558 .unwrap(); 1559 1560 let r = std::panic::catch_unwind(|| { 1561 guest.wait_vm_boot(None).unwrap(); 1562 1563 // Just check the VM booted correctly. 1564 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1565 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1566 }); 1567 kill_child(&mut child); 1568 let output = child.wait_with_output().unwrap(); 1569 1570 if let Some(mut daemon_child) = daemon_child { 1571 thread::sleep(std::time::Duration::new(5, 0)); 1572 let _ = daemon_child.kill(); 1573 let _ = daemon_child.wait(); 1574 } 1575 1576 handle_child_output(r, &output); 1577 } 1578 1579 fn _test_virtio_fs( 1580 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1581 hotplug: bool, 1582 pci_segment: Option<u16>, 1583 ) { 1584 #[cfg(target_arch = "aarch64")] 1585 let focal_image = if hotplug { 1586 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1587 } else { 1588 FOCAL_IMAGE_NAME.to_string() 1589 }; 1590 #[cfg(target_arch = "x86_64")] 1591 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1592 let focal = UbuntuDiskConfig::new(focal_image); 1593 let guest = Guest::new(Box::new(focal)); 1594 let api_socket = temp_api_path(&guest.tmp_dir); 1595 1596 let mut workload_path = dirs::home_dir().unwrap(); 1597 workload_path.push("workloads"); 1598 1599 let mut shared_dir = workload_path; 1600 shared_dir.push("shared_dir"); 1601 1602 #[cfg(target_arch = "x86_64")] 1603 let kernel_path = direct_kernel_boot_path(); 1604 #[cfg(target_arch = "aarch64")] 1605 let kernel_path = if hotplug { 1606 edk2_path() 1607 } else { 1608 direct_kernel_boot_path() 1609 }; 1610 1611 let (mut daemon_child, virtiofsd_socket_path) = 1612 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1613 1614 let mut guest_command = GuestCommand::new(&guest); 1615 guest_command 1616 .args(["--cpus", "boot=1"]) 1617 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1618 .args(["--kernel", kernel_path.to_str().unwrap()]) 1619 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1620 .default_disks() 1621 .default_net() 1622 .args(["--api-socket", &api_socket]); 1623 if pci_segment.is_some() { 1624 guest_command.args([ 1625 "--platform", 1626 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 1627 ]); 1628 } 1629 1630 let fs_params = format!( 1631 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1632 virtiofsd_socket_path, 1633 if let Some(pci_segment) = pci_segment { 1634 format!(",pci_segment={pci_segment}") 1635 } else { 1636 "".to_owned() 1637 } 1638 ); 1639 1640 if !hotplug { 1641 guest_command.args(["--fs", fs_params.as_str()]); 1642 } 1643 1644 let mut child = guest_command.capture_output().spawn().unwrap(); 1645 1646 let r = std::panic::catch_unwind(|| { 1647 guest.wait_vm_boot(None).unwrap(); 1648 1649 if hotplug { 1650 // Add fs to the VM 1651 let (cmd_success, cmd_output) = 1652 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1653 assert!(cmd_success); 1654 1655 if let Some(pci_segment) = pci_segment { 1656 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1657 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1658 ))); 1659 } else { 1660 assert!(String::from_utf8_lossy(&cmd_output) 1661 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1662 } 1663 1664 thread::sleep(std::time::Duration::new(10, 0)); 1665 } 1666 1667 // Mount shared directory through virtio_fs filesystem 1668 guest 1669 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1670 .unwrap(); 1671 1672 // Check file1 exists and its content is "foo" 1673 assert_eq!( 1674 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1675 "foo" 1676 ); 1677 // Check file2 does not exist 1678 guest 1679 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1680 .unwrap(); 1681 1682 // Check file3 exists and its content is "bar" 1683 assert_eq!( 1684 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1685 "bar" 1686 ); 1687 1688 // ACPI feature is needed. 1689 #[cfg(target_arch = "x86_64")] 1690 { 1691 guest.enable_memory_hotplug(); 1692 1693 // Add RAM to the VM 1694 let desired_ram = 1024 << 20; 1695 resize_command(&api_socket, None, Some(desired_ram), None, None); 1696 1697 thread::sleep(std::time::Duration::new(30, 0)); 1698 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1699 1700 // After the resize, check again that file1 exists and its 1701 // content is "foo". 1702 assert_eq!( 1703 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1704 "foo" 1705 ); 1706 } 1707 1708 if hotplug { 1709 // Remove from VM 1710 guest.ssh_command("sudo umount mount_dir").unwrap(); 1711 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1712 } 1713 }); 1714 1715 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1716 thread::sleep(std::time::Duration::new(10, 0)); 1717 let (daemon_child, virtiofsd_socket_path) = 1718 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1719 1720 let r = std::panic::catch_unwind(|| { 1721 thread::sleep(std::time::Duration::new(10, 0)); 1722 let fs_params = format!( 1723 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1724 virtiofsd_socket_path, 1725 if let Some(pci_segment) = pci_segment { 1726 format!(",pci_segment={pci_segment}") 1727 } else { 1728 "".to_owned() 1729 } 1730 ); 1731 1732 // Add back and check it works 1733 let (cmd_success, cmd_output) = 1734 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1735 assert!(cmd_success); 1736 if let Some(pci_segment) = pci_segment { 1737 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1738 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1739 ))); 1740 } else { 1741 assert!(String::from_utf8_lossy(&cmd_output) 1742 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1743 } 1744 1745 thread::sleep(std::time::Duration::new(10, 0)); 1746 // Mount shared directory through virtio_fs filesystem 1747 guest 1748 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1749 .unwrap(); 1750 1751 // Check file1 exists and its content is "foo" 1752 assert_eq!( 1753 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1754 "foo" 1755 ); 1756 }); 1757 1758 (r, Some(daemon_child)) 1759 } else { 1760 (r, None) 1761 }; 1762 1763 kill_child(&mut child); 1764 let output = child.wait_with_output().unwrap(); 1765 1766 let _ = daemon_child.kill(); 1767 let _ = daemon_child.wait(); 1768 1769 if let Some(mut daemon_child) = hotplug_daemon_child { 1770 let _ = daemon_child.kill(); 1771 let _ = daemon_child.wait(); 1772 } 1773 1774 handle_child_output(r, &output); 1775 } 1776 1777 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1778 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1779 let guest = Guest::new(Box::new(focal)); 1780 1781 let kernel_path = direct_kernel_boot_path(); 1782 1783 let pmem_temp_file = TempFile::new().unwrap(); 1784 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1785 1786 std::process::Command::new("mkfs.ext4") 1787 .arg(pmem_temp_file.as_path()) 1788 .output() 1789 .expect("Expect creating disk image to succeed"); 1790 1791 let mut child = GuestCommand::new(&guest) 1792 .args(["--cpus", "boot=1"]) 1793 .args(["--memory", "size=512M"]) 1794 .args(["--kernel", kernel_path.to_str().unwrap()]) 1795 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1796 .default_disks() 1797 .default_net() 1798 .args([ 1799 "--pmem", 1800 format!( 1801 "file={}{}{}", 1802 pmem_temp_file.as_path().to_str().unwrap(), 1803 if specify_size { ",size=128M" } else { "" }, 1804 if discard_writes { 1805 ",discard_writes=on" 1806 } else { 1807 "" 1808 } 1809 ) 1810 .as_str(), 1811 ]) 1812 .capture_output() 1813 .spawn() 1814 .unwrap(); 1815 1816 let r = std::panic::catch_unwind(|| { 1817 guest.wait_vm_boot(None).unwrap(); 1818 1819 // Check for the presence of /dev/pmem0 1820 assert_eq!( 1821 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1822 "/dev/pmem0" 1823 ); 1824 1825 // Check changes persist after reboot 1826 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1827 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1828 guest 1829 .ssh_command("echo test123 | sudo tee /mnt/test") 1830 .unwrap(); 1831 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1832 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1833 1834 guest.reboot_linux(0, None); 1835 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1836 assert_eq!( 1837 guest 1838 .ssh_command("sudo cat /mnt/test || true") 1839 .unwrap() 1840 .trim(), 1841 if discard_writes { "" } else { "test123" } 1842 ); 1843 }); 1844 1845 kill_child(&mut child); 1846 let output = child.wait_with_output().unwrap(); 1847 1848 handle_child_output(r, &output); 1849 } 1850 1851 fn get_fd_count(pid: u32) -> usize { 1852 fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() 1853 } 1854 1855 fn _test_virtio_vsock(hotplug: bool) { 1856 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1857 let guest = Guest::new(Box::new(focal)); 1858 1859 #[cfg(target_arch = "x86_64")] 1860 let kernel_path = direct_kernel_boot_path(); 1861 #[cfg(target_arch = "aarch64")] 1862 let kernel_path = if hotplug { 1863 edk2_path() 1864 } else { 1865 direct_kernel_boot_path() 1866 }; 1867 1868 let socket = temp_vsock_path(&guest.tmp_dir); 1869 let api_socket = temp_api_path(&guest.tmp_dir); 1870 1871 let mut cmd = GuestCommand::new(&guest); 1872 cmd.args(["--api-socket", &api_socket]); 1873 cmd.args(["--cpus", "boot=1"]); 1874 cmd.args(["--memory", "size=512M"]); 1875 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1876 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1877 cmd.default_disks(); 1878 cmd.default_net(); 1879 1880 if !hotplug { 1881 cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); 1882 } 1883 1884 let mut child = cmd.capture_output().spawn().unwrap(); 1885 1886 let r = std::panic::catch_unwind(|| { 1887 guest.wait_vm_boot(None).unwrap(); 1888 1889 if hotplug { 1890 let (cmd_success, cmd_output) = remote_command_w_output( 1891 &api_socket, 1892 "add-vsock", 1893 Some(format!("cid=3,socket={socket},id=test0").as_str()), 1894 ); 1895 assert!(cmd_success); 1896 assert!(String::from_utf8_lossy(&cmd_output) 1897 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1898 thread::sleep(std::time::Duration::new(10, 0)); 1899 // Check adding a second one fails 1900 assert!(!remote_command( 1901 &api_socket, 1902 "add-vsock", 1903 Some("cid=1234,socket=/tmp/fail") 1904 )); 1905 } 1906 1907 // Validate vsock works as expected. 1908 guest.check_vsock(socket.as_str()); 1909 guest.reboot_linux(0, None); 1910 // Validate vsock still works after a reboot. 1911 guest.check_vsock(socket.as_str()); 1912 1913 if hotplug { 1914 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1915 } 1916 }); 1917 1918 kill_child(&mut child); 1919 let output = child.wait_with_output().unwrap(); 1920 1921 handle_child_output(r, &output); 1922 } 1923 1924 fn get_ksm_pages_shared() -> u32 { 1925 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1926 .unwrap() 1927 .trim() 1928 .parse::<u32>() 1929 .unwrap() 1930 } 1931 1932 fn test_memory_mergeable(mergeable: bool) { 1933 let memory_param = if mergeable { 1934 "mergeable=on" 1935 } else { 1936 "mergeable=off" 1937 }; 1938 1939 // We are assuming the rest of the system in our CI is not using mergeable memory 1940 let ksm_ps_init = get_ksm_pages_shared(); 1941 assert!(ksm_ps_init == 0); 1942 1943 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1944 let guest1 = Guest::new(Box::new(focal1)); 1945 let mut child1 = GuestCommand::new(&guest1) 1946 .args(["--cpus", "boot=1"]) 1947 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1948 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1949 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1950 .default_disks() 1951 .args(["--net", guest1.default_net_string().as_str()]) 1952 .args(["--serial", "tty", "--console", "off"]) 1953 .capture_output() 1954 .spawn() 1955 .unwrap(); 1956 1957 let r = std::panic::catch_unwind(|| { 1958 guest1.wait_vm_boot(None).unwrap(); 1959 }); 1960 if r.is_err() { 1961 kill_child(&mut child1); 1962 let output = child1.wait_with_output().unwrap(); 1963 handle_child_output(r, &output); 1964 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1965 } 1966 1967 let ksm_ps_guest1 = get_ksm_pages_shared(); 1968 1969 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1970 let guest2 = Guest::new(Box::new(focal2)); 1971 let mut child2 = GuestCommand::new(&guest2) 1972 .args(["--cpus", "boot=1"]) 1973 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1974 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1975 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1976 .default_disks() 1977 .args(["--net", guest2.default_net_string().as_str()]) 1978 .args(["--serial", "tty", "--console", "off"]) 1979 .capture_output() 1980 .spawn() 1981 .unwrap(); 1982 1983 let r = std::panic::catch_unwind(|| { 1984 guest2.wait_vm_boot(None).unwrap(); 1985 let ksm_ps_guest2 = get_ksm_pages_shared(); 1986 1987 if mergeable { 1988 println!( 1989 "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" 1990 ); 1991 // We are expecting the number of shared pages to increase as the number of VM increases 1992 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1993 } else { 1994 assert!(ksm_ps_guest1 == 0); 1995 assert!(ksm_ps_guest2 == 0); 1996 } 1997 }); 1998 1999 kill_child(&mut child1); 2000 kill_child(&mut child2); 2001 2002 let output = child1.wait_with_output().unwrap(); 2003 child2.wait().unwrap(); 2004 2005 handle_child_output(r, &output); 2006 } 2007 2008 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 2009 let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); 2010 let reader = io::BufReader::new(smaps); 2011 2012 let mut skip_map: bool = false; 2013 let mut region_name: String = "".to_string(); 2014 let mut region_maps = HashMap::new(); 2015 for line in reader.lines() { 2016 let l = line.unwrap(); 2017 2018 if l.contains('-') { 2019 let values: Vec<&str> = l.split_whitespace().collect(); 2020 region_name = values.last().unwrap().trim().to_string(); 2021 if region_name == "0" { 2022 region_name = "anonymous".to_string() 2023 } 2024 } 2025 2026 // Each section begins with something that looks like: 2027 // Size: 2184 kB 2028 if l.starts_with("Size:") { 2029 let values: Vec<&str> = l.split_whitespace().collect(); 2030 let map_size = values[1].parse::<u32>().unwrap(); 2031 // We skip the assigned guest RAM map, its RSS is only 2032 // dependent on the guest actual memory usage. 2033 // Everything else can be added to the VMM overhead. 2034 skip_map = map_size >= guest_memory_size; 2035 continue; 2036 } 2037 2038 // If this is a map we're taking into account, then we only 2039 // count the RSS. The sum of all counted RSS is the VMM overhead. 2040 if !skip_map && l.starts_with("Rss:") { 2041 let values: Vec<&str> = l.split_whitespace().collect(); 2042 let value = values[1].trim().parse::<u32>().unwrap(); 2043 *region_maps.entry(region_name.clone()).or_insert(0) += value; 2044 } 2045 } 2046 2047 region_maps 2048 } 2049 2050 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 2051 let mut total = 0; 2052 2053 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 2054 eprintln!("{region_name}: {value}"); 2055 total += value; 2056 } 2057 2058 total 2059 } 2060 2061 fn process_rss_kib(pid: u32) -> usize { 2062 let command = format!("ps -q {pid} -o rss="); 2063 let rss = exec_host_command_output(&command); 2064 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 2065 } 2066 2067 // 10MB is our maximum accepted overhead. 2068 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 2069 2070 #[derive(PartialEq, Eq, PartialOrd)] 2071 struct Counters { 2072 rx_bytes: u64, 2073 rx_frames: u64, 2074 tx_bytes: u64, 2075 tx_frames: u64, 2076 read_bytes: u64, 2077 write_bytes: u64, 2078 read_ops: u64, 2079 write_ops: u64, 2080 } 2081 2082 fn get_counters(api_socket: &str) -> Counters { 2083 // Get counters 2084 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 2085 assert!(cmd_success); 2086 2087 let counters: HashMap<&str, HashMap<&str, u64>> = 2088 serde_json::from_slice(&cmd_output).unwrap_or_default(); 2089 2090 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 2091 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 2092 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 2093 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 2094 2095 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 2096 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 2097 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 2098 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 2099 2100 Counters { 2101 rx_bytes, 2102 rx_frames, 2103 tx_bytes, 2104 tx_frames, 2105 read_bytes, 2106 write_bytes, 2107 read_ops, 2108 write_ops, 2109 } 2110 } 2111 2112 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 2113 let (tx, rx) = mpsc::channel::<String>(); 2114 thread::spawn(move || loop { 2115 thread::sleep(std::time::Duration::new(1, 0)); 2116 let mut buf = [0; 512]; 2117 match pty.read(&mut buf) { 2118 Ok(_bytes) => { 2119 let output = std::str::from_utf8(&buf).unwrap().to_string(); 2120 match tx.send(output) { 2121 Ok(_) => (), 2122 Err(_) => break, 2123 } 2124 } 2125 Err(_) => break, 2126 } 2127 }); 2128 rx 2129 } 2130 2131 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 2132 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2133 assert!(cmd_success); 2134 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2135 assert_eq!("Pty", info["config"][pty_type]["mode"]); 2136 PathBuf::from( 2137 info["config"][pty_type]["file"] 2138 .as_str() 2139 .expect("Missing pty path"), 2140 ) 2141 } 2142 2143 // VFIO test network setup. 2144 // We reserve a different IP class for it: 172.18.0.0/24. 2145 #[cfg(target_arch = "x86_64")] 2146 fn setup_vfio_network_interfaces() { 2147 // 'vfio-br0' 2148 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 2149 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 2150 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 2151 // 'vfio-tap0' 2152 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 2153 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 2154 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 2155 // 'vfio-tap1' 2156 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 2157 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 2158 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 2159 // 'vfio-tap2' 2160 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 2161 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 2162 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 2163 // 'vfio-tap3' 2164 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 2165 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 2166 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 2167 } 2168 2169 // Tear VFIO test network down 2170 #[cfg(target_arch = "x86_64")] 2171 fn cleanup_vfio_network_interfaces() { 2172 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 2173 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 2174 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 2175 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 2176 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 2177 } 2178 2179 fn balloon_size(api_socket: &str) -> u64 { 2180 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2181 assert!(cmd_success); 2182 2183 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2184 let total_mem = &info["config"]["memory"]["size"] 2185 .to_string() 2186 .parse::<u64>() 2187 .unwrap(); 2188 let actual_mem = &info["memory_actual_size"] 2189 .to_string() 2190 .parse::<u64>() 2191 .unwrap(); 2192 total_mem - actual_mem 2193 } 2194 2195 fn vm_state(api_socket: &str) -> String { 2196 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2197 assert!(cmd_success); 2198 2199 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2200 let state = &info["state"].as_str().unwrap(); 2201 2202 state.to_string() 2203 } 2204 2205 // This test validates that it can find the virtio-iommu device at first. 2206 // It also verifies that both disks and the network card are attached to 2207 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 2208 // The last interesting part of this test is that it exercises the network 2209 // interface attached to the virtual IOMMU since this is the one used to 2210 // send all commands through SSH. 2211 fn _test_virtio_iommu(acpi: bool) { 2212 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 2213 // Focal image is still old. 2214 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 2215 // the kernel binary has been updated. 2216 #[cfg(target_arch = "aarch64")] 2217 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 2218 #[cfg(target_arch = "x86_64")] 2219 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2220 let focal = UbuntuDiskConfig::new(focal_image); 2221 let guest = Guest::new(Box::new(focal)); 2222 2223 #[cfg(target_arch = "x86_64")] 2224 let kernel_path = direct_kernel_boot_path(); 2225 #[cfg(target_arch = "aarch64")] 2226 let kernel_path = if acpi { 2227 edk2_path() 2228 } else { 2229 direct_kernel_boot_path() 2230 }; 2231 2232 let mut child = GuestCommand::new(&guest) 2233 .args(["--cpus", "boot=1"]) 2234 .args(["--memory", "size=512M"]) 2235 .args(["--kernel", kernel_path.to_str().unwrap()]) 2236 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2237 .args([ 2238 "--disk", 2239 format!( 2240 "path={},iommu=on", 2241 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2242 ) 2243 .as_str(), 2244 format!( 2245 "path={},iommu=on", 2246 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2247 ) 2248 .as_str(), 2249 ]) 2250 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 2251 .capture_output() 2252 .spawn() 2253 .unwrap(); 2254 2255 let r = std::panic::catch_unwind(|| { 2256 guest.wait_vm_boot(None).unwrap(); 2257 2258 // Verify the virtio-iommu device is present. 2259 assert!(guest 2260 .does_device_vendor_pair_match("0x1057", "0x1af4") 2261 .unwrap_or_default()); 2262 2263 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 2264 // different with ACPI. 2265 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 2266 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 2267 // and appear under folder '/sys/kernel/iommu_groups/'. 2268 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 2269 // which is the console. The first disk "0000:00:02.0" is in group '1'. 2270 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 2271 // contains "0000:00:02.0" which is the first disk. 2272 // 2273 // Verify the iommu group of the first disk. 2274 let iommu_group = !acpi as i32; 2275 assert_eq!( 2276 guest 2277 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2278 .unwrap() 2279 .trim(), 2280 "0000:00:02.0" 2281 ); 2282 2283 // Verify the iommu group of the second disk. 2284 let iommu_group = if acpi { 1 } else { 2 }; 2285 assert_eq!( 2286 guest 2287 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2288 .unwrap() 2289 .trim(), 2290 "0000:00:03.0" 2291 ); 2292 2293 // Verify the iommu group of the network card. 2294 let iommu_group = if acpi { 2 } else { 3 }; 2295 assert_eq!( 2296 guest 2297 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2298 .unwrap() 2299 .trim(), 2300 "0000:00:04.0" 2301 ); 2302 }); 2303 2304 kill_child(&mut child); 2305 let output = child.wait_with_output().unwrap(); 2306 2307 handle_child_output(r, &output); 2308 } 2309 2310 fn get_reboot_count(guest: &Guest) -> u32 { 2311 guest 2312 .ssh_command("sudo last | grep -c reboot") 2313 .unwrap() 2314 .trim() 2315 .parse::<u32>() 2316 .unwrap_or_default() 2317 } 2318 2319 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 2320 // Check for PCI device 2321 assert!(guest 2322 .does_device_vendor_pair_match("0x1063", "0x1af4") 2323 .unwrap_or_default()); 2324 2325 // Enable systemd watchdog 2326 guest 2327 .ssh_command(&format!( 2328 "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" 2329 )) 2330 .unwrap(); 2331 2332 guest.ssh_command("sudo systemctl daemon-reexec").unwrap(); 2333 } 2334 2335 fn make_guest_panic(guest: &Guest) { 2336 // Check for pvpanic device 2337 assert!(guest 2338 .does_device_vendor_pair_match("0x0011", "0x1b36") 2339 .unwrap_or_default()); 2340 2341 // Trigger guest a panic 2342 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 2343 } 2344 2345 mod common_parallel { 2346 use std::fs::OpenOptions; 2347 use std::io::SeekFrom; 2348 2349 use crate::*; 2350 2351 #[test] 2352 #[cfg(target_arch = "x86_64")] 2353 fn test_focal_hypervisor_fw() { 2354 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 2355 } 2356 2357 #[test] 2358 #[cfg(target_arch = "x86_64")] 2359 fn test_focal_ovmf() { 2360 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 2361 } 2362 2363 #[cfg(target_arch = "x86_64")] 2364 fn test_simple_launch(fw_path: String, disk_path: &str) { 2365 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 2366 let guest = Guest::new(disk_config); 2367 let event_path = temp_event_monitor_path(&guest.tmp_dir); 2368 2369 let mut child = GuestCommand::new(&guest) 2370 .args(["--cpus", "boot=1"]) 2371 .args(["--memory", "size=512M"]) 2372 .args(["--kernel", fw_path.as_str()]) 2373 .default_disks() 2374 .default_net() 2375 .args(["--serial", "tty", "--console", "off"]) 2376 .args(["--event-monitor", format!("path={event_path}").as_str()]) 2377 .capture_output() 2378 .spawn() 2379 .unwrap(); 2380 2381 let r = std::panic::catch_unwind(|| { 2382 guest.wait_vm_boot(Some(120)).unwrap(); 2383 2384 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2385 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2386 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2387 2388 let expected_sequential_events = [ 2389 &MetaEvent { 2390 event: "starting".to_string(), 2391 device_id: None, 2392 }, 2393 &MetaEvent { 2394 event: "booting".to_string(), 2395 device_id: None, 2396 }, 2397 &MetaEvent { 2398 event: "booted".to_string(), 2399 device_id: None, 2400 }, 2401 &MetaEvent { 2402 event: "activated".to_string(), 2403 device_id: Some("_disk0".to_string()), 2404 }, 2405 &MetaEvent { 2406 event: "reset".to_string(), 2407 device_id: Some("_disk0".to_string()), 2408 }, 2409 ]; 2410 assert!(check_sequential_events( 2411 &expected_sequential_events, 2412 &event_path 2413 )); 2414 2415 // It's been observed on the Bionic image that udev and snapd 2416 // services can cause some delay in the VM's shutdown. Disabling 2417 // them improves the reliability of this test. 2418 let _ = guest.ssh_command("sudo systemctl disable udev"); 2419 let _ = guest.ssh_command("sudo systemctl stop udev"); 2420 let _ = guest.ssh_command("sudo systemctl disable snapd"); 2421 let _ = guest.ssh_command("sudo systemctl stop snapd"); 2422 2423 guest.ssh_command("sudo poweroff").unwrap(); 2424 thread::sleep(std::time::Duration::new(20, 0)); 2425 let latest_events = [ 2426 &MetaEvent { 2427 event: "shutdown".to_string(), 2428 device_id: None, 2429 }, 2430 &MetaEvent { 2431 event: "deleted".to_string(), 2432 device_id: None, 2433 }, 2434 &MetaEvent { 2435 event: "shutdown".to_string(), 2436 device_id: None, 2437 }, 2438 ]; 2439 assert!(check_latest_events_exact(&latest_events, &event_path)); 2440 }); 2441 2442 kill_child(&mut child); 2443 let output = child.wait_with_output().unwrap(); 2444 2445 handle_child_output(r, &output); 2446 } 2447 2448 #[test] 2449 fn test_multi_cpu() { 2450 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2451 let jammy = UbuntuDiskConfig::new(jammy_image); 2452 let guest = Guest::new(Box::new(jammy)); 2453 2454 let mut cmd = GuestCommand::new(&guest); 2455 cmd.args(["--cpus", "boot=2,max=4"]) 2456 .args(["--memory", "size=512M"]) 2457 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2458 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2459 .capture_output() 2460 .default_disks() 2461 .default_net(); 2462 2463 let mut child = cmd.spawn().unwrap(); 2464 2465 let r = std::panic::catch_unwind(|| { 2466 guest.wait_vm_boot(Some(120)).unwrap(); 2467 2468 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2469 2470 assert_eq!( 2471 guest 2472 .ssh_command( 2473 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2474 ) 2475 .unwrap() 2476 .trim(), 2477 "smp: Brought up 1 node, 2 CPUs" 2478 ); 2479 }); 2480 2481 kill_child(&mut child); 2482 let output = child.wait_with_output().unwrap(); 2483 2484 handle_child_output(r, &output); 2485 } 2486 2487 #[test] 2488 fn test_cpu_topology_421() { 2489 test_cpu_topology(4, 2, 1, false); 2490 } 2491 2492 #[test] 2493 fn test_cpu_topology_142() { 2494 test_cpu_topology(1, 4, 2, false); 2495 } 2496 2497 #[test] 2498 fn test_cpu_topology_262() { 2499 test_cpu_topology(2, 6, 2, false); 2500 } 2501 2502 #[test] 2503 #[cfg(target_arch = "x86_64")] 2504 #[cfg(not(feature = "mshv"))] 2505 fn test_cpu_physical_bits() { 2506 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2507 let guest = Guest::new(Box::new(focal)); 2508 let max_phys_bits: u8 = 36; 2509 let mut child = GuestCommand::new(&guest) 2510 .args(["--cpus", &format!("max_phys_bits={max_phys_bits}")]) 2511 .args(["--memory", "size=512M"]) 2512 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2513 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2514 .default_disks() 2515 .default_net() 2516 .capture_output() 2517 .spawn() 2518 .unwrap(); 2519 2520 let r = std::panic::catch_unwind(|| { 2521 guest.wait_vm_boot(None).unwrap(); 2522 2523 assert!( 2524 guest 2525 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2526 .unwrap() 2527 .trim() 2528 .parse::<u8>() 2529 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2530 ); 2531 }); 2532 2533 kill_child(&mut child); 2534 let output = child.wait_with_output().unwrap(); 2535 2536 handle_child_output(r, &output); 2537 } 2538 2539 #[test] 2540 fn test_cpu_affinity() { 2541 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2542 let guest = Guest::new(Box::new(focal)); 2543 2544 // We need the host to have at least 4 CPUs if we want to be able 2545 // to run this test. 2546 let host_cpus_count = exec_host_command_output("nproc"); 2547 assert!( 2548 String::from_utf8_lossy(&host_cpus_count.stdout) 2549 .trim() 2550 .parse::<u16>() 2551 .unwrap_or(0) 2552 >= 4 2553 ); 2554 2555 let mut child = GuestCommand::new(&guest) 2556 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2557 .args(["--memory", "size=512M"]) 2558 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2559 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2560 .default_disks() 2561 .default_net() 2562 .capture_output() 2563 .spawn() 2564 .unwrap(); 2565 2566 let r = std::panic::catch_unwind(|| { 2567 guest.wait_vm_boot(None).unwrap(); 2568 let pid = child.id(); 2569 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2570 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2571 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2572 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2573 }); 2574 2575 kill_child(&mut child); 2576 let output = child.wait_with_output().unwrap(); 2577 handle_child_output(r, &output); 2578 } 2579 2580 #[test] 2581 fn test_virtio_queue_affinity() { 2582 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2583 let guest = Guest::new(Box::new(focal)); 2584 2585 // We need the host to have at least 4 CPUs if we want to be able 2586 // to run this test. 2587 let host_cpus_count = exec_host_command_output("nproc"); 2588 assert!( 2589 String::from_utf8_lossy(&host_cpus_count.stdout) 2590 .trim() 2591 .parse::<u16>() 2592 .unwrap_or(0) 2593 >= 4 2594 ); 2595 2596 let mut child = GuestCommand::new(&guest) 2597 .args(["--cpus", "boot=4"]) 2598 .args(["--memory", "size=512M"]) 2599 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2600 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2601 .args([ 2602 "--disk", 2603 format!( 2604 "path={}", 2605 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2606 ) 2607 .as_str(), 2608 format!( 2609 "path={},num_queues=4,queue_affinity=[0@[0,2],1@[1,3],2@[1],3@[3]]", 2610 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2611 ) 2612 .as_str(), 2613 ]) 2614 .default_net() 2615 .capture_output() 2616 .spawn() 2617 .unwrap(); 2618 2619 let r = std::panic::catch_unwind(|| { 2620 guest.wait_vm_boot(None).unwrap(); 2621 let pid = child.id(); 2622 let taskset_q0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2623 assert_eq!(String::from_utf8_lossy(&taskset_q0.stdout).trim(), "0,2"); 2624 let taskset_q1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2625 assert_eq!(String::from_utf8_lossy(&taskset_q1.stdout).trim(), "1,3"); 2626 let taskset_q2 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q2 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2627 assert_eq!(String::from_utf8_lossy(&taskset_q2.stdout).trim(), "1"); 2628 let taskset_q3 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q3 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2629 assert_eq!(String::from_utf8_lossy(&taskset_q3.stdout).trim(), "3"); 2630 }); 2631 2632 kill_child(&mut child); 2633 let output = child.wait_with_output().unwrap(); 2634 handle_child_output(r, &output); 2635 } 2636 2637 #[test] 2638 #[cfg(not(feature = "mshv"))] 2639 fn test_large_vm() { 2640 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2641 let guest = Guest::new(Box::new(focal)); 2642 let mut cmd = GuestCommand::new(&guest); 2643 cmd.args(["--cpus", "boot=48"]) 2644 .args(["--memory", "size=5120M"]) 2645 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2646 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2647 .args(["--serial", "tty"]) 2648 .args(["--console", "off"]) 2649 .capture_output() 2650 .default_disks() 2651 .default_net(); 2652 2653 let mut child = cmd.spawn().unwrap(); 2654 2655 guest.wait_vm_boot(None).unwrap(); 2656 2657 let r = std::panic::catch_unwind(|| { 2658 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2659 assert_eq!( 2660 guest 2661 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2662 .unwrap() 2663 .trim(), 2664 "0-47" 2665 ); 2666 2667 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2668 }); 2669 2670 kill_child(&mut child); 2671 let output = child.wait_with_output().unwrap(); 2672 2673 handle_child_output(r, &output); 2674 } 2675 2676 #[test] 2677 #[cfg(not(feature = "mshv"))] 2678 fn test_huge_memory() { 2679 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2680 let guest = Guest::new(Box::new(focal)); 2681 let mut cmd = GuestCommand::new(&guest); 2682 cmd.args(["--cpus", "boot=1"]) 2683 .args(["--memory", "size=128G"]) 2684 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2685 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2686 .capture_output() 2687 .default_disks() 2688 .default_net(); 2689 2690 let mut child = cmd.spawn().unwrap(); 2691 2692 guest.wait_vm_boot(Some(120)).unwrap(); 2693 2694 let r = std::panic::catch_unwind(|| { 2695 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2696 }); 2697 2698 kill_child(&mut child); 2699 let output = child.wait_with_output().unwrap(); 2700 2701 handle_child_output(r, &output); 2702 } 2703 2704 #[test] 2705 fn test_power_button() { 2706 _test_power_button(false); 2707 } 2708 2709 #[test] 2710 #[cfg(not(feature = "mshv"))] 2711 fn test_user_defined_memory_regions() { 2712 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2713 let guest = Guest::new(Box::new(focal)); 2714 let api_socket = temp_api_path(&guest.tmp_dir); 2715 2716 let kernel_path = direct_kernel_boot_path(); 2717 2718 let mut child = GuestCommand::new(&guest) 2719 .args(["--cpus", "boot=1"]) 2720 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2721 .args([ 2722 "--memory-zone", 2723 "id=mem0,size=1G,hotplug_size=2G", 2724 "id=mem1,size=1G,shared=on", 2725 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2726 ]) 2727 .args(["--kernel", kernel_path.to_str().unwrap()]) 2728 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2729 .args(["--api-socket", &api_socket]) 2730 .capture_output() 2731 .default_disks() 2732 .default_net() 2733 .spawn() 2734 .unwrap(); 2735 2736 let r = std::panic::catch_unwind(|| { 2737 guest.wait_vm_boot(None).unwrap(); 2738 2739 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2740 2741 guest.enable_memory_hotplug(); 2742 2743 resize_zone_command(&api_socket, "mem0", "3G"); 2744 thread::sleep(std::time::Duration::new(5, 0)); 2745 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2746 resize_zone_command(&api_socket, "mem2", "3G"); 2747 thread::sleep(std::time::Duration::new(5, 0)); 2748 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2749 resize_zone_command(&api_socket, "mem0", "2G"); 2750 thread::sleep(std::time::Duration::new(5, 0)); 2751 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2752 resize_zone_command(&api_socket, "mem2", "2G"); 2753 thread::sleep(std::time::Duration::new(5, 0)); 2754 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2755 2756 guest.reboot_linux(0, None); 2757 2758 // Check the amount of RAM after reboot 2759 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2760 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2761 2762 // Check if we can still resize down to the initial 'boot'size 2763 resize_zone_command(&api_socket, "mem0", "1G"); 2764 thread::sleep(std::time::Duration::new(5, 0)); 2765 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2766 resize_zone_command(&api_socket, "mem2", "1G"); 2767 thread::sleep(std::time::Duration::new(5, 0)); 2768 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2769 }); 2770 2771 kill_child(&mut child); 2772 let output = child.wait_with_output().unwrap(); 2773 2774 handle_child_output(r, &output); 2775 } 2776 2777 #[test] 2778 #[cfg(not(feature = "mshv"))] 2779 fn test_guest_numa_nodes() { 2780 _test_guest_numa_nodes(false); 2781 } 2782 2783 #[test] 2784 #[cfg(target_arch = "x86_64")] 2785 fn test_iommu_segments() { 2786 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2787 let focal = UbuntuDiskConfig::new(focal_image); 2788 let guest = Guest::new(Box::new(focal)); 2789 2790 // Prepare another disk file for the virtio-disk device 2791 let test_disk_path = String::from( 2792 guest 2793 .tmp_dir 2794 .as_path() 2795 .join("test-disk.raw") 2796 .to_str() 2797 .unwrap(), 2798 ); 2799 assert!( 2800 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2801 ); 2802 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2803 2804 let api_socket = temp_api_path(&guest.tmp_dir); 2805 let mut cmd = GuestCommand::new(&guest); 2806 2807 cmd.args(["--cpus", "boot=1"]) 2808 .args(["--api-socket", &api_socket]) 2809 .args(["--memory", "size=512M"]) 2810 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2811 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2812 .args([ 2813 "--platform", 2814 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS},iommu_segments=[1]"), 2815 ]) 2816 .default_disks() 2817 .capture_output() 2818 .default_net(); 2819 2820 let mut child = cmd.spawn().unwrap(); 2821 2822 guest.wait_vm_boot(None).unwrap(); 2823 2824 let r = std::panic::catch_unwind(|| { 2825 let (cmd_success, cmd_output) = remote_command_w_output( 2826 &api_socket, 2827 "add-disk", 2828 Some( 2829 format!( 2830 "path={},id=test0,pci_segment=1,iommu=on", 2831 test_disk_path.as_str() 2832 ) 2833 .as_str(), 2834 ), 2835 ); 2836 assert!(cmd_success); 2837 assert!(String::from_utf8_lossy(&cmd_output) 2838 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2839 2840 // Check IOMMU setup 2841 assert!(guest 2842 .does_device_vendor_pair_match("0x1057", "0x1af4") 2843 .unwrap_or_default()); 2844 assert_eq!( 2845 guest 2846 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2847 .unwrap() 2848 .trim(), 2849 "0001:00:01.0" 2850 ); 2851 }); 2852 2853 kill_child(&mut child); 2854 let output = child.wait_with_output().unwrap(); 2855 2856 handle_child_output(r, &output); 2857 } 2858 2859 #[test] 2860 fn test_pci_msi() { 2861 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2862 let guest = Guest::new(Box::new(focal)); 2863 let mut cmd = GuestCommand::new(&guest); 2864 cmd.args(["--cpus", "boot=1"]) 2865 .args(["--memory", "size=512M"]) 2866 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2867 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2868 .capture_output() 2869 .default_disks() 2870 .default_net(); 2871 2872 let mut child = cmd.spawn().unwrap(); 2873 2874 guest.wait_vm_boot(None).unwrap(); 2875 2876 #[cfg(target_arch = "x86_64")] 2877 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2878 #[cfg(target_arch = "aarch64")] 2879 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2880 2881 let r = std::panic::catch_unwind(|| { 2882 assert_eq!( 2883 guest 2884 .ssh_command(grep_cmd) 2885 .unwrap() 2886 .trim() 2887 .parse::<u32>() 2888 .unwrap_or_default(), 2889 12 2890 ); 2891 }); 2892 2893 kill_child(&mut child); 2894 let output = child.wait_with_output().unwrap(); 2895 2896 handle_child_output(r, &output); 2897 } 2898 2899 #[test] 2900 fn test_virtio_net_ctrl_queue() { 2901 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2902 let guest = Guest::new(Box::new(focal)); 2903 let mut cmd = GuestCommand::new(&guest); 2904 cmd.args(["--cpus", "boot=1"]) 2905 .args(["--memory", "size=512M"]) 2906 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2907 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2908 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2909 .capture_output() 2910 .default_disks(); 2911 2912 let mut child = cmd.spawn().unwrap(); 2913 2914 guest.wait_vm_boot(None).unwrap(); 2915 2916 #[cfg(target_arch = "aarch64")] 2917 let iface = "enp0s4"; 2918 #[cfg(target_arch = "x86_64")] 2919 let iface = "ens4"; 2920 2921 let r = std::panic::catch_unwind(|| { 2922 assert_eq!( 2923 guest 2924 .ssh_command( 2925 format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() 2926 ) 2927 .unwrap() 2928 .trim(), 2929 "success" 2930 ); 2931 assert_eq!( 2932 guest 2933 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 2934 .unwrap() 2935 .trim(), 2936 "3000" 2937 ); 2938 }); 2939 2940 kill_child(&mut child); 2941 let output = child.wait_with_output().unwrap(); 2942 2943 handle_child_output(r, &output); 2944 } 2945 2946 #[test] 2947 fn test_pci_multiple_segments() { 2948 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2949 let guest = Guest::new(Box::new(focal)); 2950 2951 // Prepare another disk file for the virtio-disk device 2952 let test_disk_path = String::from( 2953 guest 2954 .tmp_dir 2955 .as_path() 2956 .join("test-disk.raw") 2957 .to_str() 2958 .unwrap(), 2959 ); 2960 assert!( 2961 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2962 ); 2963 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2964 2965 let mut cmd = GuestCommand::new(&guest); 2966 cmd.args(["--cpus", "boot=1"]) 2967 .args(["--memory", "size=512M"]) 2968 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2969 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2970 .args([ 2971 "--platform", 2972 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 2973 ]) 2974 .args([ 2975 "--disk", 2976 format!( 2977 "path={}", 2978 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2979 ) 2980 .as_str(), 2981 format!( 2982 "path={}", 2983 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2984 ) 2985 .as_str(), 2986 format!("path={test_disk_path},pci_segment=15").as_str(), 2987 ]) 2988 .capture_output() 2989 .default_net(); 2990 2991 let mut child = cmd.spawn().unwrap(); 2992 2993 guest.wait_vm_boot(None).unwrap(); 2994 2995 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2996 2997 let r = std::panic::catch_unwind(|| { 2998 // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. 2999 assert_eq!( 3000 guest 3001 .ssh_command(grep_cmd) 3002 .unwrap() 3003 .trim() 3004 .parse::<u16>() 3005 .unwrap_or_default(), 3006 MAX_NUM_PCI_SEGMENTS 3007 ); 3008 3009 // Check both if /dev/vdc exists and if the block size is 4M. 3010 assert_eq!( 3011 guest 3012 .ssh_command("lsblk | grep vdc | grep -c 4M") 3013 .unwrap() 3014 .trim() 3015 .parse::<u32>() 3016 .unwrap_or_default(), 3017 1 3018 ); 3019 3020 // Mount the device. 3021 guest.ssh_command("mkdir mount_image").unwrap(); 3022 guest 3023 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 3024 .unwrap(); 3025 // Grant all users with write permission. 3026 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 3027 3028 // Write something to the device. 3029 guest 3030 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 3031 .unwrap(); 3032 3033 // Check the content of the block device. The file "foo" should 3034 // contain "bar". 3035 assert_eq!( 3036 guest 3037 .ssh_command("sudo cat mount_image/foo") 3038 .unwrap() 3039 .trim(), 3040 "bar" 3041 ); 3042 }); 3043 3044 kill_child(&mut child); 3045 let output = child.wait_with_output().unwrap(); 3046 3047 handle_child_output(r, &output); 3048 } 3049 3050 #[test] 3051 fn test_pci_multiple_segments_numa_node() { 3052 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3053 let guest = Guest::new(Box::new(focal)); 3054 let api_socket = temp_api_path(&guest.tmp_dir); 3055 #[cfg(target_arch = "x86_64")] 3056 let kernel_path = direct_kernel_boot_path(); 3057 #[cfg(target_arch = "aarch64")] 3058 let kernel_path = edk2_path(); 3059 3060 // Prepare another disk file for the virtio-disk device 3061 let test_disk_path = String::from( 3062 guest 3063 .tmp_dir 3064 .as_path() 3065 .join("test-disk.raw") 3066 .to_str() 3067 .unwrap(), 3068 ); 3069 assert!( 3070 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 3071 ); 3072 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 3073 const TEST_DISK_NODE: u16 = 1; 3074 3075 let mut child = GuestCommand::new(&guest) 3076 .args(["--platform", "num_pci_segments=2"]) 3077 .args(["--cpus", "boot=2"]) 3078 .args(["--memory", "size=0"]) 3079 .args(["--memory-zone", "id=mem0,size=256M", "id=mem1,size=256M"]) 3080 .args([ 3081 "--numa", 3082 "guest_numa_id=0,cpus=[0],distances=[1@20],memory_zones=mem0,pci_segments=[0]", 3083 "guest_numa_id=1,cpus=[1],distances=[0@20],memory_zones=mem1,pci_segments=[1]", 3084 ]) 3085 .args(["--kernel", kernel_path.to_str().unwrap()]) 3086 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3087 .args(["--api-socket", &api_socket]) 3088 .capture_output() 3089 .args([ 3090 "--disk", 3091 format!( 3092 "path={}", 3093 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3094 ) 3095 .as_str(), 3096 format!( 3097 "path={}", 3098 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3099 ) 3100 .as_str(), 3101 format!("path={test_disk_path},pci_segment={TEST_DISK_NODE}").as_str(), 3102 ]) 3103 .default_net() 3104 .spawn() 3105 .unwrap(); 3106 3107 let cmd = "cat /sys/block/vdc/device/../numa_node"; 3108 3109 let r = std::panic::catch_unwind(|| { 3110 guest.wait_vm_boot(None).unwrap(); 3111 3112 assert_eq!( 3113 guest 3114 .ssh_command(cmd) 3115 .unwrap() 3116 .trim() 3117 .parse::<u16>() 3118 .unwrap_or_default(), 3119 TEST_DISK_NODE 3120 ); 3121 }); 3122 3123 kill_child(&mut child); 3124 let output = child.wait_with_output().unwrap(); 3125 3126 handle_child_output(r, &output); 3127 } 3128 3129 #[test] 3130 fn test_direct_kernel_boot() { 3131 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3132 let guest = Guest::new(Box::new(focal)); 3133 3134 let kernel_path = direct_kernel_boot_path(); 3135 3136 let mut child = GuestCommand::new(&guest) 3137 .args(["--cpus", "boot=1"]) 3138 .args(["--memory", "size=512M"]) 3139 .args(["--kernel", kernel_path.to_str().unwrap()]) 3140 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3141 .default_disks() 3142 .default_net() 3143 .capture_output() 3144 .spawn() 3145 .unwrap(); 3146 3147 let r = std::panic::catch_unwind(|| { 3148 guest.wait_vm_boot(None).unwrap(); 3149 3150 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3151 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3152 3153 let grep_cmd = if cfg!(target_arch = "x86_64") { 3154 "grep -c PCI-MSI /proc/interrupts" 3155 } else { 3156 "grep -c ITS-MSI /proc/interrupts" 3157 }; 3158 assert_eq!( 3159 guest 3160 .ssh_command(grep_cmd) 3161 .unwrap() 3162 .trim() 3163 .parse::<u32>() 3164 .unwrap_or_default(), 3165 12 3166 ); 3167 }); 3168 3169 kill_child(&mut child); 3170 let output = child.wait_with_output().unwrap(); 3171 3172 handle_child_output(r, &output); 3173 } 3174 3175 #[test] 3176 #[cfg(target_arch = "x86_64")] 3177 fn test_direct_kernel_boot_bzimage() { 3178 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3179 let guest = Guest::new(Box::new(focal)); 3180 3181 let mut kernel_path = direct_kernel_boot_path(); 3182 // Replace the default kernel with the bzImage. 3183 kernel_path.pop(); 3184 kernel_path.push("bzImage"); 3185 3186 let mut child = GuestCommand::new(&guest) 3187 .args(["--cpus", "boot=1"]) 3188 .args(["--memory", "size=512M"]) 3189 .args(["--kernel", kernel_path.to_str().unwrap()]) 3190 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3191 .default_disks() 3192 .default_net() 3193 .capture_output() 3194 .spawn() 3195 .unwrap(); 3196 3197 let r = std::panic::catch_unwind(|| { 3198 guest.wait_vm_boot(None).unwrap(); 3199 3200 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3201 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3202 3203 let grep_cmd = if cfg!(target_arch = "x86_64") { 3204 "grep -c PCI-MSI /proc/interrupts" 3205 } else { 3206 "grep -c ITS-MSI /proc/interrupts" 3207 }; 3208 assert_eq!( 3209 guest 3210 .ssh_command(grep_cmd) 3211 .unwrap() 3212 .trim() 3213 .parse::<u32>() 3214 .unwrap_or_default(), 3215 12 3216 ); 3217 }); 3218 3219 kill_child(&mut child); 3220 let output = child.wait_with_output().unwrap(); 3221 3222 handle_child_output(r, &output); 3223 } 3224 3225 fn _test_virtio_block(image_name: &str, disable_io_uring: bool, disable_aio: bool) { 3226 let focal = UbuntuDiskConfig::new(image_name.to_string()); 3227 let guest = Guest::new(Box::new(focal)); 3228 3229 let mut workload_path = dirs::home_dir().unwrap(); 3230 workload_path.push("workloads"); 3231 3232 let mut blk_file_path = workload_path; 3233 blk_file_path.push("blk.img"); 3234 3235 let kernel_path = direct_kernel_boot_path(); 3236 3237 let mut cloud_child = GuestCommand::new(&guest) 3238 .args(["--cpus", "boot=4"]) 3239 .args(["--memory", "size=512M,shared=on"]) 3240 .args(["--kernel", kernel_path.to_str().unwrap()]) 3241 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3242 .args([ 3243 "--disk", 3244 format!( 3245 "path={}", 3246 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3247 ) 3248 .as_str(), 3249 format!( 3250 "path={}", 3251 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3252 ) 3253 .as_str(), 3254 format!( 3255 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", 3256 blk_file_path.to_str().unwrap(), 3257 disable_io_uring, 3258 disable_aio, 3259 ) 3260 .as_str(), 3261 ]) 3262 .default_net() 3263 .capture_output() 3264 .spawn() 3265 .unwrap(); 3266 3267 let r = std::panic::catch_unwind(|| { 3268 guest.wait_vm_boot(None).unwrap(); 3269 3270 // Check both if /dev/vdc exists and if the block size is 16M. 3271 assert_eq!( 3272 guest 3273 .ssh_command("lsblk | grep vdc | grep -c 16M") 3274 .unwrap() 3275 .trim() 3276 .parse::<u32>() 3277 .unwrap_or_default(), 3278 1 3279 ); 3280 3281 // Check both if /dev/vdc exists and if this block is RO. 3282 assert_eq!( 3283 guest 3284 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 3285 .unwrap() 3286 .trim() 3287 .parse::<u32>() 3288 .unwrap_or_default(), 3289 1 3290 ); 3291 3292 // Check if the number of queues is 4. 3293 assert_eq!( 3294 guest 3295 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 3296 .unwrap() 3297 .trim() 3298 .parse::<u32>() 3299 .unwrap_or_default(), 3300 4 3301 ); 3302 }); 3303 3304 let _ = cloud_child.kill(); 3305 let output = cloud_child.wait_with_output().unwrap(); 3306 3307 handle_child_output(r, &output); 3308 } 3309 3310 #[test] 3311 fn test_virtio_block_io_uring() { 3312 _test_virtio_block(FOCAL_IMAGE_NAME, false, true) 3313 } 3314 3315 #[test] 3316 fn test_virtio_block_aio() { 3317 _test_virtio_block(FOCAL_IMAGE_NAME, true, false) 3318 } 3319 3320 #[test] 3321 fn test_virtio_block_sync() { 3322 _test_virtio_block(FOCAL_IMAGE_NAME, true, true) 3323 } 3324 3325 #[test] 3326 fn test_virtio_block_qcow2() { 3327 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false, false) 3328 } 3329 3330 #[test] 3331 fn test_virtio_block_qcow2_backing_file() { 3332 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE, false, false) 3333 } 3334 3335 #[test] 3336 fn test_virtio_block_vhd() { 3337 let mut workload_path = dirs::home_dir().unwrap(); 3338 workload_path.push("workloads"); 3339 3340 let mut raw_file_path = workload_path.clone(); 3341 let mut vhd_file_path = workload_path; 3342 raw_file_path.push(FOCAL_IMAGE_NAME); 3343 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 3344 3345 // Generate VHD file from RAW file 3346 std::process::Command::new("qemu-img") 3347 .arg("convert") 3348 .arg("-p") 3349 .args(["-f", "raw"]) 3350 .args(["-O", "vpc"]) 3351 .args(["-o", "subformat=fixed"]) 3352 .arg(raw_file_path.to_str().unwrap()) 3353 .arg(vhd_file_path.to_str().unwrap()) 3354 .output() 3355 .expect("Expect generating VHD image from RAW image"); 3356 3357 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false, false) 3358 } 3359 3360 #[test] 3361 fn test_virtio_block_vhdx() { 3362 let mut workload_path = dirs::home_dir().unwrap(); 3363 workload_path.push("workloads"); 3364 3365 let mut raw_file_path = workload_path.clone(); 3366 let mut vhdx_file_path = workload_path; 3367 raw_file_path.push(FOCAL_IMAGE_NAME); 3368 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 3369 3370 // Generate dynamic VHDX file from RAW file 3371 std::process::Command::new("qemu-img") 3372 .arg("convert") 3373 .arg("-p") 3374 .args(["-f", "raw"]) 3375 .args(["-O", "vhdx"]) 3376 .arg(raw_file_path.to_str().unwrap()) 3377 .arg(vhdx_file_path.to_str().unwrap()) 3378 .output() 3379 .expect("Expect generating dynamic VHDx image from RAW image"); 3380 3381 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false, false) 3382 } 3383 3384 #[test] 3385 fn test_virtio_block_dynamic_vhdx_expand() { 3386 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 3387 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 3388 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 3389 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 3390 3391 let mut workload_path = dirs::home_dir().unwrap(); 3392 workload_path.push("workloads"); 3393 3394 let mut vhdx_file_path = workload_path; 3395 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 3396 let vhdx_path = vhdx_file_path.to_str().unwrap(); 3397 3398 // Generate a 100 MiB dynamic VHDX file 3399 std::process::Command::new("qemu-img") 3400 .arg("create") 3401 .args(["-f", "vhdx"]) 3402 .arg(vhdx_path) 3403 .arg(VIRTUAL_DISK_SIZE.to_string()) 3404 .output() 3405 .expect("Expect generating dynamic VHDx image from RAW image"); 3406 3407 // Check if the size matches with empty VHDx file size 3408 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 3409 3410 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3411 let guest = Guest::new(Box::new(focal)); 3412 let kernel_path = direct_kernel_boot_path(); 3413 3414 let mut cloud_child = GuestCommand::new(&guest) 3415 .args(["--cpus", "boot=1"]) 3416 .args(["--memory", "size=512M"]) 3417 .args(["--kernel", kernel_path.to_str().unwrap()]) 3418 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3419 .args([ 3420 "--disk", 3421 format!( 3422 "path={}", 3423 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3424 ) 3425 .as_str(), 3426 format!( 3427 "path={}", 3428 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3429 ) 3430 .as_str(), 3431 format!("path={vhdx_path}").as_str(), 3432 ]) 3433 .default_net() 3434 .capture_output() 3435 .spawn() 3436 .unwrap(); 3437 3438 let r = std::panic::catch_unwind(|| { 3439 guest.wait_vm_boot(None).unwrap(); 3440 3441 // Check both if /dev/vdc exists and if the block size is 100 MiB. 3442 assert_eq!( 3443 guest 3444 .ssh_command("lsblk | grep vdc | grep -c 100M") 3445 .unwrap() 3446 .trim() 3447 .parse::<u32>() 3448 .unwrap_or_default(), 3449 1 3450 ); 3451 3452 // Write 100 MB of data to the VHDx disk 3453 guest 3454 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 3455 .unwrap(); 3456 }); 3457 3458 // Check if the size matches with expected expanded VHDx file size 3459 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 3460 3461 kill_child(&mut cloud_child); 3462 let output = cloud_child.wait_with_output().unwrap(); 3463 3464 handle_child_output(r, &output); 3465 } 3466 3467 fn vhdx_image_size(disk_name: &str) -> u64 { 3468 std::fs::File::open(disk_name) 3469 .unwrap() 3470 .seek(SeekFrom::End(0)) 3471 .unwrap() 3472 } 3473 3474 #[test] 3475 fn test_virtio_block_direct_and_firmware() { 3476 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3477 let guest = Guest::new(Box::new(focal)); 3478 3479 // The OS disk must be copied to a location that is not backed by 3480 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 3481 // with EINVAL because tmpfs doesn't support this flag. 3482 let mut workloads_path = dirs::home_dir().unwrap(); 3483 workloads_path.push("workloads"); 3484 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 3485 let mut os_path = os_dir.as_path().to_path_buf(); 3486 os_path.push("osdisk.img"); 3487 rate_limited_copy( 3488 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3489 os_path.as_path(), 3490 ) 3491 .expect("copying of OS disk failed"); 3492 3493 let mut child = GuestCommand::new(&guest) 3494 .args(["--cpus", "boot=1"]) 3495 .args(["--memory", "size=512M"]) 3496 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 3497 .args([ 3498 "--disk", 3499 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 3500 format!( 3501 "path={}", 3502 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3503 ) 3504 .as_str(), 3505 ]) 3506 .default_net() 3507 .capture_output() 3508 .spawn() 3509 .unwrap(); 3510 3511 let r = std::panic::catch_unwind(|| { 3512 guest.wait_vm_boot(Some(120)).unwrap(); 3513 }); 3514 3515 kill_child(&mut child); 3516 let output = child.wait_with_output().unwrap(); 3517 3518 handle_child_output(r, &output); 3519 } 3520 3521 #[test] 3522 fn test_vhost_user_net_default() { 3523 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 3524 } 3525 3526 #[test] 3527 fn test_vhost_user_net_named_tap() { 3528 test_vhost_user_net( 3529 Some("mytap0"), 3530 2, 3531 &prepare_vhost_user_net_daemon, 3532 false, 3533 false, 3534 ) 3535 } 3536 3537 #[test] 3538 fn test_vhost_user_net_existing_tap() { 3539 test_vhost_user_net( 3540 Some("vunet-tap0"), 3541 2, 3542 &prepare_vhost_user_net_daemon, 3543 false, 3544 false, 3545 ) 3546 } 3547 3548 #[test] 3549 fn test_vhost_user_net_multiple_queues() { 3550 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 3551 } 3552 3553 #[test] 3554 fn test_vhost_user_net_tap_multiple_queues() { 3555 test_vhost_user_net( 3556 Some("vunet-tap1"), 3557 4, 3558 &prepare_vhost_user_net_daemon, 3559 false, 3560 false, 3561 ) 3562 } 3563 3564 #[test] 3565 fn test_vhost_user_net_host_mac() { 3566 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 3567 } 3568 3569 #[test] 3570 fn test_vhost_user_net_client_mode() { 3571 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 3572 } 3573 3574 #[test] 3575 #[cfg(not(target_arch = "aarch64"))] 3576 fn test_vhost_user_blk_default() { 3577 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 3578 } 3579 3580 #[test] 3581 #[cfg(not(target_arch = "aarch64"))] 3582 fn test_vhost_user_blk_readonly() { 3583 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3584 } 3585 3586 #[test] 3587 #[cfg(not(target_arch = "aarch64"))] 3588 fn test_vhost_user_blk_direct() { 3589 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3590 } 3591 3592 #[test] 3593 fn test_boot_from_vhost_user_blk_default() { 3594 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3595 } 3596 3597 #[test] 3598 #[cfg(target_arch = "x86_64")] 3599 fn test_split_irqchip() { 3600 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3601 let guest = Guest::new(Box::new(focal)); 3602 3603 let mut child = GuestCommand::new(&guest) 3604 .args(["--cpus", "boot=1"]) 3605 .args(["--memory", "size=512M"]) 3606 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3607 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3608 .default_disks() 3609 .default_net() 3610 .capture_output() 3611 .spawn() 3612 .unwrap(); 3613 3614 let r = std::panic::catch_unwind(|| { 3615 guest.wait_vm_boot(None).unwrap(); 3616 3617 assert_eq!( 3618 guest 3619 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3620 .unwrap() 3621 .trim() 3622 .parse::<u32>() 3623 .unwrap_or(1), 3624 0 3625 ); 3626 assert_eq!( 3627 guest 3628 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3629 .unwrap() 3630 .trim() 3631 .parse::<u32>() 3632 .unwrap_or(1), 3633 0 3634 ); 3635 }); 3636 3637 kill_child(&mut child); 3638 let output = child.wait_with_output().unwrap(); 3639 3640 handle_child_output(r, &output); 3641 } 3642 3643 #[test] 3644 #[cfg(target_arch = "x86_64")] 3645 fn test_dmi_serial_number() { 3646 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3647 let guest = Guest::new(Box::new(focal)); 3648 3649 let mut child = GuestCommand::new(&guest) 3650 .args(["--cpus", "boot=1"]) 3651 .args(["--memory", "size=512M"]) 3652 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3653 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3654 .args(["--platform", "serial_number=a=b;c=d"]) 3655 .default_disks() 3656 .default_net() 3657 .capture_output() 3658 .spawn() 3659 .unwrap(); 3660 3661 let r = std::panic::catch_unwind(|| { 3662 guest.wait_vm_boot(None).unwrap(); 3663 3664 assert_eq!( 3665 guest 3666 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3667 .unwrap() 3668 .trim(), 3669 "a=b;c=d" 3670 ); 3671 }); 3672 3673 kill_child(&mut child); 3674 let output = child.wait_with_output().unwrap(); 3675 3676 handle_child_output(r, &output); 3677 } 3678 3679 #[test] 3680 #[cfg(target_arch = "x86_64")] 3681 fn test_dmi_uuid() { 3682 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3683 let guest = Guest::new(Box::new(focal)); 3684 3685 let mut child = GuestCommand::new(&guest) 3686 .args(["--cpus", "boot=1"]) 3687 .args(["--memory", "size=512M"]) 3688 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3689 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3690 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3691 .default_disks() 3692 .default_net() 3693 .capture_output() 3694 .spawn() 3695 .unwrap(); 3696 3697 let r = std::panic::catch_unwind(|| { 3698 guest.wait_vm_boot(None).unwrap(); 3699 3700 assert_eq!( 3701 guest 3702 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3703 .unwrap() 3704 .trim(), 3705 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3706 ); 3707 }); 3708 3709 kill_child(&mut child); 3710 let output = child.wait_with_output().unwrap(); 3711 3712 handle_child_output(r, &output); 3713 } 3714 3715 #[test] 3716 #[cfg(target_arch = "x86_64")] 3717 fn test_dmi_oem_strings() { 3718 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3719 let guest = Guest::new(Box::new(focal)); 3720 3721 let s1 = "io.systemd.credential:xx=yy"; 3722 let s2 = "This is a test string"; 3723 3724 let oem_strings = format!("oem_strings=[{s1},{s2}]"); 3725 3726 let mut child = GuestCommand::new(&guest) 3727 .args(["--cpus", "boot=1"]) 3728 .args(["--memory", "size=512M"]) 3729 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3730 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3731 .args(["--platform", &oem_strings]) 3732 .default_disks() 3733 .default_net() 3734 .capture_output() 3735 .spawn() 3736 .unwrap(); 3737 3738 let r = std::panic::catch_unwind(|| { 3739 guest.wait_vm_boot(None).unwrap(); 3740 3741 assert_eq!( 3742 guest 3743 .ssh_command("sudo dmidecode --oem-string count") 3744 .unwrap() 3745 .trim(), 3746 "2" 3747 ); 3748 3749 assert_eq!( 3750 guest 3751 .ssh_command("sudo dmidecode --oem-string 1") 3752 .unwrap() 3753 .trim(), 3754 s1 3755 ); 3756 3757 assert_eq!( 3758 guest 3759 .ssh_command("sudo dmidecode --oem-string 2") 3760 .unwrap() 3761 .trim(), 3762 s2 3763 ); 3764 }); 3765 3766 kill_child(&mut child); 3767 let output = child.wait_with_output().unwrap(); 3768 3769 handle_child_output(r, &output); 3770 } 3771 3772 #[test] 3773 fn test_virtio_fs() { 3774 _test_virtio_fs(&prepare_virtiofsd, false, None) 3775 } 3776 3777 #[test] 3778 fn test_virtio_fs_hotplug() { 3779 _test_virtio_fs(&prepare_virtiofsd, true, None) 3780 } 3781 3782 #[test] 3783 #[cfg(not(feature = "mshv"))] 3784 fn test_virtio_fs_multi_segment_hotplug() { 3785 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3786 } 3787 3788 #[test] 3789 #[cfg(not(feature = "mshv"))] 3790 fn test_virtio_fs_multi_segment() { 3791 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3792 } 3793 3794 #[test] 3795 fn test_virtio_pmem_persist_writes() { 3796 test_virtio_pmem(false, false) 3797 } 3798 3799 #[test] 3800 fn test_virtio_pmem_discard_writes() { 3801 test_virtio_pmem(true, false) 3802 } 3803 3804 #[test] 3805 fn test_virtio_pmem_with_size() { 3806 test_virtio_pmem(true, true) 3807 } 3808 3809 #[test] 3810 fn test_boot_from_virtio_pmem() { 3811 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3812 let guest = Guest::new(Box::new(focal)); 3813 3814 let kernel_path = direct_kernel_boot_path(); 3815 3816 let mut child = GuestCommand::new(&guest) 3817 .args(["--cpus", "boot=1"]) 3818 .args(["--memory", "size=512M"]) 3819 .args(["--kernel", kernel_path.to_str().unwrap()]) 3820 .args([ 3821 "--disk", 3822 format!( 3823 "path={}", 3824 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3825 ) 3826 .as_str(), 3827 ]) 3828 .default_net() 3829 .args([ 3830 "--pmem", 3831 format!( 3832 "file={},size={}", 3833 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3834 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3835 .unwrap() 3836 .len() 3837 ) 3838 .as_str(), 3839 ]) 3840 .args([ 3841 "--cmdline", 3842 DIRECT_KERNEL_BOOT_CMDLINE 3843 .replace("vda1", "pmem0p1") 3844 .as_str(), 3845 ]) 3846 .capture_output() 3847 .spawn() 3848 .unwrap(); 3849 3850 let r = std::panic::catch_unwind(|| { 3851 guest.wait_vm_boot(None).unwrap(); 3852 3853 // Simple checks to validate the VM booted properly 3854 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3855 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3856 }); 3857 3858 kill_child(&mut child); 3859 let output = child.wait_with_output().unwrap(); 3860 3861 handle_child_output(r, &output); 3862 } 3863 3864 #[test] 3865 fn test_multiple_network_interfaces() { 3866 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3867 let guest = Guest::new(Box::new(focal)); 3868 3869 let kernel_path = direct_kernel_boot_path(); 3870 3871 let mut child = GuestCommand::new(&guest) 3872 .args(["--cpus", "boot=1"]) 3873 .args(["--memory", "size=512M"]) 3874 .args(["--kernel", kernel_path.to_str().unwrap()]) 3875 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3876 .default_disks() 3877 .args([ 3878 "--net", 3879 guest.default_net_string().as_str(), 3880 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3881 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3882 ]) 3883 .capture_output() 3884 .spawn() 3885 .unwrap(); 3886 3887 let r = std::panic::catch_unwind(|| { 3888 guest.wait_vm_boot(None).unwrap(); 3889 3890 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3891 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3892 3893 // 3 network interfaces + default localhost ==> 4 interfaces 3894 assert_eq!( 3895 guest 3896 .ssh_command("ip -o link | wc -l") 3897 .unwrap() 3898 .trim() 3899 .parse::<u32>() 3900 .unwrap_or_default(), 3901 4 3902 ); 3903 }); 3904 3905 kill_child(&mut child); 3906 let output = child.wait_with_output().unwrap(); 3907 3908 handle_child_output(r, &output); 3909 } 3910 3911 #[test] 3912 #[cfg(target_arch = "aarch64")] 3913 fn test_pmu_on() { 3914 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3915 let guest = Guest::new(Box::new(focal)); 3916 let mut child = GuestCommand::new(&guest) 3917 .args(["--cpus", "boot=1"]) 3918 .args(["--memory", "size=512M"]) 3919 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3920 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3921 .default_disks() 3922 .default_net() 3923 .capture_output() 3924 .spawn() 3925 .unwrap(); 3926 3927 let r = std::panic::catch_unwind(|| { 3928 guest.wait_vm_boot(None).unwrap(); 3929 3930 // Test that PMU exists. 3931 assert_eq!( 3932 guest 3933 .ssh_command(GREP_PMU_IRQ_CMD) 3934 .unwrap() 3935 .trim() 3936 .parse::<u32>() 3937 .unwrap_or_default(), 3938 1 3939 ); 3940 }); 3941 3942 kill_child(&mut child); 3943 let output = child.wait_with_output().unwrap(); 3944 3945 handle_child_output(r, &output); 3946 } 3947 3948 #[test] 3949 fn test_serial_off() { 3950 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3951 let guest = Guest::new(Box::new(focal)); 3952 let mut child = GuestCommand::new(&guest) 3953 .args(["--cpus", "boot=1"]) 3954 .args(["--memory", "size=512M"]) 3955 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3956 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3957 .default_disks() 3958 .default_net() 3959 .args(["--serial", "off"]) 3960 .capture_output() 3961 .spawn() 3962 .unwrap(); 3963 3964 let r = std::panic::catch_unwind(|| { 3965 guest.wait_vm_boot(None).unwrap(); 3966 3967 // Test that there is no ttyS0 3968 assert_eq!( 3969 guest 3970 .ssh_command(GREP_SERIAL_IRQ_CMD) 3971 .unwrap() 3972 .trim() 3973 .parse::<u32>() 3974 .unwrap_or(1), 3975 0 3976 ); 3977 }); 3978 3979 kill_child(&mut child); 3980 let output = child.wait_with_output().unwrap(); 3981 3982 handle_child_output(r, &output); 3983 } 3984 3985 #[test] 3986 fn test_serial_null() { 3987 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3988 let guest = Guest::new(Box::new(focal)); 3989 let mut cmd = GuestCommand::new(&guest); 3990 #[cfg(target_arch = "x86_64")] 3991 let console_str: &str = "console=ttyS0"; 3992 #[cfg(target_arch = "aarch64")] 3993 let console_str: &str = "console=ttyAMA0"; 3994 3995 cmd.args(["--cpus", "boot=1"]) 3996 .args(["--memory", "size=512M"]) 3997 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3998 .args([ 3999 "--cmdline", 4000 DIRECT_KERNEL_BOOT_CMDLINE 4001 .replace("console=hvc0 ", console_str) 4002 .as_str(), 4003 ]) 4004 .default_disks() 4005 .default_net() 4006 .args(["--serial", "null"]) 4007 .args(["--console", "off"]) 4008 .capture_output(); 4009 4010 let mut child = cmd.spawn().unwrap(); 4011 4012 let r = std::panic::catch_unwind(|| { 4013 guest.wait_vm_boot(None).unwrap(); 4014 4015 // Test that there is a ttyS0 4016 assert_eq!( 4017 guest 4018 .ssh_command(GREP_SERIAL_IRQ_CMD) 4019 .unwrap() 4020 .trim() 4021 .parse::<u32>() 4022 .unwrap_or_default(), 4023 1 4024 ); 4025 }); 4026 4027 kill_child(&mut child); 4028 let output = child.wait_with_output().unwrap(); 4029 handle_child_output(r, &output); 4030 4031 let r = std::panic::catch_unwind(|| { 4032 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4033 }); 4034 4035 handle_child_output(r, &output); 4036 } 4037 4038 #[test] 4039 fn test_serial_tty() { 4040 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4041 let guest = Guest::new(Box::new(focal)); 4042 4043 let kernel_path = direct_kernel_boot_path(); 4044 4045 #[cfg(target_arch = "x86_64")] 4046 let console_str: &str = "console=ttyS0"; 4047 #[cfg(target_arch = "aarch64")] 4048 let console_str: &str = "console=ttyAMA0"; 4049 4050 let mut child = GuestCommand::new(&guest) 4051 .args(["--cpus", "boot=1"]) 4052 .args(["--memory", "size=512M"]) 4053 .args(["--kernel", kernel_path.to_str().unwrap()]) 4054 .args([ 4055 "--cmdline", 4056 DIRECT_KERNEL_BOOT_CMDLINE 4057 .replace("console=hvc0 ", console_str) 4058 .as_str(), 4059 ]) 4060 .default_disks() 4061 .default_net() 4062 .args(["--serial", "tty"]) 4063 .args(["--console", "off"]) 4064 .capture_output() 4065 .spawn() 4066 .unwrap(); 4067 4068 let r = std::panic::catch_unwind(|| { 4069 guest.wait_vm_boot(None).unwrap(); 4070 4071 // Test that there is a ttyS0 4072 assert_eq!( 4073 guest 4074 .ssh_command(GREP_SERIAL_IRQ_CMD) 4075 .unwrap() 4076 .trim() 4077 .parse::<u32>() 4078 .unwrap_or_default(), 4079 1 4080 ); 4081 }); 4082 4083 // This sleep is needed to wait for the login prompt 4084 thread::sleep(std::time::Duration::new(2, 0)); 4085 4086 kill_child(&mut child); 4087 let output = child.wait_with_output().unwrap(); 4088 handle_child_output(r, &output); 4089 4090 let r = std::panic::catch_unwind(|| { 4091 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4092 }); 4093 4094 handle_child_output(r, &output); 4095 } 4096 4097 #[test] 4098 fn test_serial_file() { 4099 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4100 let guest = Guest::new(Box::new(focal)); 4101 4102 let serial_path = guest.tmp_dir.as_path().join("serial-output"); 4103 #[cfg(target_arch = "x86_64")] 4104 let console_str: &str = "console=ttyS0"; 4105 #[cfg(target_arch = "aarch64")] 4106 let console_str: &str = "console=ttyAMA0"; 4107 4108 let mut child = GuestCommand::new(&guest) 4109 .args(["--cpus", "boot=1"]) 4110 .args(["--memory", "size=512M"]) 4111 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4112 .args([ 4113 "--cmdline", 4114 DIRECT_KERNEL_BOOT_CMDLINE 4115 .replace("console=hvc0 ", console_str) 4116 .as_str(), 4117 ]) 4118 .default_disks() 4119 .default_net() 4120 .args([ 4121 "--serial", 4122 format!("file={}", serial_path.to_str().unwrap()).as_str(), 4123 ]) 4124 .capture_output() 4125 .spawn() 4126 .unwrap(); 4127 4128 let r = std::panic::catch_unwind(|| { 4129 guest.wait_vm_boot(None).unwrap(); 4130 4131 // Test that there is a ttyS0 4132 assert_eq!( 4133 guest 4134 .ssh_command(GREP_SERIAL_IRQ_CMD) 4135 .unwrap() 4136 .trim() 4137 .parse::<u32>() 4138 .unwrap_or_default(), 4139 1 4140 ); 4141 4142 guest.ssh_command("sudo shutdown -h now").unwrap(); 4143 }); 4144 4145 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4146 kill_child(&mut child); 4147 let output = child.wait_with_output().unwrap(); 4148 handle_child_output(r, &output); 4149 4150 let r = std::panic::catch_unwind(|| { 4151 // Check that the cloud-hypervisor binary actually terminated 4152 assert!(output.status.success()); 4153 4154 // Do this check after shutdown of the VM as an easy way to ensure 4155 // all writes are flushed to disk 4156 let mut f = std::fs::File::open(serial_path).unwrap(); 4157 let mut buf = String::new(); 4158 f.read_to_string(&mut buf).unwrap(); 4159 assert!(buf.contains(CONSOLE_TEST_STRING)); 4160 }); 4161 4162 handle_child_output(r, &output); 4163 } 4164 4165 #[test] 4166 fn test_pty_interaction() { 4167 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4168 let guest = Guest::new(Box::new(focal)); 4169 let api_socket = temp_api_path(&guest.tmp_dir); 4170 let serial_option = if cfg!(target_arch = "x86_64") { 4171 " console=ttyS0" 4172 } else { 4173 " console=ttyAMA0" 4174 }; 4175 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4176 4177 let mut child = GuestCommand::new(&guest) 4178 .args(["--cpus", "boot=1"]) 4179 .args(["--memory", "size=512M"]) 4180 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4181 .args(["--cmdline", &cmdline]) 4182 .default_disks() 4183 .default_net() 4184 .args(["--serial", "null"]) 4185 .args(["--console", "pty"]) 4186 .args(["--api-socket", &api_socket]) 4187 .spawn() 4188 .unwrap(); 4189 4190 let r = std::panic::catch_unwind(|| { 4191 guest.wait_vm_boot(None).unwrap(); 4192 // Get pty fd for console 4193 let console_path = get_pty_path(&api_socket, "console"); 4194 _test_pty_interaction(console_path); 4195 4196 guest.ssh_command("sudo shutdown -h now").unwrap(); 4197 }); 4198 4199 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4200 let _ = child.kill(); 4201 let output = child.wait_with_output().unwrap(); 4202 handle_child_output(r, &output); 4203 4204 let r = std::panic::catch_unwind(|| { 4205 // Check that the cloud-hypervisor binary actually terminated 4206 assert!(output.status.success()) 4207 }); 4208 handle_child_output(r, &output); 4209 } 4210 4211 #[test] 4212 fn test_serial_socket_interaction() { 4213 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4214 let guest = Guest::new(Box::new(focal)); 4215 let serial_socket = guest.tmp_dir.as_path().join("serial.socket"); 4216 let serial_socket_pty = guest.tmp_dir.as_path().join("serial.pty"); 4217 let serial_option = if cfg!(target_arch = "x86_64") { 4218 " console=ttyS0" 4219 } else { 4220 " console=ttyAMA0" 4221 }; 4222 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4223 4224 let mut child = GuestCommand::new(&guest) 4225 .args(["--cpus", "boot=1"]) 4226 .args(["--memory", "size=512M"]) 4227 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4228 .args(["--cmdline", &cmdline]) 4229 .default_disks() 4230 .default_net() 4231 .args(["--console", "null"]) 4232 .args([ 4233 "--serial", 4234 format!("socket={}", serial_socket.to_str().unwrap()).as_str(), 4235 ]) 4236 .spawn() 4237 .unwrap(); 4238 4239 let _ = std::panic::catch_unwind(|| { 4240 guest.wait_vm_boot(None).unwrap(); 4241 }); 4242 4243 let mut socat_command = Command::new("socat"); 4244 let socat_args = [ 4245 &format!("pty,link={},raw", serial_socket_pty.display()), 4246 &format!("UNIX-CONNECT:{}", serial_socket.display()), 4247 ]; 4248 socat_command.args(socat_args); 4249 4250 let mut socat_child = socat_command.spawn().unwrap(); 4251 thread::sleep(std::time::Duration::new(1, 0)); 4252 4253 let _ = std::panic::catch_unwind(|| { 4254 _test_pty_interaction(serial_socket_pty); 4255 }); 4256 4257 let _ = socat_child.kill(); 4258 let _ = socat_child.wait(); 4259 4260 let r = std::panic::catch_unwind(|| { 4261 guest.ssh_command("sudo shutdown -h now").unwrap(); 4262 }); 4263 4264 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4265 kill_child(&mut child); 4266 let output = child.wait_with_output().unwrap(); 4267 handle_child_output(r, &output); 4268 4269 let r = std::panic::catch_unwind(|| { 4270 // Check that the cloud-hypervisor binary actually terminated 4271 if !output.status.success() { 4272 panic!( 4273 "Cloud Hypervisor process failed to terminate gracefully: {:?}", 4274 output.status 4275 ); 4276 } 4277 }); 4278 handle_child_output(r, &output); 4279 } 4280 4281 #[test] 4282 fn test_virtio_console() { 4283 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4284 let guest = Guest::new(Box::new(focal)); 4285 4286 let kernel_path = direct_kernel_boot_path(); 4287 4288 let mut child = GuestCommand::new(&guest) 4289 .args(["--cpus", "boot=1"]) 4290 .args(["--memory", "size=512M"]) 4291 .args(["--kernel", kernel_path.to_str().unwrap()]) 4292 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4293 .default_disks() 4294 .default_net() 4295 .args(["--console", "tty"]) 4296 .args(["--serial", "null"]) 4297 .capture_output() 4298 .spawn() 4299 .unwrap(); 4300 4301 let text = String::from("On a branch floating down river a cricket, singing."); 4302 let cmd = format!("echo {text} | sudo tee /dev/hvc0"); 4303 4304 let r = std::panic::catch_unwind(|| { 4305 guest.wait_vm_boot(None).unwrap(); 4306 4307 assert!(guest 4308 .does_device_vendor_pair_match("0x1043", "0x1af4") 4309 .unwrap_or_default()); 4310 4311 guest.ssh_command(&cmd).unwrap(); 4312 }); 4313 4314 kill_child(&mut child); 4315 let output = child.wait_with_output().unwrap(); 4316 handle_child_output(r, &output); 4317 4318 let r = std::panic::catch_unwind(|| { 4319 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 4320 }); 4321 4322 handle_child_output(r, &output); 4323 } 4324 4325 #[test] 4326 fn test_console_file() { 4327 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4328 let guest = Guest::new(Box::new(focal)); 4329 4330 let console_path = guest.tmp_dir.as_path().join("console-output"); 4331 let mut child = GuestCommand::new(&guest) 4332 .args(["--cpus", "boot=1"]) 4333 .args(["--memory", "size=512M"]) 4334 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4335 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4336 .default_disks() 4337 .default_net() 4338 .args([ 4339 "--console", 4340 format!("file={}", console_path.to_str().unwrap()).as_str(), 4341 ]) 4342 .capture_output() 4343 .spawn() 4344 .unwrap(); 4345 4346 guest.wait_vm_boot(None).unwrap(); 4347 4348 guest.ssh_command("sudo shutdown -h now").unwrap(); 4349 4350 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4351 kill_child(&mut child); 4352 let output = child.wait_with_output().unwrap(); 4353 4354 let r = std::panic::catch_unwind(|| { 4355 // Check that the cloud-hypervisor binary actually terminated 4356 assert!(output.status.success()); 4357 4358 // Do this check after shutdown of the VM as an easy way to ensure 4359 // all writes are flushed to disk 4360 let mut f = std::fs::File::open(console_path).unwrap(); 4361 let mut buf = String::new(); 4362 f.read_to_string(&mut buf).unwrap(); 4363 4364 if !buf.contains(CONSOLE_TEST_STRING) { 4365 eprintln!( 4366 "\n\n==== Console file output ====\n\n{buf}\n\n==== End console file output ====" 4367 ); 4368 } 4369 assert!(buf.contains(CONSOLE_TEST_STRING)); 4370 }); 4371 4372 handle_child_output(r, &output); 4373 } 4374 4375 #[test] 4376 #[cfg(target_arch = "x86_64")] 4377 #[cfg(not(feature = "mshv"))] 4378 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 4379 // backed networking interfaces, bound through a simple bridge on the host. 4380 // So if the nested cloud-hypervisor succeeds in getting a directly 4381 // assigned interface from its cloud-hypervisor host, we should be able to 4382 // ssh into it, and verify that it's running with the right kernel command 4383 // line (We tag the command line from cloud-hypervisor for that purpose). 4384 // The third device is added to validate that hotplug works correctly since 4385 // it is being added to the L2 VM through hotplugging mechanism. 4386 // Also, we pass-through a virtio-blk device to the L2 VM to test the 32-bit 4387 // vfio device support 4388 fn test_vfio() { 4389 setup_vfio_network_interfaces(); 4390 4391 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 4392 let guest = Guest::new_from_ip_range(Box::new(jammy), "172.18", 0); 4393 4394 let mut workload_path = dirs::home_dir().unwrap(); 4395 workload_path.push("workloads"); 4396 4397 let kernel_path = direct_kernel_boot_path(); 4398 4399 let mut vfio_path = workload_path.clone(); 4400 vfio_path.push("vfio"); 4401 4402 let mut cloud_init_vfio_base_path = vfio_path.clone(); 4403 cloud_init_vfio_base_path.push("cloudinit.img"); 4404 4405 // We copy our cloudinit into the vfio mount point, for the nested 4406 // cloud-hypervisor guest to use. 4407 rate_limited_copy( 4408 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 4409 &cloud_init_vfio_base_path, 4410 ) 4411 .expect("copying of cloud-init disk failed"); 4412 4413 let mut vfio_disk_path = workload_path.clone(); 4414 vfio_disk_path.push("vfio.img"); 4415 4416 // Create the vfio disk image 4417 let output = Command::new("mkfs.ext4") 4418 .arg("-d") 4419 .arg(vfio_path.to_str().unwrap()) 4420 .arg(vfio_disk_path.to_str().unwrap()) 4421 .arg("2g") 4422 .output() 4423 .unwrap(); 4424 if !output.status.success() { 4425 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 4426 panic!("mkfs.ext4 command generated an error"); 4427 } 4428 4429 let mut blk_file_path = workload_path; 4430 blk_file_path.push("blk.img"); 4431 4432 let vfio_tap0 = "vfio-tap0"; 4433 let vfio_tap1 = "vfio-tap1"; 4434 let vfio_tap2 = "vfio-tap2"; 4435 let vfio_tap3 = "vfio-tap3"; 4436 4437 let mut child = GuestCommand::new(&guest) 4438 .args(["--cpus", "boot=4"]) 4439 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 4440 .args(["--kernel", kernel_path.to_str().unwrap()]) 4441 .args([ 4442 "--disk", 4443 format!( 4444 "path={}", 4445 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4446 ) 4447 .as_str(), 4448 format!( 4449 "path={}", 4450 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4451 ) 4452 .as_str(), 4453 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 4454 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 4455 ]) 4456 .args([ 4457 "--cmdline", 4458 format!( 4459 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 4460 ) 4461 .as_str(), 4462 ]) 4463 .args([ 4464 "--net", 4465 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 4466 format!( 4467 "tap={},mac={},iommu=on", 4468 vfio_tap1, guest.network.l2_guest_mac1 4469 ) 4470 .as_str(), 4471 format!( 4472 "tap={},mac={},iommu=on", 4473 vfio_tap2, guest.network.l2_guest_mac2 4474 ) 4475 .as_str(), 4476 format!( 4477 "tap={},mac={},iommu=on", 4478 vfio_tap3, guest.network.l2_guest_mac3 4479 ) 4480 .as_str(), 4481 ]) 4482 .capture_output() 4483 .spawn() 4484 .unwrap(); 4485 4486 thread::sleep(std::time::Duration::new(30, 0)); 4487 4488 let r = std::panic::catch_unwind(|| { 4489 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 4490 thread::sleep(std::time::Duration::new(120, 0)); 4491 4492 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 4493 // added to its kernel command line. 4494 // Let's ssh into it and verify that it's there. If it is it means 4495 // we're in the right guest (The L2 one) because the QEMU L1 guest 4496 // does not have this command line tag. 4497 assert!(check_matched_lines_count( 4498 guest.ssh_command_l2_1("cat /proc/cmdline").unwrap().trim(), 4499 vec!["VFIOTAG"], 4500 1 4501 )); 4502 4503 // Let's also verify from the second virtio-net device passed to 4504 // the L2 VM. 4505 assert!(check_matched_lines_count( 4506 guest.ssh_command_l2_2("cat /proc/cmdline").unwrap().trim(), 4507 vec!["VFIOTAG"], 4508 1 4509 )); 4510 4511 // Check the amount of PCI devices appearing in L2 VM. 4512 assert!(check_lines_count( 4513 guest 4514 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4515 .unwrap() 4516 .trim(), 4517 8 4518 )); 4519 4520 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 4521 assert!(check_matched_lines_count( 4522 guest.ssh_command_l2_1("lsblk").unwrap().trim(), 4523 vec!["vdc", "16M"], 4524 1 4525 )); 4526 4527 // Hotplug an extra virtio-net device through L2 VM. 4528 guest 4529 .ssh_command_l1( 4530 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 4531 ) 4532 .unwrap(); 4533 guest 4534 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 4535 .unwrap(); 4536 let vfio_hotplug_output = guest 4537 .ssh_command_l1( 4538 "sudo /mnt/ch-remote \ 4539 --api-socket=/tmp/ch_api.sock \ 4540 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 4541 ) 4542 .unwrap(); 4543 assert!(check_matched_lines_count( 4544 vfio_hotplug_output.trim(), 4545 vec!["{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}"], 4546 1 4547 )); 4548 4549 thread::sleep(std::time::Duration::new(10, 0)); 4550 4551 // Let's also verify from the third virtio-net device passed to 4552 // the L2 VM. This third device has been hotplugged through the L2 4553 // VM, so this is our way to validate hotplug works for VFIO PCI. 4554 assert!(check_matched_lines_count( 4555 guest.ssh_command_l2_3("cat /proc/cmdline").unwrap().trim(), 4556 vec!["VFIOTAG"], 4557 1 4558 )); 4559 4560 // Check the amount of PCI devices appearing in L2 VM. 4561 // There should be one more device than before, raising the count 4562 // up to 9 PCI devices. 4563 assert!(check_lines_count( 4564 guest 4565 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4566 .unwrap() 4567 .trim(), 4568 9 4569 )); 4570 4571 // Let's now verify that we can correctly remove the virtio-net 4572 // device through the "remove-device" command responsible for 4573 // unplugging VFIO devices. 4574 guest 4575 .ssh_command_l1( 4576 "sudo /mnt/ch-remote \ 4577 --api-socket=/tmp/ch_api.sock \ 4578 remove-device vfio123", 4579 ) 4580 .unwrap(); 4581 thread::sleep(std::time::Duration::new(10, 0)); 4582 4583 // Check the amount of PCI devices appearing in L2 VM is back down 4584 // to 8 devices. 4585 assert!(check_lines_count( 4586 guest 4587 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4588 .unwrap() 4589 .trim(), 4590 8 4591 )); 4592 4593 // Perform memory hotplug in L2 and validate the memory is showing 4594 // up as expected. In order to check, we will use the virtio-net 4595 // device already passed through L2 as a VFIO device, this will 4596 // verify that VFIO devices are functional with memory hotplug. 4597 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 4598 guest 4599 .ssh_command_l2_1( 4600 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 4601 ) 4602 .unwrap(); 4603 guest 4604 .ssh_command_l1( 4605 "sudo /mnt/ch-remote \ 4606 --api-socket=/tmp/ch_api.sock \ 4607 resize --memory=1073741824", 4608 ) 4609 .unwrap(); 4610 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 4611 }); 4612 4613 kill_child(&mut child); 4614 let output = child.wait_with_output().unwrap(); 4615 4616 cleanup_vfio_network_interfaces(); 4617 4618 handle_child_output(r, &output); 4619 } 4620 4621 #[test] 4622 fn test_direct_kernel_boot_noacpi() { 4623 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4624 let guest = Guest::new(Box::new(focal)); 4625 4626 let kernel_path = direct_kernel_boot_path(); 4627 4628 let mut child = GuestCommand::new(&guest) 4629 .args(["--cpus", "boot=1"]) 4630 .args(["--memory", "size=512M"]) 4631 .args(["--kernel", kernel_path.to_str().unwrap()]) 4632 .args([ 4633 "--cmdline", 4634 format!("{DIRECT_KERNEL_BOOT_CMDLINE} acpi=off").as_str(), 4635 ]) 4636 .default_disks() 4637 .default_net() 4638 .capture_output() 4639 .spawn() 4640 .unwrap(); 4641 4642 let r = std::panic::catch_unwind(|| { 4643 guest.wait_vm_boot(None).unwrap(); 4644 4645 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4646 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4647 }); 4648 4649 kill_child(&mut child); 4650 let output = child.wait_with_output().unwrap(); 4651 4652 handle_child_output(r, &output); 4653 } 4654 4655 #[test] 4656 fn test_virtio_vsock() { 4657 _test_virtio_vsock(false) 4658 } 4659 4660 #[test] 4661 fn test_virtio_vsock_hotplug() { 4662 _test_virtio_vsock(true); 4663 } 4664 4665 #[test] 4666 fn test_api_http_shutdown() { 4667 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4668 let guest = Guest::new(Box::new(focal)); 4669 4670 _test_api_shutdown(TargetApi::new_http_api(&guest.tmp_dir), guest) 4671 } 4672 4673 #[test] 4674 fn test_api_http_delete() { 4675 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4676 let guest = Guest::new(Box::new(focal)); 4677 4678 _test_api_delete(TargetApi::new_http_api(&guest.tmp_dir), guest); 4679 } 4680 4681 #[test] 4682 fn test_api_http_pause_resume() { 4683 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4684 let guest = Guest::new(Box::new(focal)); 4685 4686 _test_api_pause_resume(TargetApi::new_http_api(&guest.tmp_dir), guest) 4687 } 4688 4689 #[test] 4690 fn test_api_http_create_boot() { 4691 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4692 let guest = Guest::new(Box::new(focal)); 4693 4694 _test_api_create_boot(TargetApi::new_http_api(&guest.tmp_dir), guest) 4695 } 4696 4697 #[test] 4698 fn test_virtio_iommu() { 4699 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4700 } 4701 4702 #[test] 4703 // We cannot force the software running in the guest to reprogram the BAR 4704 // with some different addresses, but we have a reliable way of testing it 4705 // with a standard Linux kernel. 4706 // By removing a device from the PCI tree, and then rescanning the tree, 4707 // Linux consistently chooses to reorganize the PCI device BARs to other 4708 // locations in the guest address space. 4709 // This test creates a dedicated PCI network device to be checked as being 4710 // properly probed first, then removing it, and adding it again by doing a 4711 // rescan. 4712 fn test_pci_bar_reprogramming() { 4713 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4714 let guest = Guest::new(Box::new(focal)); 4715 4716 #[cfg(target_arch = "x86_64")] 4717 let kernel_path = direct_kernel_boot_path(); 4718 #[cfg(target_arch = "aarch64")] 4719 let kernel_path = edk2_path(); 4720 4721 let mut child = GuestCommand::new(&guest) 4722 .args(["--cpus", "boot=1"]) 4723 .args(["--memory", "size=512M"]) 4724 .args(["--kernel", kernel_path.to_str().unwrap()]) 4725 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4726 .default_disks() 4727 .args([ 4728 "--net", 4729 guest.default_net_string().as_str(), 4730 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4731 ]) 4732 .capture_output() 4733 .spawn() 4734 .unwrap(); 4735 4736 let r = std::panic::catch_unwind(|| { 4737 guest.wait_vm_boot(None).unwrap(); 4738 4739 // 2 network interfaces + default localhost ==> 3 interfaces 4740 assert_eq!( 4741 guest 4742 .ssh_command("ip -o link | wc -l") 4743 .unwrap() 4744 .trim() 4745 .parse::<u32>() 4746 .unwrap_or_default(), 4747 3 4748 ); 4749 4750 let init_bar_addr = guest 4751 .ssh_command( 4752 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4753 ) 4754 .unwrap(); 4755 4756 // Remove the PCI device 4757 guest 4758 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4759 .unwrap(); 4760 4761 // Only 1 network interface left + default localhost ==> 2 interfaces 4762 assert_eq!( 4763 guest 4764 .ssh_command("ip -o link | wc -l") 4765 .unwrap() 4766 .trim() 4767 .parse::<u32>() 4768 .unwrap_or_default(), 4769 2 4770 ); 4771 4772 // Remove the PCI device 4773 guest 4774 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4775 .unwrap(); 4776 4777 // Back to 2 network interface + default localhost ==> 3 interfaces 4778 assert_eq!( 4779 guest 4780 .ssh_command("ip -o link | wc -l") 4781 .unwrap() 4782 .trim() 4783 .parse::<u32>() 4784 .unwrap_or_default(), 4785 3 4786 ); 4787 4788 let new_bar_addr = guest 4789 .ssh_command( 4790 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4791 ) 4792 .unwrap(); 4793 4794 // Let's compare the BAR addresses for our virtio-net device. 4795 // They should be different as we expect the BAR reprogramming 4796 // to have happened. 4797 assert_ne!(init_bar_addr, new_bar_addr); 4798 }); 4799 4800 kill_child(&mut child); 4801 let output = child.wait_with_output().unwrap(); 4802 4803 handle_child_output(r, &output); 4804 } 4805 4806 #[test] 4807 fn test_memory_mergeable_off() { 4808 test_memory_mergeable(false) 4809 } 4810 4811 #[test] 4812 #[cfg(target_arch = "x86_64")] 4813 fn test_cpu_hotplug() { 4814 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4815 let guest = Guest::new(Box::new(focal)); 4816 let api_socket = temp_api_path(&guest.tmp_dir); 4817 4818 let kernel_path = direct_kernel_boot_path(); 4819 4820 let mut child = GuestCommand::new(&guest) 4821 .args(["--cpus", "boot=2,max=4"]) 4822 .args(["--memory", "size=512M"]) 4823 .args(["--kernel", kernel_path.to_str().unwrap()]) 4824 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4825 .default_disks() 4826 .default_net() 4827 .args(["--api-socket", &api_socket]) 4828 .capture_output() 4829 .spawn() 4830 .unwrap(); 4831 4832 let r = std::panic::catch_unwind(|| { 4833 guest.wait_vm_boot(None).unwrap(); 4834 4835 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4836 4837 // Resize the VM 4838 let desired_vcpus = 4; 4839 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4840 4841 guest 4842 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4843 .unwrap(); 4844 guest 4845 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4846 .unwrap(); 4847 thread::sleep(std::time::Duration::new(10, 0)); 4848 assert_eq!( 4849 guest.get_cpu_count().unwrap_or_default(), 4850 u32::from(desired_vcpus) 4851 ); 4852 4853 guest.reboot_linux(0, None); 4854 4855 assert_eq!( 4856 guest.get_cpu_count().unwrap_or_default(), 4857 u32::from(desired_vcpus) 4858 ); 4859 4860 // Resize the VM 4861 let desired_vcpus = 2; 4862 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4863 4864 thread::sleep(std::time::Duration::new(10, 0)); 4865 assert_eq!( 4866 guest.get_cpu_count().unwrap_or_default(), 4867 u32::from(desired_vcpus) 4868 ); 4869 4870 // Resize the VM back up to 4 4871 let desired_vcpus = 4; 4872 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4873 4874 guest 4875 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4876 .unwrap(); 4877 guest 4878 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4879 .unwrap(); 4880 thread::sleep(std::time::Duration::new(10, 0)); 4881 assert_eq!( 4882 guest.get_cpu_count().unwrap_or_default(), 4883 u32::from(desired_vcpus) 4884 ); 4885 }); 4886 4887 kill_child(&mut child); 4888 let output = child.wait_with_output().unwrap(); 4889 4890 handle_child_output(r, &output); 4891 } 4892 4893 #[test] 4894 fn test_memory_hotplug() { 4895 #[cfg(target_arch = "aarch64")] 4896 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4897 #[cfg(target_arch = "x86_64")] 4898 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4899 let focal = UbuntuDiskConfig::new(focal_image); 4900 let guest = Guest::new(Box::new(focal)); 4901 let api_socket = temp_api_path(&guest.tmp_dir); 4902 4903 #[cfg(target_arch = "aarch64")] 4904 let kernel_path = edk2_path(); 4905 #[cfg(target_arch = "x86_64")] 4906 let kernel_path = direct_kernel_boot_path(); 4907 4908 let mut child = GuestCommand::new(&guest) 4909 .args(["--cpus", "boot=2,max=4"]) 4910 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4911 .args(["--kernel", kernel_path.to_str().unwrap()]) 4912 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4913 .default_disks() 4914 .default_net() 4915 .args(["--balloon", "size=0"]) 4916 .args(["--api-socket", &api_socket]) 4917 .capture_output() 4918 .spawn() 4919 .unwrap(); 4920 4921 let r = std::panic::catch_unwind(|| { 4922 guest.wait_vm_boot(None).unwrap(); 4923 4924 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4925 4926 guest.enable_memory_hotplug(); 4927 4928 // Add RAM to the VM 4929 let desired_ram = 1024 << 20; 4930 resize_command(&api_socket, None, Some(desired_ram), None, None); 4931 4932 thread::sleep(std::time::Duration::new(10, 0)); 4933 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4934 4935 // Use balloon to remove RAM from the VM 4936 let desired_balloon = 512 << 20; 4937 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4938 4939 thread::sleep(std::time::Duration::new(10, 0)); 4940 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4941 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4942 4943 guest.reboot_linux(0, None); 4944 4945 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4946 4947 // Use balloon add RAM to the VM 4948 let desired_balloon = 0; 4949 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4950 4951 thread::sleep(std::time::Duration::new(10, 0)); 4952 4953 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4954 4955 guest.enable_memory_hotplug(); 4956 4957 // Add RAM to the VM 4958 let desired_ram = 2048 << 20; 4959 resize_command(&api_socket, None, Some(desired_ram), None, None); 4960 4961 thread::sleep(std::time::Duration::new(10, 0)); 4962 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4963 4964 // Remove RAM to the VM (only applies after reboot) 4965 let desired_ram = 1024 << 20; 4966 resize_command(&api_socket, None, Some(desired_ram), None, None); 4967 4968 guest.reboot_linux(1, None); 4969 4970 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4971 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4972 }); 4973 4974 kill_child(&mut child); 4975 let output = child.wait_with_output().unwrap(); 4976 4977 handle_child_output(r, &output); 4978 } 4979 4980 #[test] 4981 #[cfg(not(feature = "mshv"))] 4982 fn test_virtio_mem() { 4983 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4984 let guest = Guest::new(Box::new(focal)); 4985 let api_socket = temp_api_path(&guest.tmp_dir); 4986 4987 let kernel_path = direct_kernel_boot_path(); 4988 4989 let mut child = GuestCommand::new(&guest) 4990 .args(["--cpus", "boot=2,max=4"]) 4991 .args([ 4992 "--memory", 4993 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4994 ]) 4995 .args(["--kernel", kernel_path.to_str().unwrap()]) 4996 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4997 .default_disks() 4998 .default_net() 4999 .args(["--api-socket", &api_socket]) 5000 .capture_output() 5001 .spawn() 5002 .unwrap(); 5003 5004 let r = std::panic::catch_unwind(|| { 5005 guest.wait_vm_boot(None).unwrap(); 5006 5007 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5008 5009 guest.enable_memory_hotplug(); 5010 5011 // Add RAM to the VM 5012 let desired_ram = 1024 << 20; 5013 resize_command(&api_socket, None, Some(desired_ram), None, None); 5014 5015 thread::sleep(std::time::Duration::new(10, 0)); 5016 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5017 5018 // Add RAM to the VM 5019 let desired_ram = 2048 << 20; 5020 resize_command(&api_socket, None, Some(desired_ram), None, None); 5021 5022 thread::sleep(std::time::Duration::new(10, 0)); 5023 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 5024 5025 // Remove RAM from the VM 5026 let desired_ram = 1024 << 20; 5027 resize_command(&api_socket, None, Some(desired_ram), None, None); 5028 5029 thread::sleep(std::time::Duration::new(10, 0)); 5030 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5031 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5032 5033 guest.reboot_linux(0, None); 5034 5035 // Check the amount of memory after reboot is 1GiB 5036 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5037 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5038 5039 // Check we can still resize to 512MiB 5040 let desired_ram = 512 << 20; 5041 resize_command(&api_socket, None, Some(desired_ram), None, None); 5042 thread::sleep(std::time::Duration::new(10, 0)); 5043 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5044 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 5045 }); 5046 5047 kill_child(&mut child); 5048 let output = child.wait_with_output().unwrap(); 5049 5050 handle_child_output(r, &output); 5051 } 5052 5053 #[test] 5054 #[cfg(target_arch = "x86_64")] 5055 #[cfg(not(feature = "mshv"))] 5056 // Test both vCPU and memory resizing together 5057 fn test_resize() { 5058 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5059 let guest = Guest::new(Box::new(focal)); 5060 let api_socket = temp_api_path(&guest.tmp_dir); 5061 5062 let kernel_path = direct_kernel_boot_path(); 5063 5064 let mut child = GuestCommand::new(&guest) 5065 .args(["--cpus", "boot=2,max=4"]) 5066 .args(["--memory", "size=512M,hotplug_size=8192M"]) 5067 .args(["--kernel", kernel_path.to_str().unwrap()]) 5068 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5069 .default_disks() 5070 .default_net() 5071 .args(["--api-socket", &api_socket]) 5072 .capture_output() 5073 .spawn() 5074 .unwrap(); 5075 5076 let r = std::panic::catch_unwind(|| { 5077 guest.wait_vm_boot(None).unwrap(); 5078 5079 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 5080 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5081 5082 guest.enable_memory_hotplug(); 5083 5084 // Resize the VM 5085 let desired_vcpus = 4; 5086 let desired_ram = 1024 << 20; 5087 resize_command( 5088 &api_socket, 5089 Some(desired_vcpus), 5090 Some(desired_ram), 5091 None, 5092 None, 5093 ); 5094 5095 guest 5096 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 5097 .unwrap(); 5098 guest 5099 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 5100 .unwrap(); 5101 thread::sleep(std::time::Duration::new(10, 0)); 5102 assert_eq!( 5103 guest.get_cpu_count().unwrap_or_default(), 5104 u32::from(desired_vcpus) 5105 ); 5106 5107 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5108 }); 5109 5110 kill_child(&mut child); 5111 let output = child.wait_with_output().unwrap(); 5112 5113 handle_child_output(r, &output); 5114 } 5115 5116 #[test] 5117 fn test_memory_overhead() { 5118 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5119 let guest = Guest::new(Box::new(focal)); 5120 5121 let kernel_path = direct_kernel_boot_path(); 5122 5123 let guest_memory_size_kb = 512 * 1024; 5124 5125 let mut child = GuestCommand::new(&guest) 5126 .args(["--cpus", "boot=1"]) 5127 .args(["--memory", format!("size={guest_memory_size_kb}K").as_str()]) 5128 .args(["--kernel", kernel_path.to_str().unwrap()]) 5129 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5130 .default_net() 5131 .default_disks() 5132 .capture_output() 5133 .spawn() 5134 .unwrap(); 5135 5136 guest.wait_vm_boot(None).unwrap(); 5137 5138 let r = std::panic::catch_unwind(|| { 5139 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 5140 eprintln!("Guest memory overhead: {overhead} vs {MAXIMUM_VMM_OVERHEAD_KB}"); 5141 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 5142 }); 5143 5144 kill_child(&mut child); 5145 let output = child.wait_with_output().unwrap(); 5146 5147 handle_child_output(r, &output); 5148 } 5149 5150 #[test] 5151 #[cfg(target_arch = "x86_64")] 5152 // This test runs a guest with Landlock enabled and hotplugs a new disk. As 5153 // the path for the hotplug disk is not pre-added to Landlock rules, this 5154 // the test will result in a failure. 5155 fn test_landlock() { 5156 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5157 let guest = Guest::new(Box::new(focal)); 5158 5159 #[cfg(target_arch = "x86_64")] 5160 let kernel_path = direct_kernel_boot_path(); 5161 #[cfg(target_arch = "aarch64")] 5162 let kernel_path = edk2_path(); 5163 5164 let api_socket = temp_api_path(&guest.tmp_dir); 5165 5166 let mut child = GuestCommand::new(&guest) 5167 .args(["--api-socket", &api_socket]) 5168 .args(["--cpus", "boot=1"]) 5169 .args(["--memory", "size=512M"]) 5170 .args(["--kernel", kernel_path.to_str().unwrap()]) 5171 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5172 .args(["--landlock"]) 5173 .default_disks() 5174 .default_net() 5175 .capture_output() 5176 .spawn() 5177 .unwrap(); 5178 5179 let r = std::panic::catch_unwind(|| { 5180 guest.wait_vm_boot(None).unwrap(); 5181 5182 // Check /dev/vdc is not there 5183 assert_eq!( 5184 guest 5185 .ssh_command("lsblk | grep -c vdc.*16M || true") 5186 .unwrap() 5187 .trim() 5188 .parse::<u32>() 5189 .unwrap_or(1), 5190 0 5191 ); 5192 5193 // Now let's add the extra disk. 5194 let mut blk_file_path = dirs::home_dir().unwrap(); 5195 blk_file_path.push("workloads"); 5196 blk_file_path.push("blk.img"); 5197 // As the path to the hotplug disk is not pre-added, this remote 5198 // command will fail. 5199 assert!(!remote_command( 5200 &api_socket, 5201 "add-disk", 5202 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5203 )); 5204 }); 5205 5206 let _ = child.kill(); 5207 let output = child.wait_with_output().unwrap(); 5208 5209 handle_child_output(r, &output); 5210 } 5211 5212 fn _test_disk_hotplug(landlock_enabled: bool) { 5213 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5214 let guest = Guest::new(Box::new(focal)); 5215 5216 #[cfg(target_arch = "x86_64")] 5217 let kernel_path = direct_kernel_boot_path(); 5218 #[cfg(target_arch = "aarch64")] 5219 let kernel_path = edk2_path(); 5220 5221 let api_socket = temp_api_path(&guest.tmp_dir); 5222 5223 let mut blk_file_path = dirs::home_dir().unwrap(); 5224 blk_file_path.push("workloads"); 5225 blk_file_path.push("blk.img"); 5226 5227 let mut cmd = GuestCommand::new(&guest); 5228 if landlock_enabled { 5229 cmd.args(["--landlock"]).args([ 5230 "--landlock-rules", 5231 format!("path={:?},access=rw", blk_file_path).as_str(), 5232 ]); 5233 } 5234 5235 cmd.args(["--api-socket", &api_socket]) 5236 .args(["--cpus", "boot=1"]) 5237 .args(["--memory", "size=512M"]) 5238 .args(["--kernel", kernel_path.to_str().unwrap()]) 5239 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5240 .default_disks() 5241 .default_net() 5242 .capture_output(); 5243 5244 let mut child = cmd.spawn().unwrap(); 5245 5246 let r = std::panic::catch_unwind(|| { 5247 guest.wait_vm_boot(None).unwrap(); 5248 5249 // Check /dev/vdc is not there 5250 assert_eq!( 5251 guest 5252 .ssh_command("lsblk | grep -c vdc.*16M || true") 5253 .unwrap() 5254 .trim() 5255 .parse::<u32>() 5256 .unwrap_or(1), 5257 0 5258 ); 5259 5260 // Now let's add the extra disk. 5261 let (cmd_success, cmd_output) = remote_command_w_output( 5262 &api_socket, 5263 "add-disk", 5264 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5265 ); 5266 assert!(cmd_success); 5267 assert!(String::from_utf8_lossy(&cmd_output) 5268 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5269 5270 thread::sleep(std::time::Duration::new(10, 0)); 5271 5272 // Check that /dev/vdc exists and the block size is 16M. 5273 assert_eq!( 5274 guest 5275 .ssh_command("lsblk | grep vdc | grep -c 16M") 5276 .unwrap() 5277 .trim() 5278 .parse::<u32>() 5279 .unwrap_or_default(), 5280 1 5281 ); 5282 // And check the block device can be read. 5283 guest 5284 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5285 .unwrap(); 5286 5287 // Let's remove it the extra disk. 5288 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5289 thread::sleep(std::time::Duration::new(5, 0)); 5290 // And check /dev/vdc is not there 5291 assert_eq!( 5292 guest 5293 .ssh_command("lsblk | grep -c vdc.*16M || true") 5294 .unwrap() 5295 .trim() 5296 .parse::<u32>() 5297 .unwrap_or(1), 5298 0 5299 ); 5300 5301 // And add it back to validate unplug did work correctly. 5302 let (cmd_success, cmd_output) = remote_command_w_output( 5303 &api_socket, 5304 "add-disk", 5305 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5306 ); 5307 assert!(cmd_success); 5308 assert!(String::from_utf8_lossy(&cmd_output) 5309 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5310 5311 thread::sleep(std::time::Duration::new(10, 0)); 5312 5313 // Check that /dev/vdc exists and the block size is 16M. 5314 assert_eq!( 5315 guest 5316 .ssh_command("lsblk | grep vdc | grep -c 16M") 5317 .unwrap() 5318 .trim() 5319 .parse::<u32>() 5320 .unwrap_or_default(), 5321 1 5322 ); 5323 // And check the block device can be read. 5324 guest 5325 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5326 .unwrap(); 5327 5328 // Reboot the VM. 5329 guest.reboot_linux(0, None); 5330 5331 // Check still there after reboot 5332 assert_eq!( 5333 guest 5334 .ssh_command("lsblk | grep vdc | grep -c 16M") 5335 .unwrap() 5336 .trim() 5337 .parse::<u32>() 5338 .unwrap_or_default(), 5339 1 5340 ); 5341 5342 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5343 5344 thread::sleep(std::time::Duration::new(20, 0)); 5345 5346 // Check device has gone away 5347 assert_eq!( 5348 guest 5349 .ssh_command("lsblk | grep -c vdc.*16M || true") 5350 .unwrap() 5351 .trim() 5352 .parse::<u32>() 5353 .unwrap_or(1), 5354 0 5355 ); 5356 5357 guest.reboot_linux(1, None); 5358 5359 // Check device still absent 5360 assert_eq!( 5361 guest 5362 .ssh_command("lsblk | grep -c vdc.*16M || true") 5363 .unwrap() 5364 .trim() 5365 .parse::<u32>() 5366 .unwrap_or(1), 5367 0 5368 ); 5369 }); 5370 5371 kill_child(&mut child); 5372 let output = child.wait_with_output().unwrap(); 5373 5374 handle_child_output(r, &output); 5375 } 5376 5377 #[test] 5378 fn test_disk_hotplug() { 5379 _test_disk_hotplug(false) 5380 } 5381 5382 #[test] 5383 #[cfg(target_arch = "x86_64")] 5384 fn test_disk_hotplug_with_landlock() { 5385 _test_disk_hotplug(true) 5386 } 5387 5388 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 5389 const LOOP_CONFIGURE: u64 = 0x4c0a; 5390 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 5391 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 5392 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 5393 5394 #[repr(C)] 5395 struct LoopInfo64 { 5396 lo_device: u64, 5397 lo_inode: u64, 5398 lo_rdevice: u64, 5399 lo_offset: u64, 5400 lo_sizelimit: u64, 5401 lo_number: u32, 5402 lo_encrypt_type: u32, 5403 lo_encrypt_key_size: u32, 5404 lo_flags: u32, 5405 lo_file_name: [u8; 64], 5406 lo_crypt_name: [u8; 64], 5407 lo_encrypt_key: [u8; 32], 5408 lo_init: [u64; 2], 5409 } 5410 5411 impl Default for LoopInfo64 { 5412 fn default() -> Self { 5413 LoopInfo64 { 5414 lo_device: 0, 5415 lo_inode: 0, 5416 lo_rdevice: 0, 5417 lo_offset: 0, 5418 lo_sizelimit: 0, 5419 lo_number: 0, 5420 lo_encrypt_type: 0, 5421 lo_encrypt_key_size: 0, 5422 lo_flags: 0, 5423 lo_file_name: [0; 64], 5424 lo_crypt_name: [0; 64], 5425 lo_encrypt_key: [0; 32], 5426 lo_init: [0; 2], 5427 } 5428 } 5429 } 5430 5431 #[derive(Default)] 5432 #[repr(C)] 5433 struct LoopConfig { 5434 fd: u32, 5435 block_size: u32, 5436 info: LoopInfo64, 5437 _reserved: [u64; 8], 5438 } 5439 5440 // Open loop-control device 5441 let loop_ctl_file = OpenOptions::new() 5442 .read(true) 5443 .write(true) 5444 .open(LOOP_CTL_PATH) 5445 .unwrap(); 5446 5447 // Request a free loop device 5448 let loop_device_number = 5449 unsafe { libc::ioctl(loop_ctl_file.as_raw_fd(), LOOP_CTL_GET_FREE as _) }; 5450 5451 if loop_device_number < 0 { 5452 panic!("Couldn't find a free loop device"); 5453 } 5454 5455 // Create loop device path 5456 let loop_device_path = format!("{LOOP_DEVICE_PREFIX}{loop_device_number}"); 5457 5458 // Open loop device 5459 let loop_device_file = OpenOptions::new() 5460 .read(true) 5461 .write(true) 5462 .open(&loop_device_path) 5463 .unwrap(); 5464 5465 // Open backing file 5466 let backing_file = OpenOptions::new() 5467 .read(true) 5468 .write(true) 5469 .open(backing_file_path) 5470 .unwrap(); 5471 5472 let loop_config = LoopConfig { 5473 fd: backing_file.as_raw_fd() as u32, 5474 block_size, 5475 ..Default::default() 5476 }; 5477 5478 for i in 0..num_retries { 5479 let ret = unsafe { 5480 libc::ioctl( 5481 loop_device_file.as_raw_fd(), 5482 LOOP_CONFIGURE as _, 5483 &loop_config, 5484 ) 5485 }; 5486 if ret != 0 { 5487 if i < num_retries - 1 { 5488 println!( 5489 "Iteration {}: Failed to configure the loop device {}: {}", 5490 i, 5491 loop_device_path, 5492 std::io::Error::last_os_error() 5493 ); 5494 } else { 5495 panic!( 5496 "Failed {} times trying to configure the loop device {}: {}", 5497 num_retries, 5498 loop_device_path, 5499 std::io::Error::last_os_error() 5500 ); 5501 } 5502 } else { 5503 break; 5504 } 5505 5506 // Wait for a bit before retrying 5507 thread::sleep(std::time::Duration::new(5, 0)); 5508 } 5509 5510 loop_device_path 5511 } 5512 5513 #[test] 5514 fn test_virtio_block_topology() { 5515 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5516 let guest = Guest::new(Box::new(focal)); 5517 5518 let kernel_path = direct_kernel_boot_path(); 5519 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5520 5521 let output = exec_host_command_output( 5522 format!( 5523 "qemu-img create -f raw {} 16M", 5524 test_disk_path.to_str().unwrap() 5525 ) 5526 .as_str(), 5527 ); 5528 if !output.status.success() { 5529 let stdout = String::from_utf8_lossy(&output.stdout); 5530 let stderr = String::from_utf8_lossy(&output.stderr); 5531 panic!("qemu-img command failed\nstdout\n{stdout}\nstderr\n{stderr}"); 5532 } 5533 5534 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5535 5536 let mut child = GuestCommand::new(&guest) 5537 .args(["--cpus", "boot=1"]) 5538 .args(["--memory", "size=512M"]) 5539 .args(["--kernel", kernel_path.to_str().unwrap()]) 5540 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5541 .args([ 5542 "--disk", 5543 format!( 5544 "path={}", 5545 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5546 ) 5547 .as_str(), 5548 format!( 5549 "path={}", 5550 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5551 ) 5552 .as_str(), 5553 format!("path={}", &loop_dev).as_str(), 5554 ]) 5555 .default_net() 5556 .capture_output() 5557 .spawn() 5558 .unwrap(); 5559 5560 let r = std::panic::catch_unwind(|| { 5561 guest.wait_vm_boot(None).unwrap(); 5562 5563 // MIN-IO column 5564 assert_eq!( 5565 guest 5566 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5567 .unwrap() 5568 .trim() 5569 .parse::<u32>() 5570 .unwrap_or_default(), 5571 4096 5572 ); 5573 // PHY-SEC column 5574 assert_eq!( 5575 guest 5576 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5577 .unwrap() 5578 .trim() 5579 .parse::<u32>() 5580 .unwrap_or_default(), 5581 4096 5582 ); 5583 // LOG-SEC column 5584 assert_eq!( 5585 guest 5586 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5587 .unwrap() 5588 .trim() 5589 .parse::<u32>() 5590 .unwrap_or_default(), 5591 4096 5592 ); 5593 }); 5594 5595 kill_child(&mut child); 5596 let output = child.wait_with_output().unwrap(); 5597 5598 handle_child_output(r, &output); 5599 5600 Command::new("losetup") 5601 .args(["-d", &loop_dev]) 5602 .output() 5603 .expect("loop device not found"); 5604 } 5605 5606 #[test] 5607 fn test_virtio_balloon_deflate_on_oom() { 5608 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5609 let guest = Guest::new(Box::new(focal)); 5610 5611 let kernel_path = direct_kernel_boot_path(); 5612 5613 let api_socket = temp_api_path(&guest.tmp_dir); 5614 5615 //Let's start a 4G guest with balloon occupied 2G memory 5616 let mut child = GuestCommand::new(&guest) 5617 .args(["--api-socket", &api_socket]) 5618 .args(["--cpus", "boot=1"]) 5619 .args(["--memory", "size=4G"]) 5620 .args(["--kernel", kernel_path.to_str().unwrap()]) 5621 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5622 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5623 .default_disks() 5624 .default_net() 5625 .capture_output() 5626 .spawn() 5627 .unwrap(); 5628 5629 let r = std::panic::catch_unwind(|| { 5630 guest.wait_vm_boot(None).unwrap(); 5631 5632 // Wait for balloon memory's initialization and check its size. 5633 // The virtio-balloon driver might take a few seconds to report the 5634 // balloon effective size back to the VMM. 5635 thread::sleep(std::time::Duration::new(20, 0)); 5636 5637 let orig_balloon = balloon_size(&api_socket); 5638 println!("The original balloon memory size is {orig_balloon} bytes"); 5639 assert!(orig_balloon == 2147483648); 5640 5641 // Two steps to verify if the 'deflate_on_oom' parameter works. 5642 // 1st: run a command to trigger an OOM in the guest. 5643 guest 5644 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5645 .unwrap(); 5646 5647 // Give some time for the OOM to happen in the guest and be reported 5648 // back to the host. 5649 thread::sleep(std::time::Duration::new(20, 0)); 5650 5651 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5652 let deflated_balloon = balloon_size(&api_socket); 5653 println!("After deflating, balloon memory size is {deflated_balloon} bytes"); 5654 // Verify the balloon size deflated 5655 assert!(deflated_balloon < 2147483648); 5656 }); 5657 5658 kill_child(&mut child); 5659 let output = child.wait_with_output().unwrap(); 5660 5661 handle_child_output(r, &output); 5662 } 5663 5664 #[test] 5665 #[cfg(not(feature = "mshv"))] 5666 fn test_virtio_balloon_free_page_reporting() { 5667 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5668 let guest = Guest::new(Box::new(focal)); 5669 5670 //Let's start a 4G guest with balloon occupied 2G memory 5671 let mut child = GuestCommand::new(&guest) 5672 .args(["--cpus", "boot=1"]) 5673 .args(["--memory", "size=4G"]) 5674 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5675 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5676 .args(["--balloon", "size=0,free_page_reporting=on"]) 5677 .default_disks() 5678 .default_net() 5679 .capture_output() 5680 .spawn() 5681 .unwrap(); 5682 5683 let pid = child.id(); 5684 let r = std::panic::catch_unwind(|| { 5685 guest.wait_vm_boot(None).unwrap(); 5686 5687 // Check the initial RSS is less than 1GiB 5688 let rss = process_rss_kib(pid); 5689 println!("RSS {rss} < 1048576"); 5690 assert!(rss < 1048576); 5691 5692 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5693 // seconds 5694 let guest_ip = guest.network.guest_ip.clone(); 5695 thread::spawn(move || { 5696 ssh_command_ip( 5697 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5698 &guest_ip, 5699 DEFAULT_SSH_RETRIES, 5700 DEFAULT_SSH_TIMEOUT, 5701 ) 5702 .unwrap(); 5703 }); 5704 5705 // Wait for 50 seconds to make sure the stress command is consuming 5706 // the expected amount of memory. 5707 thread::sleep(std::time::Duration::new(50, 0)); 5708 let rss = process_rss_kib(pid); 5709 println!("RSS {rss} >= 2097152"); 5710 assert!(rss >= 2097152); 5711 5712 // Wait for an extra minute to make sure the stress command has 5713 // completed and that the guest reported the free pages to the VMM 5714 // through the virtio-balloon device. We expect the RSS to be under 5715 // 2GiB. 5716 thread::sleep(std::time::Duration::new(60, 0)); 5717 let rss = process_rss_kib(pid); 5718 println!("RSS {rss} < 2097152"); 5719 assert!(rss < 2097152); 5720 }); 5721 5722 kill_child(&mut child); 5723 let output = child.wait_with_output().unwrap(); 5724 5725 handle_child_output(r, &output); 5726 } 5727 5728 #[test] 5729 fn test_pmem_hotplug() { 5730 _test_pmem_hotplug(None) 5731 } 5732 5733 #[test] 5734 fn test_pmem_multi_segment_hotplug() { 5735 _test_pmem_hotplug(Some(15)) 5736 } 5737 5738 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5739 #[cfg(target_arch = "aarch64")] 5740 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5741 #[cfg(target_arch = "x86_64")] 5742 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5743 let focal = UbuntuDiskConfig::new(focal_image); 5744 let guest = Guest::new(Box::new(focal)); 5745 5746 #[cfg(target_arch = "x86_64")] 5747 let kernel_path = direct_kernel_boot_path(); 5748 #[cfg(target_arch = "aarch64")] 5749 let kernel_path = edk2_path(); 5750 5751 let api_socket = temp_api_path(&guest.tmp_dir); 5752 5753 let mut cmd = GuestCommand::new(&guest); 5754 5755 cmd.args(["--api-socket", &api_socket]) 5756 .args(["--cpus", "boot=1"]) 5757 .args(["--memory", "size=512M"]) 5758 .args(["--kernel", kernel_path.to_str().unwrap()]) 5759 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5760 .default_disks() 5761 .default_net() 5762 .capture_output(); 5763 5764 if pci_segment.is_some() { 5765 cmd.args([ 5766 "--platform", 5767 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5768 ]); 5769 } 5770 5771 let mut child = cmd.spawn().unwrap(); 5772 5773 let r = std::panic::catch_unwind(|| { 5774 guest.wait_vm_boot(None).unwrap(); 5775 5776 // Check /dev/pmem0 is not there 5777 assert_eq!( 5778 guest 5779 .ssh_command("lsblk | grep -c pmem0 || true") 5780 .unwrap() 5781 .trim() 5782 .parse::<u32>() 5783 .unwrap_or(1), 5784 0 5785 ); 5786 5787 let pmem_temp_file = TempFile::new().unwrap(); 5788 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5789 let (cmd_success, cmd_output) = remote_command_w_output( 5790 &api_socket, 5791 "add-pmem", 5792 Some(&format!( 5793 "file={},id=test0{}", 5794 pmem_temp_file.as_path().to_str().unwrap(), 5795 if let Some(pci_segment) = pci_segment { 5796 format!(",pci_segment={pci_segment}") 5797 } else { 5798 "".to_owned() 5799 } 5800 )), 5801 ); 5802 assert!(cmd_success); 5803 if let Some(pci_segment) = pci_segment { 5804 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5805 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5806 ))); 5807 } else { 5808 assert!(String::from_utf8_lossy(&cmd_output) 5809 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5810 } 5811 5812 // Check that /dev/pmem0 exists and the block size is 128M 5813 assert_eq!( 5814 guest 5815 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5816 .unwrap() 5817 .trim() 5818 .parse::<u32>() 5819 .unwrap_or_default(), 5820 1 5821 ); 5822 5823 guest.reboot_linux(0, None); 5824 5825 // Check still there after reboot 5826 assert_eq!( 5827 guest 5828 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5829 .unwrap() 5830 .trim() 5831 .parse::<u32>() 5832 .unwrap_or_default(), 5833 1 5834 ); 5835 5836 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5837 5838 thread::sleep(std::time::Duration::new(20, 0)); 5839 5840 // Check device has gone away 5841 assert_eq!( 5842 guest 5843 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5844 .unwrap() 5845 .trim() 5846 .parse::<u32>() 5847 .unwrap_or(1), 5848 0 5849 ); 5850 5851 guest.reboot_linux(1, None); 5852 5853 // Check still absent after reboot 5854 assert_eq!( 5855 guest 5856 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5857 .unwrap() 5858 .trim() 5859 .parse::<u32>() 5860 .unwrap_or(1), 5861 0 5862 ); 5863 }); 5864 5865 kill_child(&mut child); 5866 let output = child.wait_with_output().unwrap(); 5867 5868 handle_child_output(r, &output); 5869 } 5870 5871 #[test] 5872 fn test_net_hotplug() { 5873 _test_net_hotplug(None) 5874 } 5875 5876 #[test] 5877 fn test_net_multi_segment_hotplug() { 5878 _test_net_hotplug(Some(15)) 5879 } 5880 5881 fn _test_net_hotplug(pci_segment: Option<u16>) { 5882 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5883 let guest = Guest::new(Box::new(focal)); 5884 5885 #[cfg(target_arch = "x86_64")] 5886 let kernel_path = direct_kernel_boot_path(); 5887 #[cfg(target_arch = "aarch64")] 5888 let kernel_path = edk2_path(); 5889 5890 let api_socket = temp_api_path(&guest.tmp_dir); 5891 5892 // Boot without network 5893 let mut cmd = GuestCommand::new(&guest); 5894 5895 cmd.args(["--api-socket", &api_socket]) 5896 .args(["--cpus", "boot=1"]) 5897 .args(["--memory", "size=512M"]) 5898 .args(["--kernel", kernel_path.to_str().unwrap()]) 5899 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5900 .default_disks() 5901 .capture_output(); 5902 5903 if pci_segment.is_some() { 5904 cmd.args([ 5905 "--platform", 5906 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5907 ]); 5908 } 5909 5910 let mut child = cmd.spawn().unwrap(); 5911 5912 thread::sleep(std::time::Duration::new(20, 0)); 5913 5914 let r = std::panic::catch_unwind(|| { 5915 // Add network 5916 let (cmd_success, cmd_output) = remote_command_w_output( 5917 &api_socket, 5918 "add-net", 5919 Some( 5920 format!( 5921 "{}{},id=test0", 5922 guest.default_net_string(), 5923 if let Some(pci_segment) = pci_segment { 5924 format!(",pci_segment={pci_segment}") 5925 } else { 5926 "".to_owned() 5927 } 5928 ) 5929 .as_str(), 5930 ), 5931 ); 5932 assert!(cmd_success); 5933 5934 if let Some(pci_segment) = pci_segment { 5935 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5936 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5937 ))); 5938 } else { 5939 assert!(String::from_utf8_lossy(&cmd_output) 5940 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5941 } 5942 5943 thread::sleep(std::time::Duration::new(5, 0)); 5944 5945 // 1 network interfaces + default localhost ==> 2 interfaces 5946 assert_eq!( 5947 guest 5948 .ssh_command("ip -o link | wc -l") 5949 .unwrap() 5950 .trim() 5951 .parse::<u32>() 5952 .unwrap_or_default(), 5953 2 5954 ); 5955 5956 // Remove network 5957 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5958 thread::sleep(std::time::Duration::new(5, 0)); 5959 5960 let (cmd_success, cmd_output) = remote_command_w_output( 5961 &api_socket, 5962 "add-net", 5963 Some( 5964 format!( 5965 "{}{},id=test1", 5966 guest.default_net_string(), 5967 if let Some(pci_segment) = pci_segment { 5968 format!(",pci_segment={pci_segment}") 5969 } else { 5970 "".to_owned() 5971 } 5972 ) 5973 .as_str(), 5974 ), 5975 ); 5976 assert!(cmd_success); 5977 5978 if let Some(pci_segment) = pci_segment { 5979 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5980 "{{\"id\":\"test1\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5981 ))); 5982 } else { 5983 assert!(String::from_utf8_lossy(&cmd_output) 5984 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5985 } 5986 5987 thread::sleep(std::time::Duration::new(5, 0)); 5988 5989 // 1 network interfaces + default localhost ==> 2 interfaces 5990 assert_eq!( 5991 guest 5992 .ssh_command("ip -o link | wc -l") 5993 .unwrap() 5994 .trim() 5995 .parse::<u32>() 5996 .unwrap_or_default(), 5997 2 5998 ); 5999 6000 guest.reboot_linux(0, None); 6001 6002 // Check still there after reboot 6003 // 1 network interfaces + default localhost ==> 2 interfaces 6004 assert_eq!( 6005 guest 6006 .ssh_command("ip -o link | wc -l") 6007 .unwrap() 6008 .trim() 6009 .parse::<u32>() 6010 .unwrap_or_default(), 6011 2 6012 ); 6013 }); 6014 6015 kill_child(&mut child); 6016 let output = child.wait_with_output().unwrap(); 6017 6018 handle_child_output(r, &output); 6019 } 6020 6021 #[test] 6022 fn test_initramfs() { 6023 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6024 let guest = Guest::new(Box::new(focal)); 6025 let mut workload_path = dirs::home_dir().unwrap(); 6026 workload_path.push("workloads"); 6027 6028 #[cfg(target_arch = "x86_64")] 6029 let mut kernels = vec![direct_kernel_boot_path()]; 6030 #[cfg(target_arch = "aarch64")] 6031 let kernels = [direct_kernel_boot_path()]; 6032 6033 #[cfg(target_arch = "x86_64")] 6034 { 6035 let mut pvh_kernel_path = workload_path.clone(); 6036 pvh_kernel_path.push("vmlinux"); 6037 kernels.push(pvh_kernel_path); 6038 } 6039 6040 let mut initramfs_path = workload_path; 6041 initramfs_path.push("alpine_initramfs.img"); 6042 6043 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 6044 let cmdline = format!("console=hvc0 quiet TEST_STRING={test_string}"); 6045 6046 kernels.iter().for_each(|k_path| { 6047 let mut child = GuestCommand::new(&guest) 6048 .args(["--kernel", k_path.to_str().unwrap()]) 6049 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 6050 .args(["--cmdline", &cmdline]) 6051 .capture_output() 6052 .spawn() 6053 .unwrap(); 6054 6055 thread::sleep(std::time::Duration::new(20, 0)); 6056 6057 kill_child(&mut child); 6058 let output = child.wait_with_output().unwrap(); 6059 6060 let r = std::panic::catch_unwind(|| { 6061 let s = String::from_utf8_lossy(&output.stdout); 6062 6063 assert_ne!(s.lines().position(|line| line == test_string), None); 6064 }); 6065 6066 handle_child_output(r, &output); 6067 }); 6068 } 6069 6070 #[test] 6071 fn test_counters() { 6072 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6073 let guest = Guest::new(Box::new(focal)); 6074 let api_socket = temp_api_path(&guest.tmp_dir); 6075 6076 let mut cmd = GuestCommand::new(&guest); 6077 cmd.args(["--cpus", "boot=1"]) 6078 .args(["--memory", "size=512M"]) 6079 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 6080 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6081 .default_disks() 6082 .args(["--net", guest.default_net_string().as_str()]) 6083 .args(["--api-socket", &api_socket]) 6084 .capture_output(); 6085 6086 let mut child = cmd.spawn().unwrap(); 6087 6088 let r = std::panic::catch_unwind(|| { 6089 guest.wait_vm_boot(None).unwrap(); 6090 6091 let orig_counters = get_counters(&api_socket); 6092 guest 6093 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 6094 .unwrap(); 6095 6096 let new_counters = get_counters(&api_socket); 6097 6098 // Check that all the counters have increased 6099 assert!(new_counters > orig_counters); 6100 }); 6101 6102 kill_child(&mut child); 6103 let output = child.wait_with_output().unwrap(); 6104 6105 handle_child_output(r, &output); 6106 } 6107 6108 #[test] 6109 #[cfg(feature = "guest_debug")] 6110 fn test_coredump() { 6111 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6112 let guest = Guest::new(Box::new(focal)); 6113 let api_socket = temp_api_path(&guest.tmp_dir); 6114 6115 let mut cmd = GuestCommand::new(&guest); 6116 cmd.args(["--cpus", "boot=4"]) 6117 .args(["--memory", "size=4G"]) 6118 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6119 .default_disks() 6120 .args(["--net", guest.default_net_string().as_str()]) 6121 .args(["--api-socket", &api_socket]) 6122 .capture_output(); 6123 6124 let mut child = cmd.spawn().unwrap(); 6125 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6126 6127 let r = std::panic::catch_unwind(|| { 6128 guest.wait_vm_boot(None).unwrap(); 6129 6130 assert!(remote_command(&api_socket, "pause", None)); 6131 6132 assert!(remote_command( 6133 &api_socket, 6134 "coredump", 6135 Some(format!("file://{vmcore_file}").as_str()), 6136 )); 6137 6138 // the num of CORE notes should equals to vcpu 6139 let readelf_core_num_cmd = 6140 format!("readelf --all {vmcore_file} |grep CORE |grep -v Type |wc -l"); 6141 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 6142 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 6143 6144 // the num of QEMU notes should equals to vcpu 6145 let readelf_vmm_num_cmd = format!("readelf --all {vmcore_file} |grep QEMU |wc -l"); 6146 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 6147 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 6148 }); 6149 6150 kill_child(&mut child); 6151 let output = child.wait_with_output().unwrap(); 6152 6153 handle_child_output(r, &output); 6154 } 6155 6156 #[test] 6157 #[cfg(feature = "guest_debug")] 6158 fn test_coredump_no_pause() { 6159 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6160 let guest = Guest::new(Box::new(focal)); 6161 let api_socket = temp_api_path(&guest.tmp_dir); 6162 6163 let mut cmd = GuestCommand::new(&guest); 6164 cmd.args(["--cpus", "boot=4"]) 6165 .args(["--memory", "size=4G"]) 6166 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6167 .default_disks() 6168 .args(["--net", guest.default_net_string().as_str()]) 6169 .args(["--api-socket", &api_socket]) 6170 .capture_output(); 6171 6172 let mut child = cmd.spawn().unwrap(); 6173 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6174 6175 let r = std::panic::catch_unwind(|| { 6176 guest.wait_vm_boot(None).unwrap(); 6177 6178 assert!(remote_command( 6179 &api_socket, 6180 "coredump", 6181 Some(format!("file://{vmcore_file}").as_str()), 6182 )); 6183 6184 assert_eq!(vm_state(&api_socket), "Running"); 6185 }); 6186 6187 kill_child(&mut child); 6188 let output = child.wait_with_output().unwrap(); 6189 6190 handle_child_output(r, &output); 6191 } 6192 6193 #[test] 6194 fn test_watchdog() { 6195 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6196 let guest = Guest::new(Box::new(focal)); 6197 let api_socket = temp_api_path(&guest.tmp_dir); 6198 6199 let kernel_path = direct_kernel_boot_path(); 6200 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6201 6202 let mut cmd = GuestCommand::new(&guest); 6203 cmd.args(["--cpus", "boot=1"]) 6204 .args(["--memory", "size=512M"]) 6205 .args(["--kernel", kernel_path.to_str().unwrap()]) 6206 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6207 .default_disks() 6208 .args(["--net", guest.default_net_string().as_str()]) 6209 .args(["--watchdog"]) 6210 .args(["--api-socket", &api_socket]) 6211 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6212 .capture_output(); 6213 6214 let mut child = cmd.spawn().unwrap(); 6215 6216 let r = std::panic::catch_unwind(|| { 6217 guest.wait_vm_boot(None).unwrap(); 6218 6219 let mut expected_reboot_count = 1; 6220 6221 // Enable the watchdog with a 15s timeout 6222 enable_guest_watchdog(&guest, 15); 6223 6224 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6225 assert_eq!( 6226 guest 6227 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6228 .unwrap() 6229 .trim() 6230 .parse::<u32>() 6231 .unwrap_or_default(), 6232 1 6233 ); 6234 6235 // Allow some normal time to elapse to check we don't get spurious reboots 6236 thread::sleep(std::time::Duration::new(40, 0)); 6237 // Check no reboot 6238 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6239 6240 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6241 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6242 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6243 guest.wait_vm_boot(Some(50)).unwrap(); 6244 // Check a reboot is triggered by the watchdog 6245 expected_reboot_count += 1; 6246 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6247 6248 #[cfg(target_arch = "x86_64")] 6249 { 6250 // Now pause the VM and remain offline for 30s 6251 assert!(remote_command(&api_socket, "pause", None)); 6252 let latest_events = [ 6253 &MetaEvent { 6254 event: "pausing".to_string(), 6255 device_id: None, 6256 }, 6257 &MetaEvent { 6258 event: "paused".to_string(), 6259 device_id: None, 6260 }, 6261 ]; 6262 assert!(check_latest_events_exact(&latest_events, &event_path)); 6263 assert!(remote_command(&api_socket, "resume", None)); 6264 6265 // Check no reboot 6266 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6267 } 6268 }); 6269 6270 kill_child(&mut child); 6271 let output = child.wait_with_output().unwrap(); 6272 6273 handle_child_output(r, &output); 6274 } 6275 6276 #[test] 6277 fn test_pvpanic() { 6278 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 6279 let guest = Guest::new(Box::new(jammy)); 6280 let api_socket = temp_api_path(&guest.tmp_dir); 6281 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6282 6283 let kernel_path = direct_kernel_boot_path(); 6284 6285 let mut cmd = GuestCommand::new(&guest); 6286 cmd.args(["--cpus", "boot=1"]) 6287 .args(["--memory", "size=512M"]) 6288 .args(["--kernel", kernel_path.to_str().unwrap()]) 6289 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6290 .default_disks() 6291 .args(["--net", guest.default_net_string().as_str()]) 6292 .args(["--pvpanic"]) 6293 .args(["--api-socket", &api_socket]) 6294 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6295 .capture_output(); 6296 6297 let mut child = cmd.spawn().unwrap(); 6298 6299 let r = std::panic::catch_unwind(|| { 6300 guest.wait_vm_boot(None).unwrap(); 6301 6302 // Trigger guest a panic 6303 make_guest_panic(&guest); 6304 6305 // Wait a while for guest 6306 thread::sleep(std::time::Duration::new(10, 0)); 6307 6308 let expected_sequential_events = [&MetaEvent { 6309 event: "panic".to_string(), 6310 device_id: None, 6311 }]; 6312 assert!(check_latest_events_exact( 6313 &expected_sequential_events, 6314 &event_path 6315 )); 6316 }); 6317 6318 kill_child(&mut child); 6319 let output = child.wait_with_output().unwrap(); 6320 6321 handle_child_output(r, &output); 6322 } 6323 6324 #[test] 6325 fn test_tap_from_fd() { 6326 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6327 let guest = Guest::new(Box::new(focal)); 6328 let kernel_path = direct_kernel_boot_path(); 6329 6330 // Create a TAP interface with multi-queue enabled 6331 let num_queue_pairs: usize = 2; 6332 6333 use std::str::FromStr; 6334 let taps = net_util::open_tap( 6335 Some("chtap0"), 6336 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6337 None, 6338 &mut None, 6339 None, 6340 num_queue_pairs, 6341 Some(libc::O_RDWR | libc::O_NONBLOCK), 6342 ) 6343 .unwrap(); 6344 6345 let mut child = GuestCommand::new(&guest) 6346 .args(["--cpus", &format!("boot={num_queue_pairs}")]) 6347 .args(["--memory", "size=512M"]) 6348 .args(["--kernel", kernel_path.to_str().unwrap()]) 6349 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6350 .default_disks() 6351 .args([ 6352 "--net", 6353 &format!( 6354 "fd=[{},{}],mac={},num_queues={}", 6355 taps[0].as_raw_fd(), 6356 taps[1].as_raw_fd(), 6357 guest.network.guest_mac, 6358 num_queue_pairs * 2 6359 ), 6360 ]) 6361 .capture_output() 6362 .spawn() 6363 .unwrap(); 6364 6365 let r = std::panic::catch_unwind(|| { 6366 guest.wait_vm_boot(None).unwrap(); 6367 6368 assert_eq!( 6369 guest 6370 .ssh_command("ip -o link | wc -l") 6371 .unwrap() 6372 .trim() 6373 .parse::<u32>() 6374 .unwrap_or_default(), 6375 2 6376 ); 6377 6378 guest.reboot_linux(0, None); 6379 6380 assert_eq!( 6381 guest 6382 .ssh_command("ip -o link | wc -l") 6383 .unwrap() 6384 .trim() 6385 .parse::<u32>() 6386 .unwrap_or_default(), 6387 2 6388 ); 6389 }); 6390 6391 kill_child(&mut child); 6392 let output = child.wait_with_output().unwrap(); 6393 6394 handle_child_output(r, &output); 6395 } 6396 6397 // By design, a guest VM won't be able to connect to the host 6398 // machine when using a macvtap network interface (while it can 6399 // communicate externally). As a workaround, this integration 6400 // test creates two macvtap interfaces in 'bridge' mode on the 6401 // same physical net interface, one for the guest and one for 6402 // the host. With additional setup on the IP address and the 6403 // routing table, it enables the communications between the 6404 // guest VM and the host machine. 6405 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6406 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6407 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6408 let guest = Guest::new(Box::new(focal)); 6409 let api_socket = temp_api_path(&guest.tmp_dir); 6410 6411 #[cfg(target_arch = "x86_64")] 6412 let kernel_path = direct_kernel_boot_path(); 6413 #[cfg(target_arch = "aarch64")] 6414 let kernel_path = edk2_path(); 6415 6416 let phy_net = "eth0"; 6417 6418 // Create a macvtap interface for the guest VM to use 6419 assert!(exec_host_command_status(&format!( 6420 "sudo ip link add link {phy_net} name {guest_macvtap_name} type macvtap mod bridge" 6421 )) 6422 .success()); 6423 assert!(exec_host_command_status(&format!( 6424 "sudo ip link set {} address {} up", 6425 guest_macvtap_name, guest.network.guest_mac 6426 )) 6427 .success()); 6428 assert!( 6429 exec_host_command_status(&format!("sudo ip link show {guest_macvtap_name}")).success() 6430 ); 6431 6432 let tap_index = 6433 fs::read_to_string(format!("/sys/class/net/{guest_macvtap_name}/ifindex")).unwrap(); 6434 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6435 6436 assert!(exec_host_command_status(&format!("sudo chown $UID.$UID {tap_device}")).success()); 6437 6438 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6439 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6440 assert!(tap_fd1 > 0); 6441 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6442 assert!(tap_fd2 > 0); 6443 6444 // Create a macvtap on the same physical net interface for 6445 // the host machine to use 6446 assert!(exec_host_command_status(&format!( 6447 "sudo ip link add link {phy_net} name {host_macvtap_name} type macvtap mod bridge" 6448 )) 6449 .success()); 6450 // Use default mask "255.255.255.0" 6451 assert!(exec_host_command_status(&format!( 6452 "sudo ip address add {}/24 dev {}", 6453 guest.network.host_ip, host_macvtap_name 6454 )) 6455 .success()); 6456 assert!( 6457 exec_host_command_status(&format!("sudo ip link set dev {host_macvtap_name} up")) 6458 .success() 6459 ); 6460 6461 let mut guest_command = GuestCommand::new(&guest); 6462 guest_command 6463 .args(["--cpus", "boot=2"]) 6464 .args(["--memory", "size=512M"]) 6465 .args(["--kernel", kernel_path.to_str().unwrap()]) 6466 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6467 .default_disks() 6468 .args(["--api-socket", &api_socket]); 6469 6470 let net_params = format!( 6471 "fd=[{},{}],mac={},num_queues=4", 6472 tap_fd1, tap_fd2, guest.network.guest_mac 6473 ); 6474 6475 if !hotplug { 6476 guest_command.args(["--net", &net_params]); 6477 } 6478 6479 let mut child = guest_command.capture_output().spawn().unwrap(); 6480 6481 if hotplug { 6482 // Give some time to the VMM process to listen to the API 6483 // socket. This is the only requirement to avoid the following 6484 // call to ch-remote from failing. 6485 thread::sleep(std::time::Duration::new(10, 0)); 6486 // Hotplug the virtio-net device 6487 let (cmd_success, cmd_output) = 6488 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6489 assert!(cmd_success); 6490 #[cfg(target_arch = "x86_64")] 6491 assert!(String::from_utf8_lossy(&cmd_output) 6492 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6493 #[cfg(target_arch = "aarch64")] 6494 assert!(String::from_utf8_lossy(&cmd_output) 6495 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6496 } 6497 6498 // The functional connectivity provided by the virtio-net device 6499 // gets tested through wait_vm_boot() as it expects to receive a 6500 // HTTP request, and through the SSH command as well. 6501 let r = std::panic::catch_unwind(|| { 6502 guest.wait_vm_boot(None).unwrap(); 6503 6504 assert_eq!( 6505 guest 6506 .ssh_command("ip -o link | wc -l") 6507 .unwrap() 6508 .trim() 6509 .parse::<u32>() 6510 .unwrap_or_default(), 6511 2 6512 ); 6513 6514 guest.reboot_linux(0, None); 6515 6516 assert_eq!( 6517 guest 6518 .ssh_command("ip -o link | wc -l") 6519 .unwrap() 6520 .trim() 6521 .parse::<u32>() 6522 .unwrap_or_default(), 6523 2 6524 ); 6525 }); 6526 6527 kill_child(&mut child); 6528 6529 exec_host_command_status(&format!("sudo ip link del {guest_macvtap_name}")); 6530 exec_host_command_status(&format!("sudo ip link del {host_macvtap_name}")); 6531 6532 let output = child.wait_with_output().unwrap(); 6533 6534 handle_child_output(r, &output); 6535 } 6536 6537 #[test] 6538 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6539 fn test_macvtap() { 6540 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6541 } 6542 6543 #[test] 6544 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6545 fn test_macvtap_hotplug() { 6546 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6547 } 6548 6549 #[test] 6550 #[cfg(not(feature = "mshv"))] 6551 fn test_ovs_dpdk() { 6552 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6553 let guest1 = Guest::new(Box::new(focal1)); 6554 6555 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6556 let guest2 = Guest::new(Box::new(focal2)); 6557 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6558 6559 let (mut child1, mut child2) = 6560 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6561 6562 // Create the snapshot directory 6563 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6564 6565 let r = std::panic::catch_unwind(|| { 6566 // Remove one of the two ports from the OVS bridge 6567 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6568 6569 // Spawn a new netcat listener in the first VM 6570 let guest_ip = guest1.network.guest_ip.clone(); 6571 thread::spawn(move || { 6572 ssh_command_ip( 6573 "nc -l 12345", 6574 &guest_ip, 6575 DEFAULT_SSH_RETRIES, 6576 DEFAULT_SSH_TIMEOUT, 6577 ) 6578 .unwrap(); 6579 }); 6580 6581 // Wait for the server to be listening 6582 thread::sleep(std::time::Duration::new(5, 0)); 6583 6584 // Check the connection fails this time 6585 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap_err(); 6586 6587 // Add the OVS port back 6588 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6589 6590 // And finally check the connection is functional again 6591 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6592 6593 // Pause the VM 6594 assert!(remote_command(&api_socket_source, "pause", None)); 6595 6596 // Take a snapshot from the VM 6597 assert!(remote_command( 6598 &api_socket_source, 6599 "snapshot", 6600 Some(format!("file://{snapshot_dir}").as_str()), 6601 )); 6602 6603 // Wait to make sure the snapshot is completed 6604 thread::sleep(std::time::Duration::new(10, 0)); 6605 }); 6606 6607 // Shutdown the source VM 6608 kill_child(&mut child2); 6609 let output = child2.wait_with_output().unwrap(); 6610 handle_child_output(r, &output); 6611 6612 // Remove the vhost-user socket file. 6613 Command::new("rm") 6614 .arg("-f") 6615 .arg("/tmp/dpdkvhostclient2") 6616 .output() 6617 .unwrap(); 6618 6619 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6620 // Restore the VM from the snapshot 6621 let mut child2 = GuestCommand::new(&guest2) 6622 .args(["--api-socket", &api_socket_restored]) 6623 .args([ 6624 "--restore", 6625 format!("source_url=file://{snapshot_dir}").as_str(), 6626 ]) 6627 .capture_output() 6628 .spawn() 6629 .unwrap(); 6630 6631 // Wait for the VM to be restored 6632 thread::sleep(std::time::Duration::new(10, 0)); 6633 6634 let r = std::panic::catch_unwind(|| { 6635 // Resume the VM 6636 assert!(remote_command(&api_socket_restored, "resume", None)); 6637 6638 // Spawn a new netcat listener in the first VM 6639 let guest_ip = guest1.network.guest_ip.clone(); 6640 thread::spawn(move || { 6641 ssh_command_ip( 6642 "nc -l 12345", 6643 &guest_ip, 6644 DEFAULT_SSH_RETRIES, 6645 DEFAULT_SSH_TIMEOUT, 6646 ) 6647 .unwrap(); 6648 }); 6649 6650 // Wait for the server to be listening 6651 thread::sleep(std::time::Duration::new(5, 0)); 6652 6653 // And check the connection is still functional after restore 6654 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6655 }); 6656 6657 kill_child(&mut child1); 6658 kill_child(&mut child2); 6659 6660 let output = child1.wait_with_output().unwrap(); 6661 child2.wait().unwrap(); 6662 6663 cleanup_ovs_dpdk(); 6664 6665 handle_child_output(r, &output); 6666 } 6667 6668 fn setup_spdk_nvme(nvme_dir: &std::path::Path) -> Child { 6669 cleanup_spdk_nvme(); 6670 6671 assert!(exec_host_command_status(&format!( 6672 "mkdir -p {}", 6673 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6674 )) 6675 .success()); 6676 assert!(exec_host_command_status(&format!( 6677 "truncate {} -s 128M", 6678 nvme_dir.join("test-disk.raw").to_str().unwrap() 6679 )) 6680 .success()); 6681 assert!(exec_host_command_status(&format!( 6682 "mkfs.ext4 {}", 6683 nvme_dir.join("test-disk.raw").to_str().unwrap() 6684 )) 6685 .success()); 6686 6687 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6688 let child = Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6689 .args(["-i", "0", "-m", "0x1"]) 6690 .spawn() 6691 .unwrap(); 6692 thread::sleep(std::time::Duration::new(2, 0)); 6693 6694 assert!(exec_host_command_with_retries( 6695 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER", 6696 3, 6697 std::time::Duration::new(5, 0), 6698 )); 6699 assert!(exec_host_command_status(&format!( 6700 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6701 nvme_dir.join("test-disk.raw").to_str().unwrap() 6702 )) 6703 .success()); 6704 assert!(exec_host_command_status( 6705 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6706 ) 6707 .success()); 6708 assert!(exec_host_command_status( 6709 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6710 ) 6711 .success()); 6712 assert!(exec_host_command_status(&format!( 6713 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6714 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6715 )) 6716 .success()); 6717 6718 child 6719 } 6720 6721 fn cleanup_spdk_nvme() { 6722 exec_host_command_status("pkill -f nvmf_tgt"); 6723 } 6724 6725 #[test] 6726 fn test_vfio_user() { 6727 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6728 let jammy = UbuntuDiskConfig::new(jammy_image); 6729 let guest = Guest::new(Box::new(jammy)); 6730 6731 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6732 let mut spdk_child = setup_spdk_nvme(spdk_nvme_dir.as_path()); 6733 6734 let api_socket = temp_api_path(&guest.tmp_dir); 6735 let mut child = GuestCommand::new(&guest) 6736 .args(["--api-socket", &api_socket]) 6737 .args(["--cpus", "boot=1"]) 6738 .args(["--memory", "size=1G,shared=on,hugepages=on"]) 6739 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6740 .args(["--serial", "tty", "--console", "off"]) 6741 .default_disks() 6742 .default_net() 6743 .capture_output() 6744 .spawn() 6745 .unwrap(); 6746 6747 let r = std::panic::catch_unwind(|| { 6748 guest.wait_vm_boot(None).unwrap(); 6749 6750 // Hotplug the SPDK-NVMe device to the VM 6751 let (cmd_success, cmd_output) = remote_command_w_output( 6752 &api_socket, 6753 "add-user-device", 6754 Some(&format!( 6755 "socket={},id=vfio_user0", 6756 spdk_nvme_dir 6757 .as_path() 6758 .join("nvme-vfio-user/cntrl") 6759 .to_str() 6760 .unwrap(), 6761 )), 6762 ); 6763 assert!(cmd_success); 6764 assert!(String::from_utf8_lossy(&cmd_output) 6765 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6766 6767 thread::sleep(std::time::Duration::new(10, 0)); 6768 6769 // Check both if /dev/nvme exists and if the block size is 128M. 6770 assert_eq!( 6771 guest 6772 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6773 .unwrap() 6774 .trim() 6775 .parse::<u32>() 6776 .unwrap_or_default(), 6777 1 6778 ); 6779 6780 // Check changes persist after reboot 6781 assert_eq!( 6782 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6783 "" 6784 ); 6785 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6786 guest 6787 .ssh_command("echo test123 | sudo tee /mnt/test") 6788 .unwrap(); 6789 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6790 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6791 6792 guest.reboot_linux(0, None); 6793 assert_eq!( 6794 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6795 "" 6796 ); 6797 assert_eq!( 6798 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6799 "test123" 6800 ); 6801 }); 6802 6803 let _ = spdk_child.kill(); 6804 let _ = spdk_child.wait(); 6805 6806 kill_child(&mut child); 6807 let output = child.wait_with_output().unwrap(); 6808 6809 handle_child_output(r, &output); 6810 } 6811 6812 #[test] 6813 #[cfg(target_arch = "x86_64")] 6814 fn test_vdpa_block() { 6815 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6816 assert!(exec_host_command_status("lsmod | grep vdpa_sim_blk").success()); 6817 6818 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6819 let guest = Guest::new(Box::new(focal)); 6820 let api_socket = temp_api_path(&guest.tmp_dir); 6821 6822 let kernel_path = direct_kernel_boot_path(); 6823 6824 let mut child = GuestCommand::new(&guest) 6825 .args(["--cpus", "boot=2"]) 6826 .args(["--memory", "size=512M,hugepages=on"]) 6827 .args(["--kernel", kernel_path.to_str().unwrap()]) 6828 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6829 .default_disks() 6830 .default_net() 6831 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6832 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 6833 .args(["--api-socket", &api_socket]) 6834 .capture_output() 6835 .spawn() 6836 .unwrap(); 6837 6838 let r = std::panic::catch_unwind(|| { 6839 guest.wait_vm_boot(None).unwrap(); 6840 6841 // Check both if /dev/vdc exists and if the block size is 128M. 6842 assert_eq!( 6843 guest 6844 .ssh_command("lsblk | grep vdc | grep -c 128M") 6845 .unwrap() 6846 .trim() 6847 .parse::<u32>() 6848 .unwrap_or_default(), 6849 1 6850 ); 6851 6852 // Check the content of the block device after we wrote to it. 6853 // The vpda-sim-blk should let us read what we previously wrote. 6854 guest 6855 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6856 .unwrap(); 6857 assert_eq!( 6858 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6859 "foobar" 6860 ); 6861 6862 // Hotplug an extra vDPA block device behind the vIOMMU 6863 // Add a new vDPA device to the VM 6864 let (cmd_success, cmd_output) = remote_command_w_output( 6865 &api_socket, 6866 "add-vdpa", 6867 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6868 ); 6869 assert!(cmd_success); 6870 assert!(String::from_utf8_lossy(&cmd_output) 6871 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6872 6873 thread::sleep(std::time::Duration::new(10, 0)); 6874 6875 // Check IOMMU setup 6876 assert!(guest 6877 .does_device_vendor_pair_match("0x1057", "0x1af4") 6878 .unwrap_or_default()); 6879 assert_eq!( 6880 guest 6881 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6882 .unwrap() 6883 .trim(), 6884 "0001:00:01.0" 6885 ); 6886 6887 // Check both if /dev/vdd exists and if the block size is 128M. 6888 assert_eq!( 6889 guest 6890 .ssh_command("lsblk | grep vdd | grep -c 128M") 6891 .unwrap() 6892 .trim() 6893 .parse::<u32>() 6894 .unwrap_or_default(), 6895 1 6896 ); 6897 6898 // Write some content to the block device we've just plugged. 6899 guest 6900 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6901 .unwrap(); 6902 6903 // Check we can read the content back. 6904 assert_eq!( 6905 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6906 "foobar" 6907 ); 6908 6909 // Unplug the device 6910 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6911 assert!(cmd_success); 6912 thread::sleep(std::time::Duration::new(10, 0)); 6913 6914 // Check /dev/vdd doesn't exist anymore 6915 assert_eq!( 6916 guest 6917 .ssh_command("lsblk | grep -c vdd || true") 6918 .unwrap() 6919 .trim() 6920 .parse::<u32>() 6921 .unwrap_or(1), 6922 0 6923 ); 6924 }); 6925 6926 kill_child(&mut child); 6927 let output = child.wait_with_output().unwrap(); 6928 6929 handle_child_output(r, &output); 6930 } 6931 6932 #[test] 6933 #[cfg(target_arch = "x86_64")] 6934 #[ignore = "See #5756"] 6935 fn test_vdpa_net() { 6936 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6937 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6938 return; 6939 } 6940 6941 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6942 let guest = Guest::new(Box::new(focal)); 6943 6944 let kernel_path = direct_kernel_boot_path(); 6945 6946 let mut child = GuestCommand::new(&guest) 6947 .args(["--cpus", "boot=2"]) 6948 .args(["--memory", "size=512M,hugepages=on"]) 6949 .args(["--kernel", kernel_path.to_str().unwrap()]) 6950 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6951 .default_disks() 6952 .default_net() 6953 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 6954 .capture_output() 6955 .spawn() 6956 .unwrap(); 6957 6958 let r = std::panic::catch_unwind(|| { 6959 guest.wait_vm_boot(None).unwrap(); 6960 6961 // Check we can find network interface related to vDPA device 6962 assert_eq!( 6963 guest 6964 .ssh_command("ip -o link | grep -c ens6") 6965 .unwrap() 6966 .trim() 6967 .parse::<u32>() 6968 .unwrap_or(0), 6969 1 6970 ); 6971 6972 guest 6973 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 6974 .unwrap(); 6975 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 6976 6977 // Check there is no packet yet on both TX/RX of the network interface 6978 assert_eq!( 6979 guest 6980 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 6981 .unwrap() 6982 .trim() 6983 .parse::<u32>() 6984 .unwrap_or(0), 6985 2 6986 ); 6987 6988 // Send 6 packets with ping command 6989 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 6990 6991 // Check we can find 6 packets on both TX/RX of the network interface 6992 assert_eq!( 6993 guest 6994 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 6995 .unwrap() 6996 .trim() 6997 .parse::<u32>() 6998 .unwrap_or(0), 6999 2 7000 ); 7001 7002 // No need to check for hotplug as we already tested it through 7003 // test_vdpa_block() 7004 }); 7005 7006 kill_child(&mut child); 7007 let output = child.wait_with_output().unwrap(); 7008 7009 handle_child_output(r, &output); 7010 } 7011 7012 #[test] 7013 #[cfg(target_arch = "x86_64")] 7014 fn test_tpm() { 7015 let focal = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7016 let guest = Guest::new(Box::new(focal)); 7017 7018 let (mut swtpm_command, swtpm_socket_path) = prepare_swtpm_daemon(&guest.tmp_dir); 7019 7020 let mut guest_cmd = GuestCommand::new(&guest); 7021 guest_cmd 7022 .args(["--cpus", "boot=1"]) 7023 .args(["--memory", "size=1G"]) 7024 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7025 .args(["--tpm", &format!("socket={swtpm_socket_path}")]) 7026 .capture_output() 7027 .default_disks() 7028 .default_net(); 7029 7030 // Start swtpm daemon 7031 let mut swtpm_child = swtpm_command.spawn().unwrap(); 7032 thread::sleep(std::time::Duration::new(10, 0)); 7033 let mut child = guest_cmd.spawn().unwrap(); 7034 let r = std::panic::catch_unwind(|| { 7035 guest.wait_vm_boot(None).unwrap(); 7036 assert_eq!( 7037 guest.ssh_command("ls /dev/tpm0").unwrap().trim(), 7038 "/dev/tpm0" 7039 ); 7040 guest.ssh_command("sudo tpm2_selftest -f").unwrap(); 7041 guest 7042 .ssh_command("echo 'hello' > /tmp/checksum_test; ") 7043 .unwrap(); 7044 guest.ssh_command("cmp <(sudo tpm2_pcrevent /tmp/checksum_test | grep sha256 | awk '{print $2}') <(sha256sum /tmp/checksum_test| awk '{print $1}')").unwrap(); 7045 }); 7046 7047 let _ = swtpm_child.kill(); 7048 let _d_out = swtpm_child.wait_with_output().unwrap(); 7049 7050 kill_child(&mut child); 7051 let output = child.wait_with_output().unwrap(); 7052 7053 handle_child_output(r, &output); 7054 } 7055 7056 #[test] 7057 #[cfg(target_arch = "x86_64")] 7058 fn test_double_tty() { 7059 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7060 let guest = Guest::new(Box::new(focal)); 7061 let mut cmd = GuestCommand::new(&guest); 7062 let api_socket = temp_api_path(&guest.tmp_dir); 7063 let tty_str: &str = "console=hvc0 earlyprintk=ttyS0 "; 7064 // linux printk module enable console log. 7065 let con_dis_str: &str = "console [hvc0] enabled"; 7066 // linux printk module disable console log. 7067 let con_enb_str: &str = "bootconsole [earlyser0] disabled"; 7068 7069 let kernel_path = direct_kernel_boot_path(); 7070 7071 cmd.args(["--cpus", "boot=1"]) 7072 .args(["--memory", "size=512M"]) 7073 .args(["--kernel", kernel_path.to_str().unwrap()]) 7074 .args([ 7075 "--cmdline", 7076 DIRECT_KERNEL_BOOT_CMDLINE 7077 .replace("console=hvc0 ", tty_str) 7078 .as_str(), 7079 ]) 7080 .capture_output() 7081 .default_disks() 7082 .default_net() 7083 .args(["--serial", "tty"]) 7084 .args(["--console", "tty"]) 7085 .args(["--api-socket", &api_socket]); 7086 7087 let mut child = cmd.spawn().unwrap(); 7088 7089 let mut r = std::panic::catch_unwind(|| { 7090 guest.wait_vm_boot(None).unwrap(); 7091 }); 7092 7093 kill_child(&mut child); 7094 let output = child.wait_with_output().unwrap(); 7095 7096 if r.is_ok() { 7097 r = std::panic::catch_unwind(|| { 7098 let s = String::from_utf8_lossy(&output.stdout); 7099 assert!(s.contains(tty_str)); 7100 assert!(s.contains(con_dis_str)); 7101 assert!(s.contains(con_enb_str)); 7102 }); 7103 } 7104 7105 handle_child_output(r, &output); 7106 } 7107 7108 #[test] 7109 #[cfg(target_arch = "x86_64")] 7110 fn test_nmi() { 7111 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7112 let guest = Guest::new(Box::new(jammy)); 7113 let api_socket = temp_api_path(&guest.tmp_dir); 7114 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7115 7116 let kernel_path = direct_kernel_boot_path(); 7117 let cmd_line = format!("{} {}", DIRECT_KERNEL_BOOT_CMDLINE, "unknown_nmi_panic=1"); 7118 7119 let mut cmd = GuestCommand::new(&guest); 7120 cmd.args(["--cpus", "boot=4"]) 7121 .args(["--memory", "size=512M"]) 7122 .args(["--kernel", kernel_path.to_str().unwrap()]) 7123 .args(["--cmdline", cmd_line.as_str()]) 7124 .default_disks() 7125 .args(["--net", guest.default_net_string().as_str()]) 7126 .args(["--pvpanic"]) 7127 .args(["--api-socket", &api_socket]) 7128 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7129 .capture_output(); 7130 7131 let mut child = cmd.spawn().unwrap(); 7132 7133 let r = std::panic::catch_unwind(|| { 7134 guest.wait_vm_boot(None).unwrap(); 7135 7136 assert!(remote_command(&api_socket, "nmi", None)); 7137 7138 // Wait a while for guest 7139 thread::sleep(std::time::Duration::new(3, 0)); 7140 7141 let expected_sequential_events = [&MetaEvent { 7142 event: "panic".to_string(), 7143 device_id: None, 7144 }]; 7145 assert!(check_latest_events_exact( 7146 &expected_sequential_events, 7147 &event_path 7148 )); 7149 }); 7150 7151 kill_child(&mut child); 7152 let output = child.wait_with_output().unwrap(); 7153 7154 handle_child_output(r, &output); 7155 } 7156 } 7157 7158 mod dbus_api { 7159 use crate::*; 7160 7161 // Start cloud-hypervisor with no VM parameters, running both the HTTP 7162 // and DBus APIs. Alternate calls to the external APIs (HTTP and DBus) 7163 // to create a VM, boot it, and verify that it can be shut down and then 7164 // booted again. 7165 #[test] 7166 fn test_api_dbus_and_http_interleaved() { 7167 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7168 let guest = Guest::new(Box::new(focal)); 7169 let dbus_api = TargetApi::new_dbus_api(&guest.tmp_dir); 7170 let http_api = TargetApi::new_http_api(&guest.tmp_dir); 7171 7172 let mut child = GuestCommand::new(&guest) 7173 .args(dbus_api.guest_args()) 7174 .args(http_api.guest_args()) 7175 .capture_output() 7176 .spawn() 7177 .unwrap(); 7178 7179 thread::sleep(std::time::Duration::new(1, 0)); 7180 7181 // Verify API servers are running 7182 assert!(dbus_api.remote_command("ping", None)); 7183 assert!(http_api.remote_command("ping", None)); 7184 7185 // Create the VM first 7186 let cpu_count: u8 = 4; 7187 let request_body = guest.api_create_body( 7188 cpu_count, 7189 direct_kernel_boot_path().to_str().unwrap(), 7190 DIRECT_KERNEL_BOOT_CMDLINE, 7191 ); 7192 7193 let temp_config_path = guest.tmp_dir.as_path().join("config"); 7194 std::fs::write(&temp_config_path, request_body).unwrap(); 7195 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 7196 7197 let r = std::panic::catch_unwind(|| { 7198 // Create the VM 7199 assert!(dbus_api.remote_command("create", Some(create_config),)); 7200 7201 // Then boot it 7202 assert!(http_api.remote_command("boot", None)); 7203 guest.wait_vm_boot(None).unwrap(); 7204 7205 // Check that the VM booted as expected 7206 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7207 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7208 7209 // Sync and shutdown without powering off to prevent filesystem 7210 // corruption. 7211 guest.ssh_command("sync").unwrap(); 7212 guest.ssh_command("sudo shutdown -H now").unwrap(); 7213 7214 // Wait for the guest to be fully shutdown 7215 thread::sleep(std::time::Duration::new(20, 0)); 7216 7217 // Then shutdown the VM 7218 assert!(dbus_api.remote_command("shutdown", None)); 7219 7220 // Then boot it again 7221 assert!(http_api.remote_command("boot", None)); 7222 guest.wait_vm_boot(None).unwrap(); 7223 7224 // Check that the VM booted as expected 7225 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7226 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7227 }); 7228 7229 kill_child(&mut child); 7230 let output = child.wait_with_output().unwrap(); 7231 7232 handle_child_output(r, &output); 7233 } 7234 7235 #[test] 7236 fn test_api_dbus_create_boot() { 7237 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7238 let guest = Guest::new(Box::new(focal)); 7239 7240 _test_api_create_boot(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7241 } 7242 7243 #[test] 7244 fn test_api_dbus_shutdown() { 7245 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7246 let guest = Guest::new(Box::new(focal)); 7247 7248 _test_api_shutdown(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7249 } 7250 7251 #[test] 7252 fn test_api_dbus_delete() { 7253 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7254 let guest = Guest::new(Box::new(focal)); 7255 7256 _test_api_delete(TargetApi::new_dbus_api(&guest.tmp_dir), guest); 7257 } 7258 7259 #[test] 7260 fn test_api_dbus_pause_resume() { 7261 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7262 let guest = Guest::new(Box::new(focal)); 7263 7264 _test_api_pause_resume(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7265 } 7266 } 7267 7268 mod common_sequential { 7269 use std::fs::remove_dir_all; 7270 7271 use crate::*; 7272 7273 #[test] 7274 #[cfg(not(feature = "mshv"))] 7275 fn test_memory_mergeable_on() { 7276 test_memory_mergeable(true) 7277 } 7278 7279 fn snapshot_and_check_events(api_socket: &str, snapshot_dir: &str, event_path: &str) { 7280 // Pause the VM 7281 assert!(remote_command(api_socket, "pause", None)); 7282 let latest_events: [&MetaEvent; 2] = [ 7283 &MetaEvent { 7284 event: "pausing".to_string(), 7285 device_id: None, 7286 }, 7287 &MetaEvent { 7288 event: "paused".to_string(), 7289 device_id: None, 7290 }, 7291 ]; 7292 // See: #5938 7293 thread::sleep(std::time::Duration::new(1, 0)); 7294 assert!(check_latest_events_exact(&latest_events, event_path)); 7295 7296 // Take a snapshot from the VM 7297 assert!(remote_command( 7298 api_socket, 7299 "snapshot", 7300 Some(format!("file://{snapshot_dir}").as_str()), 7301 )); 7302 7303 // Wait to make sure the snapshot is completed 7304 thread::sleep(std::time::Duration::new(10, 0)); 7305 7306 let latest_events = [ 7307 &MetaEvent { 7308 event: "snapshotting".to_string(), 7309 device_id: None, 7310 }, 7311 &MetaEvent { 7312 event: "snapshotted".to_string(), 7313 device_id: None, 7314 }, 7315 ]; 7316 // See: #5938 7317 thread::sleep(std::time::Duration::new(1, 0)); 7318 assert!(check_latest_events_exact(&latest_events, event_path)); 7319 } 7320 7321 // One thing to note about this test. The virtio-net device is heavily used 7322 // through each ssh command. There's no need to perform a dedicated test to 7323 // verify the migration went well for virtio-net. 7324 #[test] 7325 #[cfg(not(feature = "mshv"))] 7326 fn test_snapshot_restore_hotplug_virtiomem() { 7327 _test_snapshot_restore(true); 7328 } 7329 7330 #[test] 7331 fn test_snapshot_restore_basic() { 7332 _test_snapshot_restore(false); 7333 } 7334 7335 fn _test_snapshot_restore(use_hotplug: bool) { 7336 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7337 let guest = Guest::new(Box::new(focal)); 7338 let kernel_path = direct_kernel_boot_path(); 7339 7340 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7341 7342 let net_id = "net123"; 7343 let net_params = format!( 7344 "id={},tap=,mac={},ip={},mask=255.255.255.0", 7345 net_id, guest.network.guest_mac, guest.network.host_ip 7346 ); 7347 let mut mem_params = "size=2G"; 7348 7349 if use_hotplug { 7350 mem_params = "size=2G,hotplug_method=virtio-mem,hotplug_size=32G" 7351 } 7352 7353 let cloudinit_params = format!( 7354 "path={},iommu=on", 7355 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7356 ); 7357 7358 let socket = temp_vsock_path(&guest.tmp_dir); 7359 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7360 7361 let mut child = GuestCommand::new(&guest) 7362 .args(["--api-socket", &api_socket_source]) 7363 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7364 .args(["--cpus", "boot=4"]) 7365 .args(["--memory", mem_params]) 7366 .args(["--balloon", "size=0"]) 7367 .args(["--kernel", kernel_path.to_str().unwrap()]) 7368 .args([ 7369 "--disk", 7370 format!( 7371 "path={}", 7372 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7373 ) 7374 .as_str(), 7375 cloudinit_params.as_str(), 7376 ]) 7377 .args(["--net", net_params.as_str()]) 7378 .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) 7379 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7380 .capture_output() 7381 .spawn() 7382 .unwrap(); 7383 7384 let console_text = String::from("On a branch floating down river a cricket, singing."); 7385 // Create the snapshot directory 7386 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7387 7388 let r = std::panic::catch_unwind(|| { 7389 guest.wait_vm_boot(None).unwrap(); 7390 7391 // Check the number of vCPUs 7392 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 7393 // Check the guest RAM 7394 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 7395 if use_hotplug { 7396 // Increase guest RAM with virtio-mem 7397 resize_command( 7398 &api_socket_source, 7399 None, 7400 Some(6 << 30), 7401 None, 7402 Some(&event_path), 7403 ); 7404 thread::sleep(std::time::Duration::new(5, 0)); 7405 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7406 // Use balloon to remove RAM from the VM 7407 resize_command( 7408 &api_socket_source, 7409 None, 7410 None, 7411 Some(1 << 30), 7412 Some(&event_path), 7413 ); 7414 thread::sleep(std::time::Duration::new(5, 0)); 7415 let total_memory = guest.get_total_memory().unwrap_or_default(); 7416 assert!(total_memory > 4_800_000); 7417 assert!(total_memory < 5_760_000); 7418 } 7419 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 7420 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7421 7422 // x86_64: We check that removing and adding back the virtio-net device 7423 // does not break the snapshot/restore support for virtio-pci. 7424 // This is an important thing to test as the hotplug will 7425 // trigger a PCI BAR reprogramming, which is a good way of 7426 // checking if the stored resources are correctly restored. 7427 // Unplug the virtio-net device 7428 // AArch64: Device hotplug is currently not supported, skipping here. 7429 #[cfg(target_arch = "x86_64")] 7430 { 7431 assert!(remote_command( 7432 &api_socket_source, 7433 "remove-device", 7434 Some(net_id), 7435 )); 7436 thread::sleep(std::time::Duration::new(10, 0)); 7437 let latest_events = [&MetaEvent { 7438 event: "device-removed".to_string(), 7439 device_id: Some(net_id.to_string()), 7440 }]; 7441 // See: #5938 7442 thread::sleep(std::time::Duration::new(1, 0)); 7443 assert!(check_latest_events_exact(&latest_events, &event_path)); 7444 7445 // Plug the virtio-net device again 7446 assert!(remote_command( 7447 &api_socket_source, 7448 "add-net", 7449 Some(net_params.as_str()), 7450 )); 7451 thread::sleep(std::time::Duration::new(10, 0)); 7452 } 7453 7454 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7455 }); 7456 7457 // Shutdown the source VM and check console output 7458 kill_child(&mut child); 7459 let output = child.wait_with_output().unwrap(); 7460 handle_child_output(r, &output); 7461 7462 let r = std::panic::catch_unwind(|| { 7463 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7464 }); 7465 7466 handle_child_output(r, &output); 7467 7468 // Remove the vsock socket file. 7469 Command::new("rm") 7470 .arg("-f") 7471 .arg(socket.as_str()) 7472 .output() 7473 .unwrap(); 7474 7475 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7476 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7477 7478 // Restore the VM from the snapshot 7479 let mut child = GuestCommand::new(&guest) 7480 .args(["--api-socket", &api_socket_restored]) 7481 .args([ 7482 "--event-monitor", 7483 format!("path={event_path_restored}").as_str(), 7484 ]) 7485 .args([ 7486 "--restore", 7487 format!("source_url=file://{snapshot_dir}").as_str(), 7488 ]) 7489 .capture_output() 7490 .spawn() 7491 .unwrap(); 7492 7493 // Wait for the VM to be restored 7494 thread::sleep(std::time::Duration::new(20, 0)); 7495 let expected_events = [ 7496 &MetaEvent { 7497 event: "starting".to_string(), 7498 device_id: None, 7499 }, 7500 &MetaEvent { 7501 event: "activated".to_string(), 7502 device_id: Some("__console".to_string()), 7503 }, 7504 &MetaEvent { 7505 event: "activated".to_string(), 7506 device_id: Some("__rng".to_string()), 7507 }, 7508 &MetaEvent { 7509 event: "restoring".to_string(), 7510 device_id: None, 7511 }, 7512 ]; 7513 assert!(check_sequential_events( 7514 &expected_events, 7515 &event_path_restored 7516 )); 7517 let latest_events = [&MetaEvent { 7518 event: "restored".to_string(), 7519 device_id: None, 7520 }]; 7521 assert!(check_latest_events_exact( 7522 &latest_events, 7523 &event_path_restored 7524 )); 7525 7526 // Remove the snapshot dir 7527 let _ = remove_dir_all(snapshot_dir.as_str()); 7528 7529 let r = std::panic::catch_unwind(|| { 7530 // Resume the VM 7531 assert!(remote_command(&api_socket_restored, "resume", None)); 7532 // There is no way that we can ensure the 'write()' to the 7533 // event file is completed when the 'resume' request is 7534 // returned successfully, because the 'write()' was done 7535 // asynchronously from a different thread of Cloud 7536 // Hypervisor (e.g. the event-monitor thread). 7537 thread::sleep(std::time::Duration::new(1, 0)); 7538 let latest_events = [ 7539 &MetaEvent { 7540 event: "resuming".to_string(), 7541 device_id: None, 7542 }, 7543 &MetaEvent { 7544 event: "resumed".to_string(), 7545 device_id: None, 7546 }, 7547 ]; 7548 assert!(check_latest_events_exact( 7549 &latest_events, 7550 &event_path_restored 7551 )); 7552 7553 // Perform same checks to validate VM has been properly restored 7554 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 7555 let total_memory = guest.get_total_memory().unwrap_or_default(); 7556 if !use_hotplug { 7557 assert!(total_memory > 1_920_000); 7558 } else { 7559 assert!(total_memory > 4_800_000); 7560 assert!(total_memory < 5_760_000); 7561 // Deflate balloon to restore entire RAM to the VM 7562 resize_command(&api_socket_restored, None, None, Some(0), None); 7563 thread::sleep(std::time::Duration::new(5, 0)); 7564 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7565 // Decrease guest RAM with virtio-mem 7566 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 7567 thread::sleep(std::time::Duration::new(5, 0)); 7568 let total_memory = guest.get_total_memory().unwrap_or_default(); 7569 assert!(total_memory > 4_800_000); 7570 assert!(total_memory < 5_760_000); 7571 } 7572 7573 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7574 }); 7575 // Shutdown the target VM and check console output 7576 kill_child(&mut child); 7577 let output = child.wait_with_output().unwrap(); 7578 handle_child_output(r, &output); 7579 7580 let r = std::panic::catch_unwind(|| { 7581 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7582 }); 7583 7584 handle_child_output(r, &output); 7585 } 7586 7587 #[test] 7588 fn test_snapshot_restore_with_fd() { 7589 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7590 let guest = Guest::new(Box::new(focal)); 7591 let kernel_path = direct_kernel_boot_path(); 7592 7593 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7594 7595 let net_id = "net123"; 7596 let num_queue_pairs: usize = 2; 7597 // use a name that does not conflict with tap dev created from other tests 7598 let tap_name = "chtap999"; 7599 use std::str::FromStr; 7600 let taps = net_util::open_tap( 7601 Some(tap_name), 7602 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 7603 None, 7604 &mut None, 7605 None, 7606 num_queue_pairs, 7607 Some(libc::O_RDWR | libc::O_NONBLOCK), 7608 ) 7609 .unwrap(); 7610 let net_params = format!( 7611 "id={},fd=[{},{}],mac={},ip={},mask=255.255.255.0,num_queues={}", 7612 net_id, 7613 taps[0].as_raw_fd(), 7614 taps[1].as_raw_fd(), 7615 guest.network.guest_mac, 7616 guest.network.host_ip, 7617 num_queue_pairs * 2 7618 ); 7619 7620 let cloudinit_params = format!( 7621 "path={},iommu=on", 7622 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7623 ); 7624 7625 let n_cpu = 2; 7626 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7627 7628 let mut child = GuestCommand::new(&guest) 7629 .args(["--api-socket", &api_socket_source]) 7630 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7631 .args(["--cpus", format!("boot={}", n_cpu).as_str()]) 7632 .args(["--memory", "size=1G"]) 7633 .args(["--kernel", kernel_path.to_str().unwrap()]) 7634 .args([ 7635 "--disk", 7636 format!( 7637 "path={}", 7638 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7639 ) 7640 .as_str(), 7641 cloudinit_params.as_str(), 7642 ]) 7643 .args(["--net", net_params.as_str()]) 7644 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7645 .capture_output() 7646 .spawn() 7647 .unwrap(); 7648 7649 let console_text = String::from("On a branch floating down river a cricket, singing."); 7650 // Create the snapshot directory 7651 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7652 7653 let r = std::panic::catch_unwind(|| { 7654 guest.wait_vm_boot(None).unwrap(); 7655 7656 // close the fds after VM boots, as CH duplicates them before using 7657 for tap in taps.iter() { 7658 unsafe { libc::close(tap.as_raw_fd()) }; 7659 } 7660 7661 // Check the number of vCPUs 7662 assert_eq!(guest.get_cpu_count().unwrap_or_default(), n_cpu); 7663 // Check the guest RAM 7664 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 7665 7666 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 7667 guest.check_devices_common(None, Some(&console_text), None); 7668 7669 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7670 }); 7671 7672 // Shutdown the source VM and check console output 7673 kill_child(&mut child); 7674 let output = child.wait_with_output().unwrap(); 7675 handle_child_output(r, &output); 7676 7677 let r = std::panic::catch_unwind(|| { 7678 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7679 }); 7680 7681 handle_child_output(r, &output); 7682 7683 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7684 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7685 7686 // Restore the VM from the snapshot 7687 let mut child = GuestCommand::new(&guest) 7688 .args(["--api-socket", &api_socket_restored]) 7689 .args([ 7690 "--event-monitor", 7691 format!("path={event_path_restored}").as_str(), 7692 ]) 7693 .capture_output() 7694 .spawn() 7695 .unwrap(); 7696 thread::sleep(std::time::Duration::new(2, 0)); 7697 7698 let taps = net_util::open_tap( 7699 Some(tap_name), 7700 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 7701 None, 7702 &mut None, 7703 None, 7704 num_queue_pairs, 7705 Some(libc::O_RDWR | libc::O_NONBLOCK), 7706 ) 7707 .unwrap(); 7708 let restore_params = format!( 7709 "source_url=file://{},net_fds=[{}@[{},{}]]", 7710 snapshot_dir, 7711 net_id, 7712 taps[0].as_raw_fd(), 7713 taps[1].as_raw_fd() 7714 ); 7715 assert!(remote_command( 7716 &api_socket_restored, 7717 "restore", 7718 Some(restore_params.as_str()) 7719 )); 7720 7721 // Wait for the VM to be restored 7722 thread::sleep(std::time::Duration::new(20, 0)); 7723 7724 // close the fds as CH duplicates them before using 7725 for tap in taps.iter() { 7726 unsafe { libc::close(tap.as_raw_fd()) }; 7727 } 7728 7729 let expected_events = [ 7730 &MetaEvent { 7731 event: "starting".to_string(), 7732 device_id: None, 7733 }, 7734 &MetaEvent { 7735 event: "activated".to_string(), 7736 device_id: Some("__console".to_string()), 7737 }, 7738 &MetaEvent { 7739 event: "activated".to_string(), 7740 device_id: Some("__rng".to_string()), 7741 }, 7742 &MetaEvent { 7743 event: "restoring".to_string(), 7744 device_id: None, 7745 }, 7746 ]; 7747 assert!(check_sequential_events( 7748 &expected_events, 7749 &event_path_restored 7750 )); 7751 let latest_events = [&MetaEvent { 7752 event: "restored".to_string(), 7753 device_id: None, 7754 }]; 7755 assert!(check_latest_events_exact( 7756 &latest_events, 7757 &event_path_restored 7758 )); 7759 7760 // Remove the snapshot dir 7761 let _ = remove_dir_all(snapshot_dir.as_str()); 7762 7763 let r = std::panic::catch_unwind(|| { 7764 // Resume the VM 7765 assert!(remote_command(&api_socket_restored, "resume", None)); 7766 // There is no way that we can ensure the 'write()' to the 7767 // event file is completed when the 'resume' request is 7768 // returned successfully, because the 'write()' was done 7769 // asynchronously from a different thread of Cloud 7770 // Hypervisor (e.g. the event-monitor thread). 7771 thread::sleep(std::time::Duration::new(1, 0)); 7772 let latest_events = [ 7773 &MetaEvent { 7774 event: "resuming".to_string(), 7775 device_id: None, 7776 }, 7777 &MetaEvent { 7778 event: "resumed".to_string(), 7779 device_id: None, 7780 }, 7781 ]; 7782 assert!(check_latest_events_exact( 7783 &latest_events, 7784 &event_path_restored 7785 )); 7786 7787 // Perform same checks to validate VM has been properly restored 7788 assert_eq!(guest.get_cpu_count().unwrap_or_default(), n_cpu); 7789 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 7790 7791 guest.check_devices_common(None, Some(&console_text), None); 7792 }); 7793 // Shutdown the target VM and check console output 7794 kill_child(&mut child); 7795 let output = child.wait_with_output().unwrap(); 7796 handle_child_output(r, &output); 7797 7798 let r = std::panic::catch_unwind(|| { 7799 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7800 }); 7801 7802 handle_child_output(r, &output); 7803 } 7804 7805 #[test] 7806 #[cfg(target_arch = "x86_64")] 7807 fn test_snapshot_restore_pvpanic() { 7808 _test_snapshot_restore_devices(true); 7809 } 7810 7811 fn _test_snapshot_restore_devices(pvpanic: bool) { 7812 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7813 let guest = Guest::new(Box::new(focal)); 7814 let kernel_path = direct_kernel_boot_path(); 7815 7816 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7817 7818 let device_params = { 7819 let mut data = vec![]; 7820 if pvpanic { 7821 data.push("--pvpanic"); 7822 } 7823 data 7824 }; 7825 7826 let socket = temp_vsock_path(&guest.tmp_dir); 7827 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7828 7829 let mut child = GuestCommand::new(&guest) 7830 .args(["--api-socket", &api_socket_source]) 7831 .args(["--event-monitor", format!("path={}", event_path).as_str()]) 7832 .args(["--cpus", "boot=2"]) 7833 .args(["--memory", "size=1G"]) 7834 .args(["--kernel", kernel_path.to_str().unwrap()]) 7835 .default_disks() 7836 .default_net() 7837 .args(["--vsock", format!("cid=3,socket={}", socket).as_str()]) 7838 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7839 .args(device_params) 7840 .capture_output() 7841 .spawn() 7842 .unwrap(); 7843 7844 let console_text = String::from("On a branch floating down river a cricket, singing."); 7845 // Create the snapshot directory 7846 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7847 7848 let r = std::panic::catch_unwind(|| { 7849 guest.wait_vm_boot(None).unwrap(); 7850 7851 // Check the number of vCPUs 7852 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 7853 7854 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7855 }); 7856 7857 // Shutdown the source VM and check console output 7858 kill_child(&mut child); 7859 let output = child.wait_with_output().unwrap(); 7860 handle_child_output(r, &output); 7861 7862 // Remove the vsock socket file. 7863 Command::new("rm") 7864 .arg("-f") 7865 .arg(socket.as_str()) 7866 .output() 7867 .unwrap(); 7868 7869 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7870 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7871 7872 // Restore the VM from the snapshot 7873 let mut child = GuestCommand::new(&guest) 7874 .args(["--api-socket", &api_socket_restored]) 7875 .args([ 7876 "--event-monitor", 7877 format!("path={event_path_restored}").as_str(), 7878 ]) 7879 .args([ 7880 "--restore", 7881 format!("source_url=file://{snapshot_dir}").as_str(), 7882 ]) 7883 .capture_output() 7884 .spawn() 7885 .unwrap(); 7886 7887 // Wait for the VM to be restored 7888 thread::sleep(std::time::Duration::new(20, 0)); 7889 7890 let latest_events = [&MetaEvent { 7891 event: "restored".to_string(), 7892 device_id: None, 7893 }]; 7894 assert!(check_latest_events_exact( 7895 &latest_events, 7896 &event_path_restored 7897 )); 7898 7899 // Remove the snapshot dir 7900 let _ = remove_dir_all(snapshot_dir.as_str()); 7901 7902 let r = std::panic::catch_unwind(|| { 7903 // Resume the VM 7904 assert!(remote_command(&api_socket_restored, "resume", None)); 7905 // There is no way that we can ensure the 'write()' to the 7906 // event file is completed when the 'resume' request is 7907 // returned successfully, because the 'write()' was done 7908 // asynchronously from a different thread of Cloud 7909 // Hypervisor (e.g. the event-monitor thread). 7910 thread::sleep(std::time::Duration::new(1, 0)); 7911 let latest_events = [ 7912 &MetaEvent { 7913 event: "resuming".to_string(), 7914 device_id: None, 7915 }, 7916 &MetaEvent { 7917 event: "resumed".to_string(), 7918 device_id: None, 7919 }, 7920 ]; 7921 assert!(check_latest_events_exact( 7922 &latest_events, 7923 &event_path_restored 7924 )); 7925 7926 // Check the number of vCPUs 7927 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 7928 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7929 7930 if pvpanic { 7931 // Trigger guest a panic 7932 make_guest_panic(&guest); 7933 // Wait a while for guest 7934 thread::sleep(std::time::Duration::new(10, 0)); 7935 7936 let expected_sequential_events = [&MetaEvent { 7937 event: "panic".to_string(), 7938 device_id: None, 7939 }]; 7940 assert!(check_latest_events_exact( 7941 &expected_sequential_events, 7942 &event_path_restored 7943 )); 7944 } 7945 }); 7946 // Shutdown the target VM and check console output 7947 kill_child(&mut child); 7948 let output = child.wait_with_output().unwrap(); 7949 handle_child_output(r, &output); 7950 7951 let r = std::panic::catch_unwind(|| { 7952 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7953 }); 7954 7955 handle_child_output(r, &output); 7956 } 7957 } 7958 7959 mod windows { 7960 use once_cell::sync::Lazy; 7961 7962 use crate::*; 7963 7964 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 7965 7966 struct WindowsGuest { 7967 guest: Guest, 7968 auth: PasswordAuth, 7969 } 7970 7971 trait FsType { 7972 const FS_FAT: u8; 7973 const FS_NTFS: u8; 7974 } 7975 impl FsType for WindowsGuest { 7976 const FS_FAT: u8 = 0; 7977 const FS_NTFS: u8 = 1; 7978 } 7979 7980 impl WindowsGuest { 7981 fn new() -> Self { 7982 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 7983 let guest = Guest::new(Box::new(disk)); 7984 let auth = PasswordAuth { 7985 username: String::from("administrator"), 7986 password: String::from("Admin123"), 7987 }; 7988 7989 WindowsGuest { guest, auth } 7990 } 7991 7992 fn guest(&self) -> &Guest { 7993 &self.guest 7994 } 7995 7996 fn ssh_cmd(&self, cmd: &str) -> String { 7997 ssh_command_ip_with_auth( 7998 cmd, 7999 &self.auth, 8000 &self.guest.network.guest_ip, 8001 DEFAULT_SSH_RETRIES, 8002 DEFAULT_SSH_TIMEOUT, 8003 ) 8004 .unwrap() 8005 } 8006 8007 fn cpu_count(&self) -> u8 { 8008 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 8009 .trim() 8010 .parse::<u8>() 8011 .unwrap_or(0) 8012 } 8013 8014 fn ram_size(&self) -> usize { 8015 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 8016 .trim() 8017 .parse::<usize>() 8018 .unwrap_or(0) 8019 } 8020 8021 fn netdev_count(&self) -> u8 { 8022 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 8023 .trim() 8024 .parse::<u8>() 8025 .unwrap_or(0) 8026 } 8027 8028 fn disk_count(&self) -> u8 { 8029 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 8030 .trim() 8031 .parse::<u8>() 8032 .unwrap_or(0) 8033 } 8034 8035 fn reboot(&self) { 8036 let _ = self.ssh_cmd("shutdown /r /t 0"); 8037 } 8038 8039 fn shutdown(&self) { 8040 let _ = self.ssh_cmd("shutdown /s /t 0"); 8041 } 8042 8043 fn run_dnsmasq(&self) -> std::process::Child { 8044 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 8045 let dhcp_host = format!( 8046 "--dhcp-host={},{}", 8047 self.guest.network.guest_mac, self.guest.network.guest_ip 8048 ); 8049 let dhcp_range = format!( 8050 "--dhcp-range=eth,{},{}", 8051 self.guest.network.guest_ip, self.guest.network.guest_ip 8052 ); 8053 8054 Command::new("dnsmasq") 8055 .arg("--no-daemon") 8056 .arg("--log-queries") 8057 .arg(listen_address.as_str()) 8058 .arg("--except-interface=lo") 8059 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 8060 .arg("--conf-file=/dev/null") 8061 .arg(dhcp_host.as_str()) 8062 .arg(dhcp_range.as_str()) 8063 .spawn() 8064 .unwrap() 8065 } 8066 8067 // TODO Cleanup image file explicitly after test, if there's some space issues. 8068 fn disk_new(&self, fs: u8, sz: usize) -> String { 8069 let mut guard = NEXT_DISK_ID.lock().unwrap(); 8070 let id = *guard; 8071 *guard = id + 1; 8072 8073 let img = PathBuf::from(format!("/tmp/test-hotplug-{id}.raw")); 8074 let _ = fs::remove_file(&img); 8075 8076 // Create an image file 8077 let out = Command::new("qemu-img") 8078 .args([ 8079 "create", 8080 "-f", 8081 "raw", 8082 img.to_str().unwrap(), 8083 format!("{sz}m").as_str(), 8084 ]) 8085 .output() 8086 .expect("qemu-img command failed") 8087 .stdout; 8088 println!("{out:?}"); 8089 8090 // Associate image to a loop device 8091 let out = Command::new("losetup") 8092 .args(["--show", "-f", img.to_str().unwrap()]) 8093 .output() 8094 .expect("failed to create loop device") 8095 .stdout; 8096 let _tmp = String::from_utf8_lossy(&out); 8097 let loop_dev = _tmp.trim(); 8098 println!("{out:?}"); 8099 8100 // Create a partition table 8101 // echo 'type=7' | sudo sfdisk "${LOOP}" 8102 let mut child = Command::new("sfdisk") 8103 .args([loop_dev]) 8104 .stdin(Stdio::piped()) 8105 .spawn() 8106 .unwrap(); 8107 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 8108 stdin 8109 .write_all("type=7".as_bytes()) 8110 .expect("failed to write stdin"); 8111 let out = child.wait_with_output().expect("sfdisk failed").stdout; 8112 println!("{out:?}"); 8113 8114 // Disengage the loop device 8115 let out = Command::new("losetup") 8116 .args(["-d", loop_dev]) 8117 .output() 8118 .expect("loop device not found") 8119 .stdout; 8120 println!("{out:?}"); 8121 8122 // Re-associate loop device pointing to the partition only 8123 let out = Command::new("losetup") 8124 .args([ 8125 "--show", 8126 "--offset", 8127 (512 * 2048).to_string().as_str(), 8128 "-f", 8129 img.to_str().unwrap(), 8130 ]) 8131 .output() 8132 .expect("failed to create loop device") 8133 .stdout; 8134 let _tmp = String::from_utf8_lossy(&out); 8135 let loop_dev = _tmp.trim(); 8136 println!("{out:?}"); 8137 8138 // Create filesystem. 8139 let fs_cmd = match fs { 8140 WindowsGuest::FS_FAT => "mkfs.msdos", 8141 WindowsGuest::FS_NTFS => "mkfs.ntfs", 8142 _ => panic!("Unknown filesystem type '{fs}'"), 8143 }; 8144 let out = Command::new(fs_cmd) 8145 .args([&loop_dev]) 8146 .output() 8147 .unwrap_or_else(|_| panic!("{fs_cmd} failed")) 8148 .stdout; 8149 println!("{out:?}"); 8150 8151 // Disengage the loop device 8152 let out = Command::new("losetup") 8153 .args(["-d", loop_dev]) 8154 .output() 8155 .unwrap_or_else(|_| panic!("loop device '{loop_dev}' not found")) 8156 .stdout; 8157 println!("{out:?}"); 8158 8159 img.to_str().unwrap().to_string() 8160 } 8161 8162 fn disks_set_rw(&self) { 8163 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 8164 } 8165 8166 fn disks_online(&self) { 8167 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 8168 } 8169 8170 fn disk_file_put(&self, fname: &str, data: &str) { 8171 let _ = self.ssh_cmd(&format!( 8172 "powershell -Command \"'{data}' | Set-Content -Path {fname}\"" 8173 )); 8174 } 8175 8176 fn disk_file_read(&self, fname: &str) -> String { 8177 self.ssh_cmd(&format!( 8178 "powershell -Command \"Get-Content -Path {fname}\"" 8179 )) 8180 } 8181 8182 fn wait_for_boot(&self) -> bool { 8183 let cmd = "dir /b c:\\ | find \"Windows\""; 8184 let tmo_max = 180; 8185 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 8186 // interval must be small. 8187 let tmo_int = 2; 8188 let out = ssh_command_ip_with_auth( 8189 cmd, 8190 &self.auth, 8191 &self.guest.network.guest_ip, 8192 { 8193 let mut ret = 1; 8194 let mut tmo_acc = 0; 8195 loop { 8196 tmo_acc += tmo_int * ret; 8197 if tmo_acc >= tmo_max { 8198 break; 8199 } 8200 ret += 1; 8201 } 8202 ret 8203 }, 8204 tmo_int, 8205 ) 8206 .unwrap(); 8207 8208 if "Windows" == out.trim() { 8209 return true; 8210 } 8211 8212 false 8213 } 8214 } 8215 8216 fn vcpu_threads_count(pid: u32) -> u8 { 8217 // ps -T -p 12345 | grep vcpu | wc -l 8218 let out = Command::new("ps") 8219 .args(["-T", "-p", format!("{pid}").as_str()]) 8220 .output() 8221 .expect("ps command failed") 8222 .stdout; 8223 String::from_utf8_lossy(&out).matches("vcpu").count() as u8 8224 } 8225 8226 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 8227 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 8228 let out = Command::new("ps") 8229 .args(["-T", "-p", format!("{pid}").as_str()]) 8230 .output() 8231 .expect("ps command failed") 8232 .stdout; 8233 let mut n = 0; 8234 String::from_utf8_lossy(&out) 8235 .split_whitespace() 8236 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 8237 n 8238 } 8239 8240 fn disk_ctrl_threads_count(pid: u32) -> u8 { 8241 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 8242 let out = Command::new("ps") 8243 .args(["-T", "-p", format!("{pid}").as_str()]) 8244 .output() 8245 .expect("ps command failed") 8246 .stdout; 8247 let mut n = 0; 8248 String::from_utf8_lossy(&out) 8249 .split_whitespace() 8250 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 8251 n 8252 } 8253 8254 #[test] 8255 fn test_windows_guest() { 8256 let windows_guest = WindowsGuest::new(); 8257 8258 let mut child = GuestCommand::new(windows_guest.guest()) 8259 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8260 .args(["--memory", "size=4G"]) 8261 .args(["--kernel", edk2_path().to_str().unwrap()]) 8262 .args(["--serial", "tty"]) 8263 .args(["--console", "off"]) 8264 .default_disks() 8265 .default_net() 8266 .capture_output() 8267 .spawn() 8268 .unwrap(); 8269 8270 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8271 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8272 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8273 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8274 8275 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8276 8277 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8278 8279 let r = std::panic::catch_unwind(|| { 8280 // Wait to make sure Windows boots up 8281 assert!(windows_guest.wait_for_boot()); 8282 8283 windows_guest.shutdown(); 8284 }); 8285 8286 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8287 let _ = child.kill(); 8288 let output = child.wait_with_output().unwrap(); 8289 8290 let _ = child_dnsmasq.kill(); 8291 let _ = child_dnsmasq.wait(); 8292 8293 handle_child_output(r, &output); 8294 } 8295 8296 #[test] 8297 fn test_windows_guest_multiple_queues() { 8298 let windows_guest = WindowsGuest::new(); 8299 8300 let mut ovmf_path = dirs::home_dir().unwrap(); 8301 ovmf_path.push("workloads"); 8302 ovmf_path.push(OVMF_NAME); 8303 8304 let mut child = GuestCommand::new(windows_guest.guest()) 8305 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 8306 .args(["--memory", "size=4G"]) 8307 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8308 .args(["--serial", "tty"]) 8309 .args(["--console", "off"]) 8310 .args([ 8311 "--disk", 8312 format!( 8313 "path={},num_queues=4", 8314 windows_guest 8315 .guest() 8316 .disk_config 8317 .disk(DiskType::OperatingSystem) 8318 .unwrap() 8319 ) 8320 .as_str(), 8321 ]) 8322 .args([ 8323 "--net", 8324 format!( 8325 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 8326 windows_guest.guest().network.guest_mac, 8327 windows_guest.guest().network.host_ip 8328 ) 8329 .as_str(), 8330 ]) 8331 .capture_output() 8332 .spawn() 8333 .unwrap(); 8334 8335 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8336 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8337 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8338 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8339 8340 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8341 8342 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8343 8344 let r = std::panic::catch_unwind(|| { 8345 // Wait to make sure Windows boots up 8346 assert!(windows_guest.wait_for_boot()); 8347 8348 windows_guest.shutdown(); 8349 }); 8350 8351 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8352 let _ = child.kill(); 8353 let output = child.wait_with_output().unwrap(); 8354 8355 let _ = child_dnsmasq.kill(); 8356 let _ = child_dnsmasq.wait(); 8357 8358 handle_child_output(r, &output); 8359 } 8360 8361 #[test] 8362 #[cfg(not(feature = "mshv"))] 8363 #[ignore = "See #4327"] 8364 fn test_windows_guest_snapshot_restore() { 8365 let windows_guest = WindowsGuest::new(); 8366 8367 let mut ovmf_path = dirs::home_dir().unwrap(); 8368 ovmf_path.push("workloads"); 8369 ovmf_path.push(OVMF_NAME); 8370 8371 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8372 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 8373 8374 let mut child = GuestCommand::new(windows_guest.guest()) 8375 .args(["--api-socket", &api_socket_source]) 8376 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8377 .args(["--memory", "size=4G"]) 8378 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8379 .args(["--serial", "tty"]) 8380 .args(["--console", "off"]) 8381 .default_disks() 8382 .default_net() 8383 .capture_output() 8384 .spawn() 8385 .unwrap(); 8386 8387 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8388 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8389 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8390 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8391 8392 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8393 8394 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8395 8396 // Wait to make sure Windows boots up 8397 assert!(windows_guest.wait_for_boot()); 8398 8399 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 8400 8401 // Pause the VM 8402 assert!(remote_command(&api_socket_source, "pause", None)); 8403 8404 // Take a snapshot from the VM 8405 assert!(remote_command( 8406 &api_socket_source, 8407 "snapshot", 8408 Some(format!("file://{snapshot_dir}").as_str()), 8409 )); 8410 8411 // Wait to make sure the snapshot is completed 8412 thread::sleep(std::time::Duration::new(30, 0)); 8413 8414 let _ = child.kill(); 8415 child.wait().unwrap(); 8416 8417 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 8418 8419 // Restore the VM from the snapshot 8420 let mut child = GuestCommand::new(windows_guest.guest()) 8421 .args(["--api-socket", &api_socket_restored]) 8422 .args([ 8423 "--restore", 8424 format!("source_url=file://{snapshot_dir}").as_str(), 8425 ]) 8426 .capture_output() 8427 .spawn() 8428 .unwrap(); 8429 8430 // Wait for the VM to be restored 8431 thread::sleep(std::time::Duration::new(20, 0)); 8432 8433 let r = std::panic::catch_unwind(|| { 8434 // Resume the VM 8435 assert!(remote_command(&api_socket_restored, "resume", None)); 8436 8437 windows_guest.shutdown(); 8438 }); 8439 8440 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8441 let _ = child.kill(); 8442 let output = child.wait_with_output().unwrap(); 8443 8444 let _ = child_dnsmasq.kill(); 8445 let _ = child_dnsmasq.wait(); 8446 8447 handle_child_output(r, &output); 8448 } 8449 8450 #[test] 8451 #[cfg(not(feature = "mshv"))] 8452 #[cfg(not(target_arch = "aarch64"))] 8453 fn test_windows_guest_cpu_hotplug() { 8454 let windows_guest = WindowsGuest::new(); 8455 8456 let mut ovmf_path = dirs::home_dir().unwrap(); 8457 ovmf_path.push("workloads"); 8458 ovmf_path.push(OVMF_NAME); 8459 8460 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8461 let api_socket = temp_api_path(&tmp_dir); 8462 8463 let mut child = GuestCommand::new(windows_guest.guest()) 8464 .args(["--api-socket", &api_socket]) 8465 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 8466 .args(["--memory", "size=4G"]) 8467 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8468 .args(["--serial", "tty"]) 8469 .args(["--console", "off"]) 8470 .default_disks() 8471 .default_net() 8472 .capture_output() 8473 .spawn() 8474 .unwrap(); 8475 8476 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8477 8478 let r = std::panic::catch_unwind(|| { 8479 // Wait to make sure Windows boots up 8480 assert!(windows_guest.wait_for_boot()); 8481 8482 let vcpu_num = 2; 8483 // Check the initial number of CPUs the guest sees 8484 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8485 // Check the initial number of vcpu threads in the CH process 8486 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8487 8488 let vcpu_num = 6; 8489 // Hotplug some CPUs 8490 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8491 // Wait to make sure CPUs are added 8492 thread::sleep(std::time::Duration::new(10, 0)); 8493 // Check the guest sees the correct number 8494 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8495 // Check the CH process has the correct number of vcpu threads 8496 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8497 8498 let vcpu_num = 4; 8499 // Remove some CPUs. Note that Windows doesn't support hot-remove. 8500 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8501 // Wait to make sure CPUs are removed 8502 thread::sleep(std::time::Duration::new(10, 0)); 8503 // Reboot to let Windows catch up 8504 windows_guest.reboot(); 8505 // Wait to make sure Windows completely rebooted 8506 thread::sleep(std::time::Duration::new(60, 0)); 8507 // Check the guest sees the correct number 8508 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8509 // Check the CH process has the correct number of vcpu threads 8510 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8511 8512 windows_guest.shutdown(); 8513 }); 8514 8515 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8516 let _ = child.kill(); 8517 let output = child.wait_with_output().unwrap(); 8518 8519 let _ = child_dnsmasq.kill(); 8520 let _ = child_dnsmasq.wait(); 8521 8522 handle_child_output(r, &output); 8523 } 8524 8525 #[test] 8526 #[cfg(not(feature = "mshv"))] 8527 #[cfg(not(target_arch = "aarch64"))] 8528 fn test_windows_guest_ram_hotplug() { 8529 let windows_guest = WindowsGuest::new(); 8530 8531 let mut ovmf_path = dirs::home_dir().unwrap(); 8532 ovmf_path.push("workloads"); 8533 ovmf_path.push(OVMF_NAME); 8534 8535 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8536 let api_socket = temp_api_path(&tmp_dir); 8537 8538 let mut child = GuestCommand::new(windows_guest.guest()) 8539 .args(["--api-socket", &api_socket]) 8540 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8541 .args(["--memory", "size=2G,hotplug_size=5G"]) 8542 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8543 .args(["--serial", "tty"]) 8544 .args(["--console", "off"]) 8545 .default_disks() 8546 .default_net() 8547 .capture_output() 8548 .spawn() 8549 .unwrap(); 8550 8551 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8552 8553 let r = std::panic::catch_unwind(|| { 8554 // Wait to make sure Windows boots up 8555 assert!(windows_guest.wait_for_boot()); 8556 8557 let ram_size = 2 * 1024 * 1024 * 1024; 8558 // Check the initial number of RAM the guest sees 8559 let current_ram_size = windows_guest.ram_size(); 8560 // This size seems to be reserved by the system and thus the 8561 // reported amount differs by this constant value. 8562 let reserved_ram_size = ram_size - current_ram_size; 8563 // Verify that there's not more than 4mb constant diff wasted 8564 // by the reserved ram. 8565 assert!(reserved_ram_size < 4 * 1024 * 1024); 8566 8567 let ram_size = 4 * 1024 * 1024 * 1024; 8568 // Hotplug some RAM 8569 resize_command(&api_socket, None, Some(ram_size), None, None); 8570 // Wait to make sure RAM has been added 8571 thread::sleep(std::time::Duration::new(10, 0)); 8572 // Check the guest sees the correct number 8573 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8574 8575 let ram_size = 3 * 1024 * 1024 * 1024; 8576 // Unplug some RAM. Note that hot-remove most likely won't work. 8577 resize_command(&api_socket, None, Some(ram_size), None, None); 8578 // Wait to make sure RAM has been added 8579 thread::sleep(std::time::Duration::new(10, 0)); 8580 // Reboot to let Windows catch up 8581 windows_guest.reboot(); 8582 // Wait to make sure guest completely rebooted 8583 thread::sleep(std::time::Duration::new(60, 0)); 8584 // Check the guest sees the correct number 8585 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8586 8587 windows_guest.shutdown(); 8588 }); 8589 8590 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8591 let _ = child.kill(); 8592 let output = child.wait_with_output().unwrap(); 8593 8594 let _ = child_dnsmasq.kill(); 8595 let _ = child_dnsmasq.wait(); 8596 8597 handle_child_output(r, &output); 8598 } 8599 8600 #[test] 8601 #[cfg(not(feature = "mshv"))] 8602 fn test_windows_guest_netdev_hotplug() { 8603 let windows_guest = WindowsGuest::new(); 8604 8605 let mut ovmf_path = dirs::home_dir().unwrap(); 8606 ovmf_path.push("workloads"); 8607 ovmf_path.push(OVMF_NAME); 8608 8609 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8610 let api_socket = temp_api_path(&tmp_dir); 8611 8612 let mut child = GuestCommand::new(windows_guest.guest()) 8613 .args(["--api-socket", &api_socket]) 8614 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8615 .args(["--memory", "size=4G"]) 8616 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8617 .args(["--serial", "tty"]) 8618 .args(["--console", "off"]) 8619 .default_disks() 8620 .default_net() 8621 .capture_output() 8622 .spawn() 8623 .unwrap(); 8624 8625 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8626 8627 let r = std::panic::catch_unwind(|| { 8628 // Wait to make sure Windows boots up 8629 assert!(windows_guest.wait_for_boot()); 8630 8631 // Initially present network device 8632 let netdev_num = 1; 8633 assert_eq!(windows_guest.netdev_count(), netdev_num); 8634 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8635 8636 // Hotplug network device 8637 let (cmd_success, cmd_output) = remote_command_w_output( 8638 &api_socket, 8639 "add-net", 8640 Some(windows_guest.guest().default_net_string().as_str()), 8641 ); 8642 assert!(cmd_success); 8643 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 8644 thread::sleep(std::time::Duration::new(5, 0)); 8645 // Verify the device is on the system 8646 let netdev_num = 2; 8647 assert_eq!(windows_guest.netdev_count(), netdev_num); 8648 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8649 8650 // Remove network device 8651 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 8652 assert!(cmd_success); 8653 thread::sleep(std::time::Duration::new(5, 0)); 8654 // Verify the device has been removed 8655 let netdev_num = 1; 8656 assert_eq!(windows_guest.netdev_count(), netdev_num); 8657 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8658 8659 windows_guest.shutdown(); 8660 }); 8661 8662 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8663 let _ = child.kill(); 8664 let output = child.wait_with_output().unwrap(); 8665 8666 let _ = child_dnsmasq.kill(); 8667 let _ = child_dnsmasq.wait(); 8668 8669 handle_child_output(r, &output); 8670 } 8671 8672 #[test] 8673 #[ignore = "See #6037"] 8674 #[cfg(not(feature = "mshv"))] 8675 #[cfg(not(target_arch = "aarch64"))] 8676 fn test_windows_guest_disk_hotplug() { 8677 let windows_guest = WindowsGuest::new(); 8678 8679 let mut ovmf_path = dirs::home_dir().unwrap(); 8680 ovmf_path.push("workloads"); 8681 ovmf_path.push(OVMF_NAME); 8682 8683 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8684 let api_socket = temp_api_path(&tmp_dir); 8685 8686 let mut child = GuestCommand::new(windows_guest.guest()) 8687 .args(["--api-socket", &api_socket]) 8688 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8689 .args(["--memory", "size=4G"]) 8690 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8691 .args(["--serial", "tty"]) 8692 .args(["--console", "off"]) 8693 .default_disks() 8694 .default_net() 8695 .capture_output() 8696 .spawn() 8697 .unwrap(); 8698 8699 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8700 8701 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 8702 8703 let r = std::panic::catch_unwind(|| { 8704 // Wait to make sure Windows boots up 8705 assert!(windows_guest.wait_for_boot()); 8706 8707 // Initially present disk device 8708 let disk_num = 1; 8709 assert_eq!(windows_guest.disk_count(), disk_num); 8710 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8711 8712 // Hotplug disk device 8713 let (cmd_success, cmd_output) = remote_command_w_output( 8714 &api_socket, 8715 "add-disk", 8716 Some(format!("path={disk},readonly=off").as_str()), 8717 ); 8718 assert!(cmd_success); 8719 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 8720 thread::sleep(std::time::Duration::new(5, 0)); 8721 // Online disk device 8722 windows_guest.disks_set_rw(); 8723 windows_guest.disks_online(); 8724 // Verify the device is on the system 8725 let disk_num = 2; 8726 assert_eq!(windows_guest.disk_count(), disk_num); 8727 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8728 8729 let data = "hello"; 8730 let fname = "d:\\world"; 8731 windows_guest.disk_file_put(fname, data); 8732 8733 // Unmount disk device 8734 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 8735 assert!(cmd_success); 8736 thread::sleep(std::time::Duration::new(5, 0)); 8737 // Verify the device has been removed 8738 let disk_num = 1; 8739 assert_eq!(windows_guest.disk_count(), disk_num); 8740 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8741 8742 // Remount and check the file exists with the expected contents 8743 let (cmd_success, _cmd_output) = remote_command_w_output( 8744 &api_socket, 8745 "add-disk", 8746 Some(format!("path={disk},readonly=off").as_str()), 8747 ); 8748 assert!(cmd_success); 8749 thread::sleep(std::time::Duration::new(5, 0)); 8750 let out = windows_guest.disk_file_read(fname); 8751 assert_eq!(data, out.trim()); 8752 8753 // Intentionally no unmount, it'll happen at shutdown. 8754 8755 windows_guest.shutdown(); 8756 }); 8757 8758 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8759 let _ = child.kill(); 8760 let output = child.wait_with_output().unwrap(); 8761 8762 let _ = child_dnsmasq.kill(); 8763 let _ = child_dnsmasq.wait(); 8764 8765 handle_child_output(r, &output); 8766 } 8767 8768 #[test] 8769 #[ignore = "See #6037"] 8770 #[cfg(not(feature = "mshv"))] 8771 #[cfg(not(target_arch = "aarch64"))] 8772 fn test_windows_guest_disk_hotplug_multi() { 8773 let windows_guest = WindowsGuest::new(); 8774 8775 let mut ovmf_path = dirs::home_dir().unwrap(); 8776 ovmf_path.push("workloads"); 8777 ovmf_path.push(OVMF_NAME); 8778 8779 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8780 let api_socket = temp_api_path(&tmp_dir); 8781 8782 let mut child = GuestCommand::new(windows_guest.guest()) 8783 .args(["--api-socket", &api_socket]) 8784 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8785 .args(["--memory", "size=2G"]) 8786 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8787 .args(["--serial", "tty"]) 8788 .args(["--console", "off"]) 8789 .default_disks() 8790 .default_net() 8791 .capture_output() 8792 .spawn() 8793 .unwrap(); 8794 8795 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8796 8797 // Predefined data to used at various test stages 8798 let disk_test_data: [[String; 4]; 2] = [ 8799 [ 8800 "_disk2".to_string(), 8801 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 8802 "d:\\world".to_string(), 8803 "hello".to_string(), 8804 ], 8805 [ 8806 "_disk3".to_string(), 8807 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 8808 "e:\\hello".to_string(), 8809 "world".to_string(), 8810 ], 8811 ]; 8812 8813 let r = std::panic::catch_unwind(|| { 8814 // Wait to make sure Windows boots up 8815 assert!(windows_guest.wait_for_boot()); 8816 8817 // Initially present disk device 8818 let disk_num = 1; 8819 assert_eq!(windows_guest.disk_count(), disk_num); 8820 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8821 8822 for it in &disk_test_data { 8823 let disk_id = it[0].as_str(); 8824 let disk = it[1].as_str(); 8825 // Hotplug disk device 8826 let (cmd_success, cmd_output) = remote_command_w_output( 8827 &api_socket, 8828 "add-disk", 8829 Some(format!("path={disk},readonly=off").as_str()), 8830 ); 8831 assert!(cmd_success); 8832 assert!(String::from_utf8_lossy(&cmd_output) 8833 .contains(format!("\"id\":\"{disk_id}\"").as_str())); 8834 thread::sleep(std::time::Duration::new(5, 0)); 8835 // Online disk devices 8836 windows_guest.disks_set_rw(); 8837 windows_guest.disks_online(); 8838 } 8839 // Verify the devices are on the system 8840 let disk_num = (disk_test_data.len() + 1) as u8; 8841 assert_eq!(windows_guest.disk_count(), disk_num); 8842 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8843 8844 // Put test data 8845 for it in &disk_test_data { 8846 let fname = it[2].as_str(); 8847 let data = it[3].as_str(); 8848 windows_guest.disk_file_put(fname, data); 8849 } 8850 8851 // Unmount disk devices 8852 for it in &disk_test_data { 8853 let disk_id = it[0].as_str(); 8854 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 8855 assert!(cmd_success); 8856 thread::sleep(std::time::Duration::new(5, 0)); 8857 } 8858 8859 // Verify the devices have been removed 8860 let disk_num = 1; 8861 assert_eq!(windows_guest.disk_count(), disk_num); 8862 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8863 8864 // Remount 8865 for it in &disk_test_data { 8866 let disk = it[1].as_str(); 8867 let (cmd_success, _cmd_output) = remote_command_w_output( 8868 &api_socket, 8869 "add-disk", 8870 Some(format!("path={disk},readonly=off").as_str()), 8871 ); 8872 assert!(cmd_success); 8873 thread::sleep(std::time::Duration::new(5, 0)); 8874 } 8875 8876 // Check the files exists with the expected contents 8877 for it in &disk_test_data { 8878 let fname = it[2].as_str(); 8879 let data = it[3].as_str(); 8880 let out = windows_guest.disk_file_read(fname); 8881 assert_eq!(data, out.trim()); 8882 } 8883 8884 // Intentionally no unmount, it'll happen at shutdown. 8885 8886 windows_guest.shutdown(); 8887 }); 8888 8889 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8890 let _ = child.kill(); 8891 let output = child.wait_with_output().unwrap(); 8892 8893 let _ = child_dnsmasq.kill(); 8894 let _ = child_dnsmasq.wait(); 8895 8896 handle_child_output(r, &output); 8897 } 8898 8899 #[test] 8900 #[cfg(not(feature = "mshv"))] 8901 #[cfg(not(target_arch = "aarch64"))] 8902 fn test_windows_guest_netdev_multi() { 8903 let windows_guest = WindowsGuest::new(); 8904 8905 let mut ovmf_path = dirs::home_dir().unwrap(); 8906 ovmf_path.push("workloads"); 8907 ovmf_path.push(OVMF_NAME); 8908 8909 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8910 let api_socket = temp_api_path(&tmp_dir); 8911 8912 let mut child = GuestCommand::new(windows_guest.guest()) 8913 .args(["--api-socket", &api_socket]) 8914 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8915 .args(["--memory", "size=4G"]) 8916 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8917 .args(["--serial", "tty"]) 8918 .args(["--console", "off"]) 8919 .default_disks() 8920 // The multi net dev config is borrowed from test_multiple_network_interfaces 8921 .args([ 8922 "--net", 8923 windows_guest.guest().default_net_string().as_str(), 8924 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 8925 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 8926 ]) 8927 .capture_output() 8928 .spawn() 8929 .unwrap(); 8930 8931 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8932 8933 let r = std::panic::catch_unwind(|| { 8934 // Wait to make sure Windows boots up 8935 assert!(windows_guest.wait_for_boot()); 8936 8937 let netdev_num = 3; 8938 assert_eq!(windows_guest.netdev_count(), netdev_num); 8939 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8940 8941 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 8942 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 8943 8944 windows_guest.shutdown(); 8945 }); 8946 8947 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8948 let _ = child.kill(); 8949 let output = child.wait_with_output().unwrap(); 8950 8951 let _ = child_dnsmasq.kill(); 8952 let _ = child_dnsmasq.wait(); 8953 8954 handle_child_output(r, &output); 8955 } 8956 } 8957 8958 #[cfg(target_arch = "x86_64")] 8959 mod sgx { 8960 use crate::*; 8961 8962 #[test] 8963 fn test_sgx() { 8964 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 8965 let jammy = UbuntuDiskConfig::new(jammy_image); 8966 let guest = Guest::new(Box::new(jammy)); 8967 8968 let mut child = GuestCommand::new(&guest) 8969 .args(["--cpus", "boot=1"]) 8970 .args(["--memory", "size=512M"]) 8971 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8972 .default_disks() 8973 .default_net() 8974 .args(["--sgx-epc", "id=epc0,size=64M"]) 8975 .capture_output() 8976 .spawn() 8977 .unwrap(); 8978 8979 let r = std::panic::catch_unwind(|| { 8980 guest.wait_vm_boot(None).unwrap(); 8981 8982 // Check if SGX is correctly detected in the guest. 8983 guest.check_sgx_support().unwrap(); 8984 8985 // Validate the SGX EPC section is 64MiB. 8986 assert_eq!( 8987 guest 8988 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 8989 .unwrap() 8990 .trim(), 8991 "0x0000000004000000" 8992 ); 8993 }); 8994 8995 let _ = child.kill(); 8996 let output = child.wait_with_output().unwrap(); 8997 8998 handle_child_output(r, &output); 8999 } 9000 } 9001 9002 #[cfg(target_arch = "x86_64")] 9003 mod vfio { 9004 use crate::*; 9005 const NVIDIA_VFIO_DEVICE: &str = "/sys/bus/pci/devices/0002:00:01.0"; 9006 9007 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 9008 let jammy = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string()); 9009 let guest = Guest::new(Box::new(jammy)); 9010 let api_socket = temp_api_path(&guest.tmp_dir); 9011 9012 let mut child = GuestCommand::new(&guest) 9013 .args(["--cpus", "boot=4"]) 9014 .args([ 9015 "--memory", 9016 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(), 9017 ]) 9018 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9019 .args(["--device", format!("path={NVIDIA_VFIO_DEVICE}").as_str()]) 9020 .args(["--api-socket", &api_socket]) 9021 .default_disks() 9022 .default_net() 9023 .capture_output() 9024 .spawn() 9025 .unwrap(); 9026 9027 let r = std::panic::catch_unwind(|| { 9028 guest.wait_vm_boot(None).unwrap(); 9029 9030 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9031 9032 guest.enable_memory_hotplug(); 9033 9034 // Add RAM to the VM 9035 let desired_ram = 6 << 30; 9036 resize_command(&api_socket, None, Some(desired_ram), None, None); 9037 thread::sleep(std::time::Duration::new(30, 0)); 9038 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9039 9040 // Check the VFIO device works when RAM is increased to 6GiB 9041 guest.check_nvidia_gpu(); 9042 }); 9043 9044 let _ = child.kill(); 9045 let output = child.wait_with_output().unwrap(); 9046 9047 handle_child_output(r, &output); 9048 } 9049 9050 #[test] 9051 fn test_nvidia_card_memory_hotplug_acpi() { 9052 test_nvidia_card_memory_hotplug("acpi") 9053 } 9054 9055 #[test] 9056 fn test_nvidia_card_memory_hotplug_virtio_mem() { 9057 test_nvidia_card_memory_hotplug("virtio-mem") 9058 } 9059 9060 #[test] 9061 fn test_nvidia_card_pci_hotplug() { 9062 let jammy = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string()); 9063 let guest = Guest::new(Box::new(jammy)); 9064 let api_socket = temp_api_path(&guest.tmp_dir); 9065 9066 let mut child = GuestCommand::new(&guest) 9067 .args(["--cpus", "boot=4"]) 9068 .args(["--memory", "size=4G"]) 9069 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9070 .args(["--api-socket", &api_socket]) 9071 .default_disks() 9072 .default_net() 9073 .capture_output() 9074 .spawn() 9075 .unwrap(); 9076 9077 let r = std::panic::catch_unwind(|| { 9078 guest.wait_vm_boot(None).unwrap(); 9079 9080 // Hotplug the card to the VM 9081 let (cmd_success, cmd_output) = remote_command_w_output( 9082 &api_socket, 9083 "add-device", 9084 Some(format!("id=vfio0,path={NVIDIA_VFIO_DEVICE}").as_str()), 9085 ); 9086 assert!(cmd_success); 9087 assert!(String::from_utf8_lossy(&cmd_output) 9088 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 9089 9090 thread::sleep(std::time::Duration::new(10, 0)); 9091 9092 // Check the VFIO device works after hotplug 9093 guest.check_nvidia_gpu(); 9094 }); 9095 9096 let _ = child.kill(); 9097 let output = child.wait_with_output().unwrap(); 9098 9099 handle_child_output(r, &output); 9100 } 9101 9102 #[test] 9103 fn test_nvidia_card_reboot() { 9104 let jammy = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string()); 9105 let guest = Guest::new(Box::new(jammy)); 9106 let api_socket = temp_api_path(&guest.tmp_dir); 9107 9108 let mut child = GuestCommand::new(&guest) 9109 .args(["--cpus", "boot=4"]) 9110 .args(["--memory", "size=4G"]) 9111 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9112 .args(["--device", format!("path={NVIDIA_VFIO_DEVICE}").as_str()]) 9113 .args(["--api-socket", &api_socket]) 9114 .default_disks() 9115 .default_net() 9116 .capture_output() 9117 .spawn() 9118 .unwrap(); 9119 9120 let r = std::panic::catch_unwind(|| { 9121 guest.wait_vm_boot(None).unwrap(); 9122 9123 // Check the VFIO device works after boot 9124 guest.check_nvidia_gpu(); 9125 9126 guest.reboot_linux(0, None); 9127 9128 // Check the VFIO device works after reboot 9129 guest.check_nvidia_gpu(); 9130 }); 9131 9132 let _ = child.kill(); 9133 let output = child.wait_with_output().unwrap(); 9134 9135 handle_child_output(r, &output); 9136 } 9137 } 9138 9139 mod live_migration { 9140 use crate::*; 9141 9142 fn start_live_migration( 9143 migration_socket: &str, 9144 src_api_socket: &str, 9145 dest_api_socket: &str, 9146 local: bool, 9147 ) -> bool { 9148 // Start to receive migration from the destination VM 9149 let mut receive_migration = Command::new(clh_command("ch-remote")) 9150 .args([ 9151 &format!("--api-socket={dest_api_socket}"), 9152 "receive-migration", 9153 &format! {"unix:{migration_socket}"}, 9154 ]) 9155 .stderr(Stdio::piped()) 9156 .stdout(Stdio::piped()) 9157 .spawn() 9158 .unwrap(); 9159 // Give it '1s' to make sure the 'migration_socket' file is properly created 9160 thread::sleep(std::time::Duration::new(1, 0)); 9161 // Start to send migration from the source VM 9162 9163 let mut args = [ 9164 format!("--api-socket={}", &src_api_socket), 9165 "send-migration".to_string(), 9166 format! {"unix:{migration_socket}"}, 9167 ] 9168 .to_vec(); 9169 9170 if local { 9171 args.insert(2, "--local".to_string()); 9172 } 9173 9174 let mut send_migration = Command::new(clh_command("ch-remote")) 9175 .args(&args) 9176 .stderr(Stdio::piped()) 9177 .stdout(Stdio::piped()) 9178 .spawn() 9179 .unwrap(); 9180 9181 // The 'send-migration' command should be executed successfully within the given timeout 9182 let send_success = if let Some(status) = send_migration 9183 .wait_timeout(std::time::Duration::from_secs(30)) 9184 .unwrap() 9185 { 9186 status.success() 9187 } else { 9188 false 9189 }; 9190 9191 if !send_success { 9192 let _ = send_migration.kill(); 9193 let output = send_migration.wait_with_output().unwrap(); 9194 eprintln!( 9195 "\n\n==== Start 'send_migration' output ==== \ 9196 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 9197 \n\n==== End 'send_migration' output ====\n\n", 9198 String::from_utf8_lossy(&output.stdout), 9199 String::from_utf8_lossy(&output.stderr) 9200 ); 9201 } 9202 9203 // The 'receive-migration' command should be executed successfully within the given timeout 9204 let receive_success = if let Some(status) = receive_migration 9205 .wait_timeout(std::time::Duration::from_secs(30)) 9206 .unwrap() 9207 { 9208 status.success() 9209 } else { 9210 false 9211 }; 9212 9213 if !receive_success { 9214 let _ = receive_migration.kill(); 9215 let output = receive_migration.wait_with_output().unwrap(); 9216 eprintln!( 9217 "\n\n==== Start 'receive_migration' output ==== \ 9218 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 9219 \n\n==== End 'receive_migration' output ====\n\n", 9220 String::from_utf8_lossy(&output.stdout), 9221 String::from_utf8_lossy(&output.stderr) 9222 ); 9223 } 9224 9225 send_success && receive_success 9226 } 9227 9228 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 9229 let mut src_vm = src_vm; 9230 let mut dest_vm = dest_vm; 9231 9232 let _ = src_vm.kill(); 9233 let src_output = src_vm.wait_with_output().unwrap(); 9234 eprintln!( 9235 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 9236 String::from_utf8_lossy(&src_output.stdout) 9237 ); 9238 eprintln!( 9239 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 9240 String::from_utf8_lossy(&src_output.stderr) 9241 ); 9242 let _ = dest_vm.kill(); 9243 let dest_output = dest_vm.wait_with_output().unwrap(); 9244 eprintln!( 9245 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 9246 String::from_utf8_lossy(&dest_output.stdout) 9247 ); 9248 eprintln!( 9249 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 9250 String::from_utf8_lossy(&dest_output.stderr) 9251 ); 9252 9253 if let Some(ovs_vm) = ovs_vm { 9254 let mut ovs_vm = ovs_vm; 9255 let _ = ovs_vm.kill(); 9256 let ovs_output = ovs_vm.wait_with_output().unwrap(); 9257 eprintln!( 9258 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 9259 String::from_utf8_lossy(&ovs_output.stdout) 9260 ); 9261 eprintln!( 9262 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 9263 String::from_utf8_lossy(&ovs_output.stderr) 9264 ); 9265 9266 cleanup_ovs_dpdk(); 9267 } 9268 9269 panic!("Test failed: {message}") 9270 } 9271 9272 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 9273 // same host. It ensures the following behaviors: 9274 // 1. The source VM is up and functional (including various virtio-devices are working properly); 9275 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 9276 // 3. The source VM terminated gracefully after live migration; 9277 // 4. The destination VM is functional (including various virtio-devices are working properly) after 9278 // live migration; 9279 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 9280 fn _test_live_migration(upgrade_test: bool, local: bool) { 9281 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9282 let guest = Guest::new(Box::new(focal)); 9283 let kernel_path = direct_kernel_boot_path(); 9284 let console_text = String::from("On a branch floating down river a cricket, singing."); 9285 let net_id = "net123"; 9286 let net_params = format!( 9287 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9288 net_id, guest.network.guest_mac, guest.network.host_ip 9289 ); 9290 9291 let memory_param: &[&str] = if local { 9292 &["--memory", "size=4G,shared=on"] 9293 } else { 9294 &["--memory", "size=4G"] 9295 }; 9296 9297 let boot_vcpus = 2; 9298 let max_vcpus = 4; 9299 9300 let pmem_temp_file = TempFile::new().unwrap(); 9301 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9302 std::process::Command::new("mkfs.ext4") 9303 .arg(pmem_temp_file.as_path()) 9304 .output() 9305 .expect("Expect creating disk image to succeed"); 9306 let pmem_path = String::from("/dev/pmem0"); 9307 9308 // Start the source VM 9309 let src_vm_path = if !upgrade_test { 9310 clh_command("cloud-hypervisor") 9311 } else { 9312 cloud_hypervisor_release_path() 9313 }; 9314 let src_api_socket = temp_api_path(&guest.tmp_dir); 9315 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9316 src_vm_cmd 9317 .args([ 9318 "--cpus", 9319 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9320 ]) 9321 .args(memory_param) 9322 .args(["--kernel", kernel_path.to_str().unwrap()]) 9323 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9324 .default_disks() 9325 .args(["--net", net_params.as_str()]) 9326 .args(["--api-socket", &src_api_socket]) 9327 .args([ 9328 "--pmem", 9329 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9330 ]); 9331 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9332 9333 // Start the destination VM 9334 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9335 dest_api_socket.push_str(".dest"); 9336 let mut dest_child = GuestCommand::new(&guest) 9337 .args(["--api-socket", &dest_api_socket]) 9338 .capture_output() 9339 .spawn() 9340 .unwrap(); 9341 9342 let r = std::panic::catch_unwind(|| { 9343 guest.wait_vm_boot(None).unwrap(); 9344 9345 // Make sure the source VM is functional 9346 // Check the number of vCPUs 9347 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9348 9349 // Check the guest RAM 9350 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9351 9352 // Check the guest virtio-devices, e.g. block, rng, console, and net 9353 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9354 9355 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9356 // to make sure that removing and adding back the virtio-net device does 9357 // not break the live-migration support for virtio-pci. 9358 #[cfg(target_arch = "x86_64")] 9359 { 9360 assert!(remote_command( 9361 &src_api_socket, 9362 "remove-device", 9363 Some(net_id), 9364 )); 9365 thread::sleep(std::time::Duration::new(10, 0)); 9366 9367 // Plug the virtio-net device again 9368 assert!(remote_command( 9369 &src_api_socket, 9370 "add-net", 9371 Some(net_params.as_str()), 9372 )); 9373 thread::sleep(std::time::Duration::new(10, 0)); 9374 } 9375 9376 // Start the live-migration 9377 let migration_socket = String::from( 9378 guest 9379 .tmp_dir 9380 .as_path() 9381 .join("live-migration.sock") 9382 .to_str() 9383 .unwrap(), 9384 ); 9385 9386 assert!( 9387 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9388 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9389 ); 9390 }); 9391 9392 // Check and report any errors occurred during the live-migration 9393 if r.is_err() { 9394 print_and_panic( 9395 src_child, 9396 dest_child, 9397 None, 9398 "Error occurred during live-migration", 9399 ); 9400 } 9401 9402 // Check the source vm has been terminated successful (give it '3s' to settle) 9403 thread::sleep(std::time::Duration::new(3, 0)); 9404 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 9405 print_and_panic( 9406 src_child, 9407 dest_child, 9408 None, 9409 "source VM was not terminated successfully.", 9410 ); 9411 }; 9412 9413 // Post live-migration check to make sure the destination VM is functional 9414 let r = std::panic::catch_unwind(|| { 9415 // Perform same checks to validate VM has been properly migrated 9416 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9417 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9418 9419 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9420 }); 9421 9422 // Clean-up the destination VM and make sure it terminated correctly 9423 let _ = dest_child.kill(); 9424 let dest_output = dest_child.wait_with_output().unwrap(); 9425 handle_child_output(r, &dest_output); 9426 9427 // Check the destination VM has the expected 'console_text' from its output 9428 let r = std::panic::catch_unwind(|| { 9429 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9430 }); 9431 handle_child_output(r, &dest_output); 9432 } 9433 9434 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 9435 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9436 let guest = Guest::new(Box::new(focal)); 9437 let kernel_path = direct_kernel_boot_path(); 9438 let console_text = String::from("On a branch floating down river a cricket, singing."); 9439 let net_id = "net123"; 9440 let net_params = format!( 9441 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9442 net_id, guest.network.guest_mac, guest.network.host_ip 9443 ); 9444 9445 let memory_param: &[&str] = if local { 9446 &[ 9447 "--memory", 9448 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 9449 "--balloon", 9450 "size=0", 9451 ] 9452 } else { 9453 &[ 9454 "--memory", 9455 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 9456 "--balloon", 9457 "size=0", 9458 ] 9459 }; 9460 9461 let boot_vcpus = 2; 9462 let max_vcpus = 4; 9463 9464 let pmem_temp_file = TempFile::new().unwrap(); 9465 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9466 std::process::Command::new("mkfs.ext4") 9467 .arg(pmem_temp_file.as_path()) 9468 .output() 9469 .expect("Expect creating disk image to succeed"); 9470 let pmem_path = String::from("/dev/pmem0"); 9471 9472 // Start the source VM 9473 let src_vm_path = if !upgrade_test { 9474 clh_command("cloud-hypervisor") 9475 } else { 9476 cloud_hypervisor_release_path() 9477 }; 9478 let src_api_socket = temp_api_path(&guest.tmp_dir); 9479 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9480 src_vm_cmd 9481 .args([ 9482 "--cpus", 9483 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9484 ]) 9485 .args(memory_param) 9486 .args(["--kernel", kernel_path.to_str().unwrap()]) 9487 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9488 .default_disks() 9489 .args(["--net", net_params.as_str()]) 9490 .args(["--api-socket", &src_api_socket]) 9491 .args([ 9492 "--pmem", 9493 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9494 ]); 9495 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9496 9497 // Start the destination VM 9498 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9499 dest_api_socket.push_str(".dest"); 9500 let mut dest_child = GuestCommand::new(&guest) 9501 .args(["--api-socket", &dest_api_socket]) 9502 .capture_output() 9503 .spawn() 9504 .unwrap(); 9505 9506 let r = std::panic::catch_unwind(|| { 9507 guest.wait_vm_boot(None).unwrap(); 9508 9509 // Make sure the source VM is functional 9510 // Check the number of vCPUs 9511 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9512 9513 // Check the guest RAM 9514 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9515 // Increase the guest RAM 9516 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 9517 thread::sleep(std::time::Duration::new(5, 0)); 9518 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9519 // Use balloon to remove RAM from the VM 9520 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 9521 thread::sleep(std::time::Duration::new(5, 0)); 9522 let total_memory = guest.get_total_memory().unwrap_or_default(); 9523 assert!(total_memory > 4_800_000); 9524 assert!(total_memory < 5_760_000); 9525 9526 // Check the guest virtio-devices, e.g. block, rng, console, and net 9527 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9528 9529 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9530 // to make sure that removing and adding back the virtio-net device does 9531 // not break the live-migration support for virtio-pci. 9532 #[cfg(target_arch = "x86_64")] 9533 { 9534 assert!(remote_command( 9535 &src_api_socket, 9536 "remove-device", 9537 Some(net_id), 9538 )); 9539 thread::sleep(std::time::Duration::new(10, 0)); 9540 9541 // Plug the virtio-net device again 9542 assert!(remote_command( 9543 &src_api_socket, 9544 "add-net", 9545 Some(net_params.as_str()), 9546 )); 9547 thread::sleep(std::time::Duration::new(10, 0)); 9548 } 9549 9550 // Start the live-migration 9551 let migration_socket = String::from( 9552 guest 9553 .tmp_dir 9554 .as_path() 9555 .join("live-migration.sock") 9556 .to_str() 9557 .unwrap(), 9558 ); 9559 9560 assert!( 9561 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9562 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9563 ); 9564 }); 9565 9566 // Check and report any errors occurred during the live-migration 9567 if r.is_err() { 9568 print_and_panic( 9569 src_child, 9570 dest_child, 9571 None, 9572 "Error occurred during live-migration", 9573 ); 9574 } 9575 9576 // Check the source vm has been terminated successful (give it '3s' to settle) 9577 thread::sleep(std::time::Duration::new(3, 0)); 9578 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 9579 print_and_panic( 9580 src_child, 9581 dest_child, 9582 None, 9583 "source VM was not terminated successfully.", 9584 ); 9585 }; 9586 9587 // Post live-migration check to make sure the destination VM is functional 9588 let r = std::panic::catch_unwind(|| { 9589 // Perform same checks to validate VM has been properly migrated 9590 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9591 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9592 9593 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9594 9595 // Perform checks on guest RAM using balloon 9596 let total_memory = guest.get_total_memory().unwrap_or_default(); 9597 assert!(total_memory > 4_800_000); 9598 assert!(total_memory < 5_760_000); 9599 // Deflate balloon to restore entire RAM to the VM 9600 resize_command(&dest_api_socket, None, None, Some(0), None); 9601 thread::sleep(std::time::Duration::new(5, 0)); 9602 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9603 // Decrease guest RAM with virtio-mem 9604 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 9605 thread::sleep(std::time::Duration::new(5, 0)); 9606 let total_memory = guest.get_total_memory().unwrap_or_default(); 9607 assert!(total_memory > 4_800_000); 9608 assert!(total_memory < 5_760_000); 9609 }); 9610 9611 // Clean-up the destination VM and make sure it terminated correctly 9612 let _ = dest_child.kill(); 9613 let dest_output = dest_child.wait_with_output().unwrap(); 9614 handle_child_output(r, &dest_output); 9615 9616 // Check the destination VM has the expected 'console_text' from its output 9617 let r = std::panic::catch_unwind(|| { 9618 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9619 }); 9620 handle_child_output(r, &dest_output); 9621 } 9622 9623 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 9624 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9625 let guest = Guest::new(Box::new(focal)); 9626 let kernel_path = direct_kernel_boot_path(); 9627 let console_text = String::from("On a branch floating down river a cricket, singing."); 9628 let net_id = "net123"; 9629 let net_params = format!( 9630 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9631 net_id, guest.network.guest_mac, guest.network.host_ip 9632 ); 9633 9634 let memory_param: &[&str] = if local { 9635 &[ 9636 "--memory", 9637 "size=0,hotplug_method=virtio-mem,shared=on", 9638 "--memory-zone", 9639 "id=mem0,size=1G,hotplug_size=4G,shared=on", 9640 "id=mem1,size=1G,hotplug_size=4G,shared=on", 9641 "id=mem2,size=2G,hotplug_size=4G,shared=on", 9642 "--numa", 9643 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9644 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9645 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9646 ] 9647 } else { 9648 &[ 9649 "--memory", 9650 "size=0,hotplug_method=virtio-mem", 9651 "--memory-zone", 9652 "id=mem0,size=1G,hotplug_size=4G", 9653 "id=mem1,size=1G,hotplug_size=4G", 9654 "id=mem2,size=2G,hotplug_size=4G", 9655 "--numa", 9656 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9657 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9658 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9659 ] 9660 }; 9661 9662 let boot_vcpus = 6; 9663 let max_vcpus = 12; 9664 9665 let pmem_temp_file = TempFile::new().unwrap(); 9666 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9667 std::process::Command::new("mkfs.ext4") 9668 .arg(pmem_temp_file.as_path()) 9669 .output() 9670 .expect("Expect creating disk image to succeed"); 9671 let pmem_path = String::from("/dev/pmem0"); 9672 9673 // Start the source VM 9674 let src_vm_path = if !upgrade_test { 9675 clh_command("cloud-hypervisor") 9676 } else { 9677 cloud_hypervisor_release_path() 9678 }; 9679 let src_api_socket = temp_api_path(&guest.tmp_dir); 9680 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9681 src_vm_cmd 9682 .args([ 9683 "--cpus", 9684 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9685 ]) 9686 .args(memory_param) 9687 .args(["--kernel", kernel_path.to_str().unwrap()]) 9688 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9689 .default_disks() 9690 .args(["--net", net_params.as_str()]) 9691 .args(["--api-socket", &src_api_socket]) 9692 .args([ 9693 "--pmem", 9694 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9695 ]); 9696 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9697 9698 // Start the destination VM 9699 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9700 dest_api_socket.push_str(".dest"); 9701 let mut dest_child = GuestCommand::new(&guest) 9702 .args(["--api-socket", &dest_api_socket]) 9703 .capture_output() 9704 .spawn() 9705 .unwrap(); 9706 9707 let r = std::panic::catch_unwind(|| { 9708 guest.wait_vm_boot(None).unwrap(); 9709 9710 // Make sure the source VM is functional 9711 // Check the number of vCPUs 9712 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9713 9714 // Check the guest RAM 9715 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 9716 9717 // Check the guest virtio-devices, e.g. block, rng, console, and net 9718 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9719 9720 // Check the NUMA parameters are applied correctly and resize 9721 // each zone to test the case where we migrate a VM with the 9722 // virtio-mem regions being used. 9723 { 9724 guest.check_numa_common( 9725 Some(&[960_000, 960_000, 1_920_000]), 9726 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9727 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9728 ); 9729 9730 // AArch64 currently does not support hotplug, and therefore we only 9731 // test hotplug-related function on x86_64 here. 9732 #[cfg(target_arch = "x86_64")] 9733 { 9734 guest.enable_memory_hotplug(); 9735 9736 // Resize every memory zone and check each associated NUMA node 9737 // has been assigned the right amount of memory. 9738 resize_zone_command(&src_api_socket, "mem0", "2G"); 9739 resize_zone_command(&src_api_socket, "mem1", "2G"); 9740 resize_zone_command(&src_api_socket, "mem2", "3G"); 9741 thread::sleep(std::time::Duration::new(5, 0)); 9742 9743 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 9744 } 9745 } 9746 9747 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9748 // to make sure that removing and adding back the virtio-net device does 9749 // not break the live-migration support for virtio-pci. 9750 #[cfg(target_arch = "x86_64")] 9751 { 9752 assert!(remote_command( 9753 &src_api_socket, 9754 "remove-device", 9755 Some(net_id), 9756 )); 9757 thread::sleep(std::time::Duration::new(10, 0)); 9758 9759 // Plug the virtio-net device again 9760 assert!(remote_command( 9761 &src_api_socket, 9762 "add-net", 9763 Some(net_params.as_str()), 9764 )); 9765 thread::sleep(std::time::Duration::new(10, 0)); 9766 } 9767 9768 // Start the live-migration 9769 let migration_socket = String::from( 9770 guest 9771 .tmp_dir 9772 .as_path() 9773 .join("live-migration.sock") 9774 .to_str() 9775 .unwrap(), 9776 ); 9777 9778 assert!( 9779 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9780 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9781 ); 9782 }); 9783 9784 // Check and report any errors occurred during the live-migration 9785 if r.is_err() { 9786 print_and_panic( 9787 src_child, 9788 dest_child, 9789 None, 9790 "Error occurred during live-migration", 9791 ); 9792 } 9793 9794 // Check the source vm has been terminated successful (give it '3s' to settle) 9795 thread::sleep(std::time::Duration::new(3, 0)); 9796 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 9797 print_and_panic( 9798 src_child, 9799 dest_child, 9800 None, 9801 "source VM was not terminated successfully.", 9802 ); 9803 }; 9804 9805 // Post live-migration check to make sure the destination VM is functional 9806 let r = std::panic::catch_unwind(|| { 9807 // Perform same checks to validate VM has been properly migrated 9808 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9809 #[cfg(target_arch = "x86_64")] 9810 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 9811 #[cfg(target_arch = "aarch64")] 9812 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9813 9814 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9815 9816 // Perform NUMA related checks 9817 { 9818 #[cfg(target_arch = "aarch64")] 9819 { 9820 guest.check_numa_common( 9821 Some(&[960_000, 960_000, 1_920_000]), 9822 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9823 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9824 ); 9825 } 9826 9827 // AArch64 currently does not support hotplug, and therefore we only 9828 // test hotplug-related function on x86_64 here. 9829 #[cfg(target_arch = "x86_64")] 9830 { 9831 guest.check_numa_common( 9832 Some(&[1_920_000, 1_920_000, 2_880_000]), 9833 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9834 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9835 ); 9836 9837 guest.enable_memory_hotplug(); 9838 9839 // Resize every memory zone and check each associated NUMA node 9840 // has been assigned the right amount of memory. 9841 resize_zone_command(&dest_api_socket, "mem0", "4G"); 9842 resize_zone_command(&dest_api_socket, "mem1", "4G"); 9843 resize_zone_command(&dest_api_socket, "mem2", "4G"); 9844 // Resize to the maximum amount of CPUs and check each NUMA 9845 // node has been assigned the right CPUs set. 9846 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 9847 thread::sleep(std::time::Duration::new(5, 0)); 9848 9849 guest.check_numa_common( 9850 Some(&[3_840_000, 3_840_000, 3_840_000]), 9851 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 9852 None, 9853 ); 9854 } 9855 } 9856 }); 9857 9858 // Clean-up the destination VM and make sure it terminated correctly 9859 let _ = dest_child.kill(); 9860 let dest_output = dest_child.wait_with_output().unwrap(); 9861 handle_child_output(r, &dest_output); 9862 9863 // Check the destination VM has the expected 'console_text' from its output 9864 let r = std::panic::catch_unwind(|| { 9865 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9866 }); 9867 handle_child_output(r, &dest_output); 9868 } 9869 9870 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 9871 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9872 let guest = Guest::new(Box::new(focal)); 9873 let kernel_path = direct_kernel_boot_path(); 9874 let console_text = String::from("On a branch floating down river a cricket, singing."); 9875 let net_id = "net123"; 9876 let net_params = format!( 9877 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9878 net_id, guest.network.guest_mac, guest.network.host_ip 9879 ); 9880 9881 let memory_param: &[&str] = if local { 9882 &["--memory", "size=4G,shared=on"] 9883 } else { 9884 &["--memory", "size=4G"] 9885 }; 9886 9887 let boot_vcpus = 2; 9888 let max_vcpus = 4; 9889 9890 let pmem_temp_file = TempFile::new().unwrap(); 9891 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9892 std::process::Command::new("mkfs.ext4") 9893 .arg(pmem_temp_file.as_path()) 9894 .output() 9895 .expect("Expect creating disk image to succeed"); 9896 let pmem_path = String::from("/dev/pmem0"); 9897 9898 // Start the source VM 9899 let src_vm_path = if !upgrade_test { 9900 clh_command("cloud-hypervisor") 9901 } else { 9902 cloud_hypervisor_release_path() 9903 }; 9904 let src_api_socket = temp_api_path(&guest.tmp_dir); 9905 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9906 src_vm_cmd 9907 .args([ 9908 "--cpus", 9909 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9910 ]) 9911 .args(memory_param) 9912 .args(["--kernel", kernel_path.to_str().unwrap()]) 9913 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9914 .default_disks() 9915 .args(["--net", net_params.as_str()]) 9916 .args(["--api-socket", &src_api_socket]) 9917 .args([ 9918 "--pmem", 9919 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9920 ]) 9921 .args(["--watchdog"]); 9922 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9923 9924 // Start the destination VM 9925 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9926 dest_api_socket.push_str(".dest"); 9927 let mut dest_child = GuestCommand::new(&guest) 9928 .args(["--api-socket", &dest_api_socket]) 9929 .capture_output() 9930 .spawn() 9931 .unwrap(); 9932 9933 let r = std::panic::catch_unwind(|| { 9934 guest.wait_vm_boot(None).unwrap(); 9935 9936 // Make sure the source VM is functional 9937 // Check the number of vCPUs 9938 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9939 // Check the guest RAM 9940 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9941 // Check the guest virtio-devices, e.g. block, rng, console, and net 9942 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9943 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9944 // to make sure that removing and adding back the virtio-net device does 9945 // not break the live-migration support for virtio-pci. 9946 #[cfg(target_arch = "x86_64")] 9947 { 9948 assert!(remote_command( 9949 &src_api_socket, 9950 "remove-device", 9951 Some(net_id), 9952 )); 9953 thread::sleep(std::time::Duration::new(10, 0)); 9954 9955 // Plug the virtio-net device again 9956 assert!(remote_command( 9957 &src_api_socket, 9958 "add-net", 9959 Some(net_params.as_str()), 9960 )); 9961 thread::sleep(std::time::Duration::new(10, 0)); 9962 } 9963 9964 // Enable watchdog and ensure its functional 9965 let expected_reboot_count = 1; 9966 // Enable the watchdog with a 15s timeout 9967 enable_guest_watchdog(&guest, 15); 9968 9969 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9970 assert_eq!( 9971 guest 9972 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 9973 .unwrap() 9974 .trim() 9975 .parse::<u32>() 9976 .unwrap_or_default(), 9977 1 9978 ); 9979 // Allow some normal time to elapse to check we don't get spurious reboots 9980 thread::sleep(std::time::Duration::new(40, 0)); 9981 // Check no reboot 9982 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9983 9984 // Start the live-migration 9985 let migration_socket = String::from( 9986 guest 9987 .tmp_dir 9988 .as_path() 9989 .join("live-migration.sock") 9990 .to_str() 9991 .unwrap(), 9992 ); 9993 9994 assert!( 9995 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9996 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9997 ); 9998 }); 9999 10000 // Check and report any errors occurred during the live-migration 10001 if r.is_err() { 10002 print_and_panic( 10003 src_child, 10004 dest_child, 10005 None, 10006 "Error occurred during live-migration", 10007 ); 10008 } 10009 10010 // Check the source vm has been terminated successful (give it '3s' to settle) 10011 thread::sleep(std::time::Duration::new(3, 0)); 10012 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 10013 print_and_panic( 10014 src_child, 10015 dest_child, 10016 None, 10017 "source VM was not terminated successfully.", 10018 ); 10019 }; 10020 10021 // Post live-migration check to make sure the destination VM is functional 10022 let r = std::panic::catch_unwind(|| { 10023 // Perform same checks to validate VM has been properly migrated 10024 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10025 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10026 10027 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 10028 10029 // Perform checks on watchdog 10030 let mut expected_reboot_count = 1; 10031 10032 // Allow some normal time to elapse to check we don't get spurious reboots 10033 thread::sleep(std::time::Duration::new(40, 0)); 10034 // Check no reboot 10035 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10036 10037 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 10038 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 10039 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 10040 guest.wait_vm_boot(Some(50)).unwrap(); 10041 // Check a reboot is triggered by the watchdog 10042 expected_reboot_count += 1; 10043 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10044 10045 #[cfg(target_arch = "x86_64")] 10046 { 10047 // Now pause the VM and remain offline for 30s 10048 assert!(remote_command(&dest_api_socket, "pause", None)); 10049 thread::sleep(std::time::Duration::new(30, 0)); 10050 assert!(remote_command(&dest_api_socket, "resume", None)); 10051 10052 // Check no reboot 10053 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10054 } 10055 }); 10056 10057 // Clean-up the destination VM and make sure it terminated correctly 10058 let _ = dest_child.kill(); 10059 let dest_output = dest_child.wait_with_output().unwrap(); 10060 handle_child_output(r, &dest_output); 10061 10062 // Check the destination VM has the expected 'console_text' from its output 10063 let r = std::panic::catch_unwind(|| { 10064 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 10065 }); 10066 handle_child_output(r, &dest_output); 10067 } 10068 10069 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 10070 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10071 let ovs_guest = Guest::new(Box::new(ovs_focal)); 10072 10073 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10074 let migration_guest = Guest::new(Box::new(migration_focal)); 10075 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 10076 10077 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 10078 let (mut ovs_child, mut src_child) = 10079 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 10080 10081 // Start the destination VM 10082 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 10083 dest_api_socket.push_str(".dest"); 10084 let mut dest_child = GuestCommand::new(&migration_guest) 10085 .args(["--api-socket", &dest_api_socket]) 10086 .capture_output() 10087 .spawn() 10088 .unwrap(); 10089 10090 let r = std::panic::catch_unwind(|| { 10091 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 10092 thread::sleep(std::time::Duration::new(1, 0)); 10093 10094 // Start the live-migration 10095 let migration_socket = String::from( 10096 migration_guest 10097 .tmp_dir 10098 .as_path() 10099 .join("live-migration.sock") 10100 .to_str() 10101 .unwrap(), 10102 ); 10103 10104 assert!( 10105 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 10106 "Unsuccessful command: 'send-migration' or 'receive-migration'." 10107 ); 10108 }); 10109 10110 // Check and report any errors occurred during the live-migration 10111 if r.is_err() { 10112 print_and_panic( 10113 src_child, 10114 dest_child, 10115 Some(ovs_child), 10116 "Error occurred during live-migration", 10117 ); 10118 } 10119 10120 // Check the source vm has been terminated successful (give it '3s' to settle) 10121 thread::sleep(std::time::Duration::new(3, 0)); 10122 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 10123 print_and_panic( 10124 src_child, 10125 dest_child, 10126 Some(ovs_child), 10127 "source VM was not terminated successfully.", 10128 ); 10129 }; 10130 10131 // Post live-migration check to make sure the destination VM is functional 10132 let r = std::panic::catch_unwind(|| { 10133 // Perform same checks to validate VM has been properly migrated 10134 // Spawn a new netcat listener in the OVS VM 10135 let guest_ip = ovs_guest.network.guest_ip.clone(); 10136 thread::spawn(move || { 10137 ssh_command_ip( 10138 "nc -l 12345", 10139 &guest_ip, 10140 DEFAULT_SSH_RETRIES, 10141 DEFAULT_SSH_TIMEOUT, 10142 ) 10143 .unwrap(); 10144 }); 10145 10146 // Wait for the server to be listening 10147 thread::sleep(std::time::Duration::new(5, 0)); 10148 10149 // And check the connection is still functional after live-migration 10150 migration_guest 10151 .ssh_command("nc -vz 172.100.0.1 12345") 10152 .unwrap(); 10153 }); 10154 10155 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 10156 let _ = dest_child.kill(); 10157 let _ = ovs_child.kill(); 10158 let dest_output = dest_child.wait_with_output().unwrap(); 10159 let ovs_output = ovs_child.wait_with_output().unwrap(); 10160 10161 cleanup_ovs_dpdk(); 10162 10163 handle_child_output(r, &dest_output); 10164 handle_child_output(Ok(()), &ovs_output); 10165 } 10166 10167 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 10168 // same host with Landlock enabled on both VMs. The test validates the following: 10169 // 1. The source VM is up and functional 10170 // 2. Ensure Landlock is enabled on source VM by hotplugging a disk. As the path for this 10171 // disk is not known to the source VM this step will fail. 10172 // 3. The 'send-migration' and 'receive-migration' command finished successfully; 10173 // 4. The source VM terminated gracefully after live migration; 10174 // 5. The destination VM is functional after live migration; 10175 // 6. Ensure Landlock is enabled on destination VM by hotplugging a disk. As the path for 10176 // this disk is not known to the destination VM this step will fail. 10177 fn _test_live_migration_with_landlock() { 10178 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10179 let guest = Guest::new(Box::new(focal)); 10180 let kernel_path = direct_kernel_boot_path(); 10181 let net_id = "net123"; 10182 let net_params = format!( 10183 "id={},tap=,mac={},ip={},mask=255.255.255.0", 10184 net_id, guest.network.guest_mac, guest.network.host_ip 10185 ); 10186 10187 let boot_vcpus = 2; 10188 let max_vcpus = 4; 10189 10190 let mut blk_file_path = dirs::home_dir().unwrap(); 10191 blk_file_path.push("workloads"); 10192 blk_file_path.push("blk.img"); 10193 10194 let src_api_socket = temp_api_path(&guest.tmp_dir); 10195 let mut src_child = GuestCommand::new(&guest) 10196 .args([ 10197 "--cpus", 10198 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 10199 ]) 10200 .args(["--memory", "size=4G,shared=on"]) 10201 .args(["--kernel", kernel_path.to_str().unwrap()]) 10202 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10203 .default_disks() 10204 .args(["--api-socket", &src_api_socket]) 10205 .args(["--landlock"]) 10206 .args(["--net", net_params.as_str()]) 10207 .args([ 10208 "--landlock-rules", 10209 format!("path={:?},access=rw", guest.tmp_dir.as_path()).as_str(), 10210 ]) 10211 .capture_output() 10212 .spawn() 10213 .unwrap(); 10214 10215 // Start the destination VM 10216 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 10217 dest_api_socket.push_str(".dest"); 10218 let mut dest_child = GuestCommand::new(&guest) 10219 .args(["--api-socket", &dest_api_socket]) 10220 .capture_output() 10221 .spawn() 10222 .unwrap(); 10223 10224 let r = std::panic::catch_unwind(|| { 10225 guest.wait_vm_boot(None).unwrap(); 10226 10227 // Make sure the source VM is functaionl 10228 // Check the number of vCPUs 10229 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10230 10231 // Check the guest RAM 10232 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10233 10234 // Check Landlock is enabled by hot-plugging a disk. 10235 assert!(!remote_command( 10236 &src_api_socket, 10237 "add-disk", 10238 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 10239 )); 10240 10241 // Start the live-migration 10242 let migration_socket = String::from( 10243 guest 10244 .tmp_dir 10245 .as_path() 10246 .join("live-migration.sock") 10247 .to_str() 10248 .unwrap(), 10249 ); 10250 10251 assert!( 10252 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, true), 10253 "Unsuccessful command: 'send-migration' or 'receive-migration'." 10254 ); 10255 }); 10256 10257 // Check and report any errors occurred during the live-migration 10258 if r.is_err() { 10259 print_and_panic( 10260 src_child, 10261 dest_child, 10262 None, 10263 "Error occurred during live-migration", 10264 ); 10265 } 10266 10267 // Check the source vm has been terminated successful (give it '3s' to settle) 10268 thread::sleep(std::time::Duration::new(3, 0)); 10269 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 10270 print_and_panic( 10271 src_child, 10272 dest_child, 10273 None, 10274 "source VM was not terminated successfully.", 10275 ); 10276 }; 10277 10278 // Post live-migration check to make sure the destination VM is funcational 10279 let r = std::panic::catch_unwind(|| { 10280 // Perform same checks to validate VM has been properly migrated 10281 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10282 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10283 }); 10284 10285 // Check Landlock is enabled on destination VM by hot-plugging a disk. 10286 assert!(!remote_command( 10287 &dest_api_socket, 10288 "add-disk", 10289 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 10290 )); 10291 10292 // Clean-up the destination VM and make sure it terminated correctly 10293 let _ = dest_child.kill(); 10294 let dest_output = dest_child.wait_with_output().unwrap(); 10295 handle_child_output(r, &dest_output); 10296 } 10297 10298 mod live_migration_parallel { 10299 use super::*; 10300 #[test] 10301 fn test_live_migration_basic() { 10302 _test_live_migration(false, false) 10303 } 10304 10305 #[test] 10306 fn test_live_migration_local() { 10307 _test_live_migration(false, true) 10308 } 10309 10310 #[test] 10311 fn test_live_migration_watchdog() { 10312 _test_live_migration_watchdog(false, false) 10313 } 10314 10315 #[test] 10316 fn test_live_migration_watchdog_local() { 10317 _test_live_migration_watchdog(false, true) 10318 } 10319 10320 #[test] 10321 fn test_live_upgrade_basic() { 10322 _test_live_migration(true, false) 10323 } 10324 10325 #[test] 10326 fn test_live_upgrade_local() { 10327 _test_live_migration(true, true) 10328 } 10329 10330 #[test] 10331 fn test_live_upgrade_watchdog() { 10332 _test_live_migration_watchdog(true, false) 10333 } 10334 10335 #[test] 10336 fn test_live_upgrade_watchdog_local() { 10337 _test_live_migration_watchdog(true, true) 10338 } 10339 #[test] 10340 #[cfg(target_arch = "x86_64")] 10341 fn test_live_migration_with_landlock() { 10342 _test_live_migration_with_landlock() 10343 } 10344 } 10345 10346 mod live_migration_sequential { 10347 use super::*; 10348 10349 // NUMA & balloon live migration tests are large so run sequentially 10350 10351 #[test] 10352 fn test_live_migration_balloon() { 10353 _test_live_migration_balloon(false, false) 10354 } 10355 10356 #[test] 10357 fn test_live_migration_balloon_local() { 10358 _test_live_migration_balloon(false, true) 10359 } 10360 10361 #[test] 10362 fn test_live_upgrade_balloon() { 10363 _test_live_migration_balloon(true, false) 10364 } 10365 10366 #[test] 10367 fn test_live_upgrade_balloon_local() { 10368 _test_live_migration_balloon(true, true) 10369 } 10370 10371 #[test] 10372 #[cfg(not(feature = "mshv"))] 10373 fn test_live_migration_numa() { 10374 _test_live_migration_numa(false, false) 10375 } 10376 10377 #[test] 10378 #[cfg(not(feature = "mshv"))] 10379 fn test_live_migration_numa_local() { 10380 _test_live_migration_numa(false, true) 10381 } 10382 10383 #[test] 10384 #[cfg(not(feature = "mshv"))] 10385 fn test_live_upgrade_numa() { 10386 _test_live_migration_numa(true, false) 10387 } 10388 10389 #[test] 10390 #[cfg(not(feature = "mshv"))] 10391 fn test_live_upgrade_numa_local() { 10392 _test_live_migration_numa(true, true) 10393 } 10394 10395 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 10396 #[test] 10397 #[ignore = "See #5532"] 10398 #[cfg(target_arch = "x86_64")] 10399 #[cfg(not(feature = "mshv"))] 10400 fn test_live_migration_ovs_dpdk() { 10401 _test_live_migration_ovs_dpdk(false, false); 10402 } 10403 10404 #[test] 10405 #[cfg(target_arch = "x86_64")] 10406 #[cfg(not(feature = "mshv"))] 10407 fn test_live_migration_ovs_dpdk_local() { 10408 _test_live_migration_ovs_dpdk(false, true); 10409 } 10410 10411 #[test] 10412 #[ignore = "See #5532"] 10413 #[cfg(target_arch = "x86_64")] 10414 #[cfg(not(feature = "mshv"))] 10415 fn test_live_upgrade_ovs_dpdk() { 10416 _test_live_migration_ovs_dpdk(true, false); 10417 } 10418 10419 #[test] 10420 #[ignore = "See #5532"] 10421 #[cfg(target_arch = "x86_64")] 10422 #[cfg(not(feature = "mshv"))] 10423 fn test_live_upgrade_ovs_dpdk_local() { 10424 _test_live_migration_ovs_dpdk(true, true); 10425 } 10426 } 10427 } 10428 10429 #[cfg(target_arch = "aarch64")] 10430 mod aarch64_acpi { 10431 use crate::*; 10432 10433 #[test] 10434 fn test_simple_launch_acpi() { 10435 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10436 10437 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 10438 let guest = Guest::new(disk_config); 10439 10440 let mut child = GuestCommand::new(&guest) 10441 .args(["--cpus", "boot=1"]) 10442 .args(["--memory", "size=512M"]) 10443 .args(["--kernel", edk2_path().to_str().unwrap()]) 10444 .default_disks() 10445 .default_net() 10446 .args(["--serial", "tty", "--console", "off"]) 10447 .capture_output() 10448 .spawn() 10449 .unwrap(); 10450 10451 let r = std::panic::catch_unwind(|| { 10452 guest.wait_vm_boot(Some(120)).unwrap(); 10453 10454 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 10455 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 10456 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 10457 }); 10458 10459 let _ = child.kill(); 10460 let output = child.wait_with_output().unwrap(); 10461 10462 handle_child_output(r, &output); 10463 }); 10464 } 10465 10466 #[test] 10467 fn test_guest_numa_nodes_acpi() { 10468 _test_guest_numa_nodes(true); 10469 } 10470 10471 #[test] 10472 fn test_cpu_topology_421_acpi() { 10473 test_cpu_topology(4, 2, 1, true); 10474 } 10475 10476 #[test] 10477 fn test_cpu_topology_142_acpi() { 10478 test_cpu_topology(1, 4, 2, true); 10479 } 10480 10481 #[test] 10482 fn test_cpu_topology_262_acpi() { 10483 test_cpu_topology(2, 6, 2, true); 10484 } 10485 10486 #[test] 10487 fn test_power_button_acpi() { 10488 _test_power_button(true); 10489 } 10490 10491 #[test] 10492 fn test_virtio_iommu() { 10493 _test_virtio_iommu(true) 10494 } 10495 } 10496 10497 mod rate_limiter { 10498 use super::*; 10499 10500 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 10501 // compared to given 'limit' rate. 10502 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 10503 let upper_limit = limit * (1_f64 + difference); 10504 let lower_limit = limit * (1_f64 - difference); 10505 10506 if measured > lower_limit && measured < upper_limit { 10507 return true; 10508 } 10509 10510 eprintln!( 10511 "\n\n==== Start 'check_rate_limit' failed ==== \ 10512 \n\nmeasured={measured}, , lower_limit={lower_limit}, upper_limit={upper_limit} \ 10513 \n\n==== End 'check_rate_limit' failed ====\n\n" 10514 ); 10515 10516 false 10517 } 10518 10519 fn _test_rate_limiter_net(rx: bool) { 10520 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10521 let guest = Guest::new(Box::new(focal)); 10522 10523 let test_timeout = 10; 10524 let num_queues = 2; 10525 let queue_size = 256; 10526 let bw_size = 10485760_u64; // bytes 10527 let bw_refill_time = 100; // ms 10528 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 10529 10530 let net_params = format!( 10531 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 10532 guest.network.guest_mac, 10533 guest.network.host_ip, 10534 num_queues, 10535 queue_size, 10536 bw_size, 10537 bw_refill_time, 10538 ); 10539 10540 let mut child = GuestCommand::new(&guest) 10541 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 10542 .args(["--memory", "size=4G"]) 10543 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10544 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10545 .default_disks() 10546 .args(["--net", net_params.as_str()]) 10547 .capture_output() 10548 .spawn() 10549 .unwrap(); 10550 10551 let r = std::panic::catch_unwind(|| { 10552 guest.wait_vm_boot(None).unwrap(); 10553 let measured_bps = 10554 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, true) 10555 .unwrap(); 10556 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 10557 }); 10558 10559 let _ = child.kill(); 10560 let output = child.wait_with_output().unwrap(); 10561 handle_child_output(r, &output); 10562 } 10563 10564 #[test] 10565 fn test_rate_limiter_net_rx() { 10566 _test_rate_limiter_net(true); 10567 } 10568 10569 #[test] 10570 fn test_rate_limiter_net_tx() { 10571 _test_rate_limiter_net(false); 10572 } 10573 10574 fn _test_rate_limiter_block(bandwidth: bool, num_queues: u32) { 10575 let test_timeout = 10; 10576 let fio_ops = FioOps::RandRW; 10577 10578 let bw_size = if bandwidth { 10579 10485760_u64 // bytes 10580 } else { 10581 100_u64 // I/O 10582 }; 10583 let bw_refill_time = 100; // ms 10584 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10585 10586 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10587 let guest = Guest::new(Box::new(focal)); 10588 let api_socket = temp_api_path(&guest.tmp_dir); 10589 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10590 let blk_rate_limiter_test_img = 10591 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 10592 10593 // Create the test block image 10594 assert!(exec_host_command_output(&format!( 10595 "dd if=/dev/zero of={blk_rate_limiter_test_img} bs=1M count=1024" 10596 )) 10597 .status 10598 .success()); 10599 10600 let test_blk_params = if bandwidth { 10601 format!( 10602 "path={blk_rate_limiter_test_img},num_queues={num_queues},bw_size={bw_size},bw_refill_time={bw_refill_time}" 10603 ) 10604 } else { 10605 format!( 10606 "path={blk_rate_limiter_test_img},num_queues={num_queues},ops_size={bw_size},ops_refill_time={bw_refill_time}" 10607 ) 10608 }; 10609 10610 let mut child = GuestCommand::new(&guest) 10611 .args(["--cpus", &format!("boot={num_queues}")]) 10612 .args(["--memory", "size=4G"]) 10613 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10614 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10615 .args([ 10616 "--disk", 10617 format!( 10618 "path={}", 10619 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10620 ) 10621 .as_str(), 10622 format!( 10623 "path={}", 10624 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10625 ) 10626 .as_str(), 10627 test_blk_params.as_str(), 10628 ]) 10629 .default_net() 10630 .args(["--api-socket", &api_socket]) 10631 .capture_output() 10632 .spawn() 10633 .unwrap(); 10634 10635 let r = std::panic::catch_unwind(|| { 10636 guest.wait_vm_boot(None).unwrap(); 10637 10638 let fio_command = format!( 10639 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 10640 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10641 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10642 ); 10643 let output = guest.ssh_command(&fio_command).unwrap(); 10644 10645 // Parse fio output 10646 let measured_rate = if bandwidth { 10647 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 10648 } else { 10649 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 10650 }; 10651 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10652 }); 10653 10654 let _ = child.kill(); 10655 let output = child.wait_with_output().unwrap(); 10656 handle_child_output(r, &output); 10657 } 10658 10659 fn _test_rate_limiter_group_block(bandwidth: bool, num_queues: u32, num_disks: u32) { 10660 let test_timeout = 10; 10661 let fio_ops = FioOps::RandRW; 10662 10663 let bw_size = if bandwidth { 10664 10485760_u64 // bytes 10665 } else { 10666 100_u64 // I/O 10667 }; 10668 let bw_refill_time = 100; // ms 10669 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10670 10671 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10672 let guest = Guest::new(Box::new(focal)); 10673 let api_socket = temp_api_path(&guest.tmp_dir); 10674 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10675 10676 let rate_limit_group_arg = if bandwidth { 10677 format!("id=group0,bw_size={bw_size},bw_refill_time={bw_refill_time}") 10678 } else { 10679 format!("id=group0,ops_size={bw_size},ops_refill_time={bw_refill_time}") 10680 }; 10681 10682 let mut disk_args = vec![ 10683 "--disk".to_string(), 10684 format!( 10685 "path={}", 10686 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10687 ), 10688 format!( 10689 "path={}", 10690 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10691 ), 10692 ]; 10693 10694 for i in 0..num_disks { 10695 let test_img_path = String::from( 10696 test_img_dir 10697 .as_path() 10698 .join(format!("blk{}.img", i)) 10699 .to_str() 10700 .unwrap(), 10701 ); 10702 10703 assert!(exec_host_command_output(&format!( 10704 "dd if=/dev/zero of={test_img_path} bs=1M count=1024" 10705 )) 10706 .status 10707 .success()); 10708 10709 disk_args.push(format!( 10710 "path={test_img_path},num_queues={num_queues},rate_limit_group=group0" 10711 )); 10712 } 10713 10714 let mut child = GuestCommand::new(&guest) 10715 .args(["--cpus", &format!("boot={}", num_queues * num_disks)]) 10716 .args(["--memory", "size=4G"]) 10717 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10718 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10719 .args(["--rate-limit-group", &rate_limit_group_arg]) 10720 .args(disk_args) 10721 .default_net() 10722 .args(["--api-socket", &api_socket]) 10723 .capture_output() 10724 .spawn() 10725 .unwrap(); 10726 10727 let r = std::panic::catch_unwind(|| { 10728 guest.wait_vm_boot(None).unwrap(); 10729 10730 let mut fio_command = format!( 10731 "sudo fio --name=global --output-format=json \ 10732 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10733 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10734 ); 10735 10736 // Generate additional argument for each disk: 10737 // --name=job0 --filename=/dev/vdc \ 10738 // --name=job1 --filename=/dev/vdd \ 10739 // --name=job2 --filename=/dev/vde \ 10740 // ... 10741 for i in 0..num_disks { 10742 let c: char = 'c'; 10743 let arg = format!( 10744 " --name=job{i} --filename=/dev/vd{}", 10745 char::from_u32((c as u32) + i).unwrap() 10746 ); 10747 fio_command += &arg; 10748 } 10749 let output = guest.ssh_command(&fio_command).unwrap(); 10750 10751 // Parse fio output 10752 let measured_rate = if bandwidth { 10753 parse_fio_output(&output, &fio_ops, num_queues * num_disks).unwrap() 10754 } else { 10755 parse_fio_output_iops(&output, &fio_ops, num_queues * num_disks).unwrap() 10756 }; 10757 assert!(check_rate_limit(measured_rate, limit_rate, 0.2)); 10758 }); 10759 10760 let _ = child.kill(); 10761 let output = child.wait_with_output().unwrap(); 10762 handle_child_output(r, &output); 10763 } 10764 10765 #[test] 10766 fn test_rate_limiter_block_bandwidth() { 10767 _test_rate_limiter_block(true, 1); 10768 _test_rate_limiter_block(true, 2) 10769 } 10770 10771 #[test] 10772 fn test_rate_limiter_group_block_bandwidth() { 10773 _test_rate_limiter_group_block(true, 1, 1); 10774 _test_rate_limiter_group_block(true, 2, 1); 10775 _test_rate_limiter_group_block(true, 1, 2); 10776 _test_rate_limiter_group_block(true, 2, 2); 10777 } 10778 10779 #[test] 10780 fn test_rate_limiter_block_iops() { 10781 _test_rate_limiter_block(false, 1); 10782 _test_rate_limiter_block(false, 2); 10783 } 10784 10785 #[test] 10786 fn test_rate_limiter_group_block_iops() { 10787 _test_rate_limiter_group_block(false, 1, 1); 10788 _test_rate_limiter_group_block(false, 2, 1); 10789 _test_rate_limiter_group_block(false, 1, 2); 10790 _test_rate_limiter_group_block(false, 2, 2); 10791 } 10792 } 10793