1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use std::collections::HashMap; 14 use std::io::{BufRead, Read, Seek, Write}; 15 use std::os::unix::io::AsRawFd; 16 use std::path::PathBuf; 17 use std::process::{Child, Command, Stdio}; 18 use std::string::String; 19 use std::sync::mpsc::Receiver; 20 use std::sync::{mpsc, Mutex}; 21 use std::{fs, io, thread}; 22 23 use net_util::MacAddr; 24 use test_infra::*; 25 use vmm_sys_util::tempdir::TempDir; 26 use vmm_sys_util::tempfile::TempFile; 27 use wait_timeout::ChildExt; 28 29 // Constant taken from the VMM crate. 30 const MAX_NUM_PCI_SEGMENTS: u16 = 96; 31 32 #[cfg(target_arch = "x86_64")] 33 mod x86_64 { 34 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 35 pub const JAMMY_VFIO_IMAGE_NAME: &str = 36 "jammy-server-cloudimg-amd64-custom-vfio-20241012-0.raw"; 37 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 38 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 39 "focal-server-cloudimg-amd64-custom-20210609-0-backing.qcow2"; 40 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 41 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 42 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20230119-0.raw"; 43 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2022-amd64-2.raw"; 44 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 45 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 46 } 47 48 #[cfg(target_arch = "x86_64")] 49 use x86_64::*; 50 51 #[cfg(target_arch = "aarch64")] 52 mod aarch64 { 53 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 54 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 55 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 56 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 57 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 58 "focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2"; 59 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 60 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 61 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 62 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 63 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 64 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 65 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 66 } 67 68 #[cfg(target_arch = "aarch64")] 69 use aarch64::*; 70 71 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 72 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 73 74 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 75 76 // This enum exists to make it more convenient to 77 // implement test for both D-Bus and REST APIs. 78 enum TargetApi { 79 // API socket 80 HttpApi(String), 81 // well known service name, object path 82 DBusApi(String, String), 83 } 84 85 impl TargetApi { 86 fn new_http_api(tmp_dir: &TempDir) -> Self { 87 Self::HttpApi(temp_api_path(tmp_dir)) 88 } 89 90 fn new_dbus_api(tmp_dir: &TempDir) -> Self { 91 // `tmp_dir` is in the form of "/tmp/chXXXXXX" 92 // and we take the `chXXXXXX` part as a unique identifier for the guest 93 let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); 94 95 Self::DBusApi( 96 format!("org.cloudhypervisor.{id}"), 97 format!("/org/cloudhypervisor/{id}"), 98 ) 99 } 100 101 fn guest_args(&self) -> Vec<String> { 102 match self { 103 TargetApi::HttpApi(api_socket) => { 104 vec![format!("--api-socket={}", api_socket.as_str())] 105 } 106 TargetApi::DBusApi(service_name, object_path) => { 107 vec![ 108 format!("--dbus-service-name={}", service_name.as_str()), 109 format!("--dbus-object-path={}", object_path.as_str()), 110 ] 111 } 112 } 113 } 114 115 fn remote_args(&self) -> Vec<String> { 116 // `guest_args` and `remote_args` are consistent with each other 117 self.guest_args() 118 } 119 120 fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { 121 let mut cmd = Command::new(clh_command("ch-remote")); 122 cmd.args(self.remote_args()); 123 cmd.arg(command); 124 125 if let Some(arg) = arg { 126 cmd.arg(arg); 127 } 128 129 let output = cmd.output().unwrap(); 130 if output.status.success() { 131 true 132 } else { 133 eprintln!("Error running ch-remote command: {:?}", &cmd); 134 let stderr = String::from_utf8_lossy(&output.stderr); 135 eprintln!("stderr: {stderr}"); 136 false 137 } 138 } 139 } 140 141 // Start cloud-hypervisor with no VM parameters, only the API server running. 142 // From the API: Create a VM, boot it and check that it looks as expected. 143 fn _test_api_create_boot(target_api: TargetApi, guest: Guest) { 144 let mut child = GuestCommand::new(&guest) 145 .args(target_api.guest_args()) 146 .capture_output() 147 .spawn() 148 .unwrap(); 149 150 thread::sleep(std::time::Duration::new(1, 0)); 151 152 // Verify API server is running 153 assert!(target_api.remote_command("ping", None)); 154 155 // Create the VM first 156 let cpu_count: u8 = 4; 157 let request_body = guest.api_create_body( 158 cpu_count, 159 direct_kernel_boot_path().to_str().unwrap(), 160 DIRECT_KERNEL_BOOT_CMDLINE, 161 ); 162 163 let temp_config_path = guest.tmp_dir.as_path().join("config"); 164 std::fs::write(&temp_config_path, request_body).unwrap(); 165 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 166 167 assert!(target_api.remote_command("create", Some(create_config),)); 168 169 // Then boot it 170 assert!(target_api.remote_command("boot", None)); 171 thread::sleep(std::time::Duration::new(20, 0)); 172 173 let r = std::panic::catch_unwind(|| { 174 // Check that the VM booted as expected 175 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 176 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 177 }); 178 179 kill_child(&mut child); 180 let output = child.wait_with_output().unwrap(); 181 182 handle_child_output(r, &output); 183 } 184 185 // Start cloud-hypervisor with no VM parameters, only the API server running. 186 // From the API: Create a VM, boot it and check it can be shutdown and then 187 // booted again 188 fn _test_api_shutdown(target_api: TargetApi, guest: Guest) { 189 let mut child = GuestCommand::new(&guest) 190 .args(target_api.guest_args()) 191 .capture_output() 192 .spawn() 193 .unwrap(); 194 195 thread::sleep(std::time::Duration::new(1, 0)); 196 197 // Verify API server is running 198 assert!(target_api.remote_command("ping", None)); 199 200 // Create the VM first 201 let cpu_count: u8 = 4; 202 let request_body = guest.api_create_body( 203 cpu_count, 204 direct_kernel_boot_path().to_str().unwrap(), 205 DIRECT_KERNEL_BOOT_CMDLINE, 206 ); 207 208 let temp_config_path = guest.tmp_dir.as_path().join("config"); 209 std::fs::write(&temp_config_path, request_body).unwrap(); 210 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 211 212 let r = std::panic::catch_unwind(|| { 213 assert!(target_api.remote_command("create", Some(create_config))); 214 215 // Then boot it 216 assert!(target_api.remote_command("boot", None)); 217 218 guest.wait_vm_boot(None).unwrap(); 219 220 // Check that the VM booted as expected 221 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 222 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 223 224 // Sync and shutdown without powering off to prevent filesystem 225 // corruption. 226 guest.ssh_command("sync").unwrap(); 227 guest.ssh_command("sudo shutdown -H now").unwrap(); 228 229 // Wait for the guest to be fully shutdown 230 thread::sleep(std::time::Duration::new(20, 0)); 231 232 // Then shut it down 233 assert!(target_api.remote_command("shutdown", None)); 234 235 // Then boot it again 236 assert!(target_api.remote_command("boot", None)); 237 238 guest.wait_vm_boot(None).unwrap(); 239 240 // Check that the VM booted as expected 241 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 242 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 243 }); 244 245 kill_child(&mut child); 246 let output = child.wait_with_output().unwrap(); 247 248 handle_child_output(r, &output); 249 } 250 251 // Start cloud-hypervisor with no VM parameters, only the API server running. 252 // From the API: Create a VM, boot it and check it can be deleted and then recreated 253 // booted again. 254 fn _test_api_delete(target_api: TargetApi, guest: Guest) { 255 let mut child = GuestCommand::new(&guest) 256 .args(target_api.guest_args()) 257 .capture_output() 258 .spawn() 259 .unwrap(); 260 261 thread::sleep(std::time::Duration::new(1, 0)); 262 263 // Verify API server is running 264 assert!(target_api.remote_command("ping", None)); 265 266 // Create the VM first 267 let cpu_count: u8 = 4; 268 let request_body = guest.api_create_body( 269 cpu_count, 270 direct_kernel_boot_path().to_str().unwrap(), 271 DIRECT_KERNEL_BOOT_CMDLINE, 272 ); 273 let temp_config_path = guest.tmp_dir.as_path().join("config"); 274 std::fs::write(&temp_config_path, request_body).unwrap(); 275 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 276 277 let r = std::panic::catch_unwind(|| { 278 assert!(target_api.remote_command("create", Some(create_config))); 279 280 // Then boot it 281 assert!(target_api.remote_command("boot", None)); 282 283 guest.wait_vm_boot(None).unwrap(); 284 285 // Check that the VM booted as expected 286 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 287 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 288 289 // Sync and shutdown without powering off to prevent filesystem 290 // corruption. 291 guest.ssh_command("sync").unwrap(); 292 guest.ssh_command("sudo shutdown -H now").unwrap(); 293 294 // Wait for the guest to be fully shutdown 295 thread::sleep(std::time::Duration::new(20, 0)); 296 297 // Then delete it 298 assert!(target_api.remote_command("delete", None)); 299 300 assert!(target_api.remote_command("create", Some(create_config))); 301 302 // Then boot it again 303 assert!(target_api.remote_command("boot", None)); 304 305 guest.wait_vm_boot(None).unwrap(); 306 307 // Check that the VM booted as expected 308 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 309 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 310 }); 311 312 kill_child(&mut child); 313 let output = child.wait_with_output().unwrap(); 314 315 handle_child_output(r, &output); 316 } 317 318 // Start cloud-hypervisor with no VM parameters, only the API server running. 319 // From the API: Create a VM, boot it and check that it looks as expected. 320 // Then we pause the VM, check that it's no longer available. 321 // Finally we resume the VM and check that it's available. 322 fn _test_api_pause_resume(target_api: TargetApi, guest: Guest) { 323 let mut child = GuestCommand::new(&guest) 324 .args(target_api.guest_args()) 325 .capture_output() 326 .spawn() 327 .unwrap(); 328 329 thread::sleep(std::time::Duration::new(1, 0)); 330 331 // Verify API server is running 332 assert!(target_api.remote_command("ping", None)); 333 334 // Create the VM first 335 let cpu_count: u8 = 4; 336 let request_body = guest.api_create_body( 337 cpu_count, 338 direct_kernel_boot_path().to_str().unwrap(), 339 DIRECT_KERNEL_BOOT_CMDLINE, 340 ); 341 342 let temp_config_path = guest.tmp_dir.as_path().join("config"); 343 std::fs::write(&temp_config_path, request_body).unwrap(); 344 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 345 346 assert!(target_api.remote_command("create", Some(create_config))); 347 348 // Then boot it 349 assert!(target_api.remote_command("boot", None)); 350 thread::sleep(std::time::Duration::new(20, 0)); 351 352 let r = std::panic::catch_unwind(|| { 353 // Check that the VM booted as expected 354 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 355 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 356 357 // We now pause the VM 358 assert!(target_api.remote_command("pause", None)); 359 360 // Check pausing again fails 361 assert!(!target_api.remote_command("pause", None)); 362 363 thread::sleep(std::time::Duration::new(2, 0)); 364 365 // SSH into the VM should fail 366 ssh_command_ip( 367 "grep -c processor /proc/cpuinfo", 368 &guest.network.guest_ip, 369 2, 370 5, 371 ) 372 .unwrap_err(); 373 374 // Resume the VM 375 assert!(target_api.remote_command("resume", None)); 376 377 // Check resuming again fails 378 assert!(!target_api.remote_command("resume", None)); 379 380 thread::sleep(std::time::Duration::new(2, 0)); 381 382 // Now we should be able to SSH back in and get the right number of CPUs 383 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 384 }); 385 386 kill_child(&mut child); 387 let output = child.wait_with_output().unwrap(); 388 389 handle_child_output(r, &output); 390 } 391 392 fn _test_pty_interaction(pty_path: PathBuf) { 393 let mut cf = std::fs::OpenOptions::new() 394 .write(true) 395 .read(true) 396 .open(pty_path) 397 .unwrap(); 398 399 // Some dumb sleeps but we don't want to write 400 // before the console is up and we don't want 401 // to try and write the next line before the 402 // login process is ready. 403 thread::sleep(std::time::Duration::new(5, 0)); 404 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 405 thread::sleep(std::time::Duration::new(2, 0)); 406 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 407 thread::sleep(std::time::Duration::new(2, 0)); 408 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 409 thread::sleep(std::time::Duration::new(2, 0)); 410 411 // read pty and ensure they have a login shell 412 // some fairly hacky workarounds to avoid looping 413 // forever in case the channel is blocked getting output 414 let ptyc = pty_read(cf); 415 let mut empty = 0; 416 let mut prev = String::new(); 417 loop { 418 thread::sleep(std::time::Duration::new(2, 0)); 419 match ptyc.try_recv() { 420 Ok(line) => { 421 empty = 0; 422 prev = prev + &line; 423 if prev.contains("test_pty_console") { 424 break; 425 } 426 } 427 Err(mpsc::TryRecvError::Empty) => { 428 empty += 1; 429 assert!(empty <= 5, "No login on pty"); 430 } 431 _ => { 432 panic!("No login on pty") 433 } 434 } 435 } 436 } 437 438 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 439 let mut workload_path = dirs::home_dir().unwrap(); 440 workload_path.push("workloads"); 441 442 let mut virtiofsd_path = workload_path; 443 virtiofsd_path.push("virtiofsd"); 444 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 445 446 let virtiofsd_socket_path = 447 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 448 449 // Start the daemon 450 let child = Command::new(virtiofsd_path.as_str()) 451 .args(["--shared-dir", shared_dir]) 452 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 453 .args(["--cache", "never"]) 454 .spawn() 455 .unwrap(); 456 457 thread::sleep(std::time::Duration::new(10, 0)); 458 459 (child, virtiofsd_socket_path) 460 } 461 462 fn prepare_vubd( 463 tmp_dir: &TempDir, 464 blk_img: &str, 465 num_queues: usize, 466 rdonly: bool, 467 direct: bool, 468 ) -> (std::process::Child, String) { 469 let mut workload_path = dirs::home_dir().unwrap(); 470 workload_path.push("workloads"); 471 472 let mut blk_file_path = workload_path; 473 blk_file_path.push(blk_img); 474 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 475 476 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 477 478 // Start the daemon 479 let child = Command::new(clh_command("vhost_user_block")) 480 .args([ 481 "--block-backend", 482 format!( 483 "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" 484 ) 485 .as_str(), 486 ]) 487 .spawn() 488 .unwrap(); 489 490 thread::sleep(std::time::Duration::new(10, 0)); 491 492 (child, vubd_socket_path) 493 } 494 495 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 496 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 497 } 498 499 fn temp_api_path(tmp_dir: &TempDir) -> String { 500 String::from( 501 tmp_dir 502 .as_path() 503 .join("cloud-hypervisor.sock") 504 .to_str() 505 .unwrap(), 506 ) 507 } 508 509 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 510 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 511 } 512 513 // Creates the directory and returns the path. 514 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 515 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 516 std::fs::create_dir(&snapshot_dir).unwrap(); 517 snapshot_dir 518 } 519 520 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 521 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 522 vmcore_file 523 } 524 525 // Creates the path for direct kernel boot and return the path. 526 // For x86_64, this function returns the vmlinux kernel path. 527 // For AArch64, this function returns the PE kernel path. 528 fn direct_kernel_boot_path() -> PathBuf { 529 let mut workload_path = dirs::home_dir().unwrap(); 530 workload_path.push("workloads"); 531 532 let mut kernel_path = workload_path; 533 #[cfg(target_arch = "x86_64")] 534 kernel_path.push("vmlinux"); 535 #[cfg(target_arch = "aarch64")] 536 kernel_path.push("Image"); 537 538 kernel_path 539 } 540 541 fn edk2_path() -> PathBuf { 542 let mut workload_path = dirs::home_dir().unwrap(); 543 workload_path.push("workloads"); 544 let mut edk2_path = workload_path; 545 edk2_path.push(OVMF_NAME); 546 547 edk2_path 548 } 549 550 fn cloud_hypervisor_release_path() -> String { 551 let mut workload_path = dirs::home_dir().unwrap(); 552 workload_path.push("workloads"); 553 554 let mut ch_release_path = workload_path; 555 #[cfg(target_arch = "x86_64")] 556 ch_release_path.push("cloud-hypervisor-static"); 557 #[cfg(target_arch = "aarch64")] 558 ch_release_path.push("cloud-hypervisor-static-aarch64"); 559 560 ch_release_path.into_os_string().into_string().unwrap() 561 } 562 563 fn prepare_vhost_user_net_daemon( 564 tmp_dir: &TempDir, 565 ip: &str, 566 tap: Option<&str>, 567 mtu: Option<u16>, 568 num_queues: usize, 569 client_mode: bool, 570 ) -> (std::process::Command, String) { 571 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 572 573 // Start the daemon 574 let mut net_params = format!( 575 "ip={ip},mask=255.255.255.0,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" 576 ); 577 578 if let Some(tap) = tap { 579 net_params.push_str(format!(",tap={tap}").as_str()); 580 } 581 582 if let Some(mtu) = mtu { 583 net_params.push_str(format!(",mtu={mtu}").as_str()); 584 } 585 586 let mut command = Command::new(clh_command("vhost_user_net")); 587 command.args(["--net-backend", net_params.as_str()]); 588 589 (command, vunet_socket_path) 590 } 591 592 fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { 593 let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); 594 let swtpm_socket_path = String::from( 595 tmp_dir 596 .as_path() 597 .join("swtpm") 598 .join("swtpm.sock") 599 .to_str() 600 .unwrap(), 601 ); 602 std::fs::create_dir(&swtpm_tpm_dir).unwrap(); 603 604 let mut swtpm_command = Command::new("swtpm"); 605 let swtpm_args = [ 606 "socket", 607 "--tpmstate", 608 &format!("dir={swtpm_tpm_dir}"), 609 "--ctrl", 610 &format!("type=unixio,path={swtpm_socket_path}"), 611 "--flags", 612 "startup-clear", 613 "--tpm2", 614 ]; 615 swtpm_command.args(swtpm_args); 616 617 (swtpm_command, swtpm_socket_path) 618 } 619 620 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 621 let mut cmd = Command::new(clh_command("ch-remote")); 622 cmd.args([&format!("--api-socket={api_socket}"), command]); 623 624 if let Some(arg) = arg { 625 cmd.arg(arg); 626 } 627 let output = cmd.output().unwrap(); 628 if output.status.success() { 629 true 630 } else { 631 eprintln!("Error running ch-remote command: {:?}", &cmd); 632 let stderr = String::from_utf8_lossy(&output.stderr); 633 eprintln!("stderr: {stderr}"); 634 false 635 } 636 } 637 638 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 639 let mut cmd = Command::new(clh_command("ch-remote")); 640 cmd.args([&format!("--api-socket={api_socket}"), command]); 641 642 if let Some(arg) = arg { 643 cmd.arg(arg); 644 } 645 646 let output = cmd.output().expect("Failed to launch ch-remote"); 647 648 (output.status.success(), output.stdout) 649 } 650 651 fn resize_command( 652 api_socket: &str, 653 desired_vcpus: Option<u8>, 654 desired_ram: Option<usize>, 655 desired_balloon: Option<usize>, 656 event_file: Option<&str>, 657 ) -> bool { 658 let mut cmd = Command::new(clh_command("ch-remote")); 659 cmd.args([&format!("--api-socket={api_socket}"), "resize"]); 660 661 if let Some(desired_vcpus) = desired_vcpus { 662 cmd.arg(format!("--cpus={desired_vcpus}")); 663 } 664 665 if let Some(desired_ram) = desired_ram { 666 cmd.arg(format!("--memory={desired_ram}")); 667 } 668 669 if let Some(desired_balloon) = desired_balloon { 670 cmd.arg(format!("--balloon={desired_balloon}")); 671 } 672 673 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 674 675 if let Some(event_path) = event_file { 676 let latest_events = [ 677 &MetaEvent { 678 event: "resizing".to_string(), 679 device_id: None, 680 }, 681 &MetaEvent { 682 event: "resized".to_string(), 683 device_id: None, 684 }, 685 ]; 686 // See: #5938 687 thread::sleep(std::time::Duration::new(1, 0)); 688 assert!(check_latest_events_exact(&latest_events, event_path)); 689 } 690 691 ret 692 } 693 694 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 695 let mut cmd = Command::new(clh_command("ch-remote")); 696 cmd.args([ 697 &format!("--api-socket={api_socket}"), 698 "resize-zone", 699 &format!("--id={id}"), 700 &format!("--size={desired_size}"), 701 ]); 702 703 cmd.status().expect("Failed to launch ch-remote").success() 704 } 705 706 // setup OVS-DPDK bridge and ports 707 fn setup_ovs_dpdk() { 708 // setup OVS-DPDK 709 assert!(exec_host_command_status("service openvswitch-switch start").success()); 710 assert!(exec_host_command_status("ovs-vsctl init").success()); 711 assert!( 712 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 713 .success() 714 ); 715 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 716 717 // Create OVS-DPDK bridge and ports 718 assert!(exec_host_command_status( 719 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 720 ) 721 .success()); 722 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 723 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 724 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 725 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 726 } 727 fn cleanup_ovs_dpdk() { 728 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 729 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 730 } 731 // Setup two guests and ensure they are connected through ovs-dpdk 732 fn setup_ovs_dpdk_guests( 733 guest1: &Guest, 734 guest2: &Guest, 735 api_socket: &str, 736 release_binary: bool, 737 ) -> (Child, Child) { 738 setup_ovs_dpdk(); 739 740 let clh_path = if !release_binary { 741 clh_command("cloud-hypervisor") 742 } else { 743 cloud_hypervisor_release_path() 744 }; 745 746 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 747 .args(["--cpus", "boot=2"]) 748 .args(["--memory", "size=0,shared=on"]) 749 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 750 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 751 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 752 .default_disks() 753 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 754 .capture_output() 755 .spawn() 756 .unwrap(); 757 758 #[cfg(target_arch = "x86_64")] 759 let guest_net_iface = "ens5"; 760 #[cfg(target_arch = "aarch64")] 761 let guest_net_iface = "enp0s5"; 762 763 let r = std::panic::catch_unwind(|| { 764 guest1.wait_vm_boot(None).unwrap(); 765 766 guest1 767 .ssh_command(&format!( 768 "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" 769 )) 770 .unwrap(); 771 guest1 772 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 773 .unwrap(); 774 775 let guest_ip = guest1.network.guest_ip.clone(); 776 thread::spawn(move || { 777 ssh_command_ip( 778 "nc -l 12345", 779 &guest_ip, 780 DEFAULT_SSH_RETRIES, 781 DEFAULT_SSH_TIMEOUT, 782 ) 783 .unwrap(); 784 }); 785 }); 786 if r.is_err() { 787 cleanup_ovs_dpdk(); 788 789 let _ = child1.kill(); 790 let output = child1.wait_with_output().unwrap(); 791 handle_child_output(r, &output); 792 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 793 } 794 795 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 796 .args(["--api-socket", api_socket]) 797 .args(["--cpus", "boot=2"]) 798 .args(["--memory", "size=0,shared=on"]) 799 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 800 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 801 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 802 .default_disks() 803 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 804 .capture_output() 805 .spawn() 806 .unwrap(); 807 808 let r = std::panic::catch_unwind(|| { 809 guest2.wait_vm_boot(None).unwrap(); 810 811 guest2 812 .ssh_command(&format!( 813 "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" 814 )) 815 .unwrap(); 816 guest2 817 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 818 .unwrap(); 819 820 // Check the connection works properly between the two VMs 821 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 822 }); 823 if r.is_err() { 824 cleanup_ovs_dpdk(); 825 826 let _ = child1.kill(); 827 let _ = child2.kill(); 828 let output = child2.wait_with_output().unwrap(); 829 handle_child_output(r, &output); 830 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 831 } 832 833 (child1, child2) 834 } 835 836 enum FwType { 837 Ovmf, 838 RustHypervisorFirmware, 839 } 840 841 fn fw_path(_fw_type: FwType) -> String { 842 let mut workload_path = dirs::home_dir().unwrap(); 843 workload_path.push("workloads"); 844 845 let mut fw_path = workload_path; 846 #[cfg(target_arch = "aarch64")] 847 fw_path.push("CLOUDHV_EFI.fd"); 848 #[cfg(target_arch = "x86_64")] 849 { 850 match _fw_type { 851 FwType::Ovmf => fw_path.push(OVMF_NAME), 852 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 853 } 854 } 855 856 fw_path.to_str().unwrap().to_string() 857 } 858 859 #[derive(Debug)] 860 struct MetaEvent { 861 event: String, 862 device_id: Option<String>, 863 } 864 865 impl MetaEvent { 866 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 867 let mut matched = false; 868 if v["event"].as_str().unwrap() == self.event { 869 if let Some(device_id) = &self.device_id { 870 if v["properties"]["id"].as_str().unwrap() == device_id { 871 matched = true 872 } 873 } else { 874 matched = true; 875 } 876 } 877 matched 878 } 879 } 880 881 // Parse the event_monitor file based on the format that each event 882 // is followed by a double newline 883 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 884 let content = fs::read(event_file).unwrap(); 885 let mut ret = Vec::new(); 886 for entry in String::from_utf8_lossy(&content) 887 .trim() 888 .split("\n\n") 889 .collect::<Vec<&str>>() 890 { 891 ret.push(serde_json::from_str(entry).unwrap()); 892 } 893 894 ret 895 } 896 897 // Return true if all events from the input 'expected_events' are matched sequentially 898 // with events from the 'event_file' 899 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 900 let json_events = parse_event_file(event_file); 901 let len = expected_events.len(); 902 let mut idx = 0; 903 for e in &json_events { 904 if idx == len { 905 break; 906 } 907 if expected_events[idx].match_with_json_event(e) { 908 idx += 1; 909 } 910 } 911 912 let ret = idx == len; 913 914 if !ret { 915 eprintln!( 916 "\n\n==== Start 'check_sequential_events' failed ==== \ 917 \n\nexpected_events={:?}\nactual_events={:?} \ 918 \n\n==== End 'check_sequential_events' failed ====", 919 expected_events, json_events, 920 ); 921 } 922 923 ret 924 } 925 926 // Return true if all events from the input 'expected_events' are matched exactly 927 // with events from the 'event_file' 928 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 929 let json_events = parse_event_file(event_file); 930 assert!(expected_events.len() <= json_events.len()); 931 let json_events = &json_events[..expected_events.len()]; 932 933 for (idx, e) in json_events.iter().enumerate() { 934 if !expected_events[idx].match_with_json_event(e) { 935 eprintln!( 936 "\n\n==== Start 'check_sequential_events_exact' failed ==== \ 937 \n\nexpected_events={:?}\nactual_events={:?} \ 938 \n\n==== End 'check_sequential_events_exact' failed ====", 939 expected_events, json_events, 940 ); 941 942 return false; 943 } 944 } 945 946 true 947 } 948 949 // Return true if events from the input 'latest_events' are matched exactly 950 // with the most recent events from the 'event_file' 951 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 952 let json_events = parse_event_file(event_file); 953 assert!(latest_events.len() <= json_events.len()); 954 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 955 956 for (idx, e) in json_events.iter().enumerate() { 957 if !latest_events[idx].match_with_json_event(e) { 958 eprintln!( 959 "\n\n==== Start 'check_latest_events_exact' failed ==== \ 960 \n\nexpected_events={:?}\nactual_events={:?} \ 961 \n\n==== End 'check_latest_events_exact' failed ====", 962 latest_events, json_events, 963 ); 964 965 return false; 966 } 967 } 968 969 true 970 } 971 972 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 973 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 974 let guest = Guest::new(Box::new(focal)); 975 let total_vcpus = threads_per_core * cores_per_package * packages; 976 let direct_kernel_boot_path = direct_kernel_boot_path(); 977 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 978 let fw_path = fw_path(FwType::RustHypervisorFirmware); 979 if use_fw { 980 kernel_path = fw_path.as_str(); 981 } 982 983 let mut child = GuestCommand::new(&guest) 984 .args([ 985 "--cpus", 986 &format!( 987 "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" 988 ), 989 ]) 990 .args(["--memory", "size=512M"]) 991 .args(["--kernel", kernel_path]) 992 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 993 .default_disks() 994 .default_net() 995 .capture_output() 996 .spawn() 997 .unwrap(); 998 999 let r = std::panic::catch_unwind(|| { 1000 guest.wait_vm_boot(None).unwrap(); 1001 assert_eq!( 1002 guest.get_cpu_count().unwrap_or_default(), 1003 u32::from(total_vcpus) 1004 ); 1005 assert_eq!( 1006 guest 1007 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1008 .unwrap() 1009 .trim() 1010 .parse::<u8>() 1011 .unwrap_or(0), 1012 threads_per_core 1013 ); 1014 1015 assert_eq!( 1016 guest 1017 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1018 .unwrap() 1019 .trim() 1020 .parse::<u8>() 1021 .unwrap_or(0), 1022 cores_per_package 1023 ); 1024 1025 assert_eq!( 1026 guest 1027 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1028 .unwrap() 1029 .trim() 1030 .parse::<u8>() 1031 .unwrap_or(0), 1032 packages 1033 ); 1034 1035 #[cfg(target_arch = "x86_64")] 1036 { 1037 let mut cpu_id = 0; 1038 for package_id in 0..packages { 1039 for core_id in 0..cores_per_package { 1040 for _ in 0..threads_per_core { 1041 assert_eq!( 1042 guest 1043 .ssh_command(&format!("cat /sys/devices/system/cpu/cpu{cpu_id}/topology/physical_package_id")) 1044 .unwrap() 1045 .trim() 1046 .parse::<u8>() 1047 .unwrap_or(0), 1048 package_id 1049 ); 1050 1051 assert_eq!( 1052 guest 1053 .ssh_command(&format!( 1054 "cat /sys/devices/system/cpu/cpu{cpu_id}/topology/core_id" 1055 )) 1056 .unwrap() 1057 .trim() 1058 .parse::<u8>() 1059 .unwrap_or(0), 1060 core_id 1061 ); 1062 1063 cpu_id += 1; 1064 } 1065 } 1066 } 1067 } 1068 }); 1069 1070 kill_child(&mut child); 1071 let output = child.wait_with_output().unwrap(); 1072 1073 handle_child_output(r, &output); 1074 } 1075 1076 #[allow(unused_variables)] 1077 fn _test_guest_numa_nodes(acpi: bool) { 1078 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1079 let guest = Guest::new(Box::new(focal)); 1080 let api_socket = temp_api_path(&guest.tmp_dir); 1081 #[cfg(target_arch = "x86_64")] 1082 let kernel_path = direct_kernel_boot_path(); 1083 #[cfg(target_arch = "aarch64")] 1084 let kernel_path = if acpi { 1085 edk2_path() 1086 } else { 1087 direct_kernel_boot_path() 1088 }; 1089 1090 let mut child = GuestCommand::new(&guest) 1091 .args(["--cpus", "boot=6,max=12"]) 1092 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 1093 .args([ 1094 "--memory-zone", 1095 "id=mem0,size=1G,hotplug_size=3G", 1096 "id=mem1,size=2G,hotplug_size=3G", 1097 "id=mem2,size=3G,hotplug_size=3G", 1098 ]) 1099 .args([ 1100 "--numa", 1101 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 1102 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 1103 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 1104 ]) 1105 .args(["--kernel", kernel_path.to_str().unwrap()]) 1106 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1107 .args(["--api-socket", &api_socket]) 1108 .capture_output() 1109 .default_disks() 1110 .default_net() 1111 .spawn() 1112 .unwrap(); 1113 1114 let r = std::panic::catch_unwind(|| { 1115 guest.wait_vm_boot(None).unwrap(); 1116 1117 guest.check_numa_common( 1118 Some(&[960_000, 1_920_000, 2_880_000]), 1119 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 1120 Some(&["10 15 20", "20 10 25", "25 30 10"]), 1121 ); 1122 1123 // AArch64 currently does not support hotplug, and therefore we only 1124 // test hotplug-related function on x86_64 here. 1125 #[cfg(target_arch = "x86_64")] 1126 { 1127 guest.enable_memory_hotplug(); 1128 1129 // Resize every memory zone and check each associated NUMA node 1130 // has been assigned the right amount of memory. 1131 resize_zone_command(&api_socket, "mem0", "4G"); 1132 resize_zone_command(&api_socket, "mem1", "4G"); 1133 resize_zone_command(&api_socket, "mem2", "4G"); 1134 // Resize to the maximum amount of CPUs and check each NUMA 1135 // node has been assigned the right CPUs set. 1136 resize_command(&api_socket, Some(12), None, None, None); 1137 thread::sleep(std::time::Duration::new(5, 0)); 1138 1139 guest.check_numa_common( 1140 Some(&[3_840_000, 3_840_000, 3_840_000]), 1141 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 1142 None, 1143 ); 1144 } 1145 }); 1146 1147 kill_child(&mut child); 1148 let output = child.wait_with_output().unwrap(); 1149 1150 handle_child_output(r, &output); 1151 } 1152 1153 #[allow(unused_variables)] 1154 fn _test_power_button(acpi: bool) { 1155 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1156 let guest = Guest::new(Box::new(focal)); 1157 let mut cmd = GuestCommand::new(&guest); 1158 let api_socket = temp_api_path(&guest.tmp_dir); 1159 1160 #[cfg(target_arch = "x86_64")] 1161 let kernel_path = direct_kernel_boot_path(); 1162 #[cfg(target_arch = "aarch64")] 1163 let kernel_path = if acpi { 1164 edk2_path() 1165 } else { 1166 direct_kernel_boot_path() 1167 }; 1168 1169 cmd.args(["--cpus", "boot=1"]) 1170 .args(["--memory", "size=512M"]) 1171 .args(["--kernel", kernel_path.to_str().unwrap()]) 1172 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1173 .capture_output() 1174 .default_disks() 1175 .default_net() 1176 .args(["--api-socket", &api_socket]); 1177 1178 let child = cmd.spawn().unwrap(); 1179 1180 let r = std::panic::catch_unwind(|| { 1181 guest.wait_vm_boot(None).unwrap(); 1182 assert!(remote_command(&api_socket, "power-button", None)); 1183 }); 1184 1185 let output = child.wait_with_output().unwrap(); 1186 assert!(output.status.success()); 1187 handle_child_output(r, &output); 1188 } 1189 1190 type PrepareNetDaemon = dyn Fn( 1191 &TempDir, 1192 &str, 1193 Option<&str>, 1194 Option<u16>, 1195 usize, 1196 bool, 1197 ) -> (std::process::Command, String); 1198 1199 fn test_vhost_user_net( 1200 tap: Option<&str>, 1201 num_queues: usize, 1202 prepare_daemon: &PrepareNetDaemon, 1203 generate_host_mac: bool, 1204 client_mode_daemon: bool, 1205 ) { 1206 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1207 let guest = Guest::new(Box::new(focal)); 1208 let api_socket = temp_api_path(&guest.tmp_dir); 1209 1210 let kernel_path = direct_kernel_boot_path(); 1211 1212 let host_mac = if generate_host_mac { 1213 Some(MacAddr::local_random()) 1214 } else { 1215 None 1216 }; 1217 1218 let mtu = Some(3000); 1219 1220 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 1221 &guest.tmp_dir, 1222 &guest.network.host_ip, 1223 tap, 1224 mtu, 1225 num_queues, 1226 client_mode_daemon, 1227 ); 1228 1229 let net_params = format!( 1230 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 1231 guest.network.guest_mac, 1232 vunet_socket_path, 1233 num_queues, 1234 if let Some(host_mac) = host_mac { 1235 format!(",host_mac={host_mac}") 1236 } else { 1237 "".to_owned() 1238 }, 1239 if client_mode_daemon { 1240 "server" 1241 } else { 1242 "client" 1243 }, 1244 ); 1245 1246 let mut ch_command = GuestCommand::new(&guest); 1247 ch_command 1248 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 1249 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1250 .args(["--kernel", kernel_path.to_str().unwrap()]) 1251 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1252 .default_disks() 1253 .args(["--net", net_params.as_str()]) 1254 .args(["--api-socket", &api_socket]) 1255 .capture_output(); 1256 1257 let mut daemon_child: std::process::Child; 1258 let mut child: std::process::Child; 1259 1260 if client_mode_daemon { 1261 child = ch_command.spawn().unwrap(); 1262 // Make sure the VMM is waiting for the backend to connect 1263 thread::sleep(std::time::Duration::new(10, 0)); 1264 daemon_child = daemon_command.spawn().unwrap(); 1265 } else { 1266 daemon_child = daemon_command.spawn().unwrap(); 1267 // Make sure the backend is waiting for the VMM to connect 1268 thread::sleep(std::time::Duration::new(10, 0)); 1269 child = ch_command.spawn().unwrap(); 1270 } 1271 1272 let r = std::panic::catch_unwind(|| { 1273 guest.wait_vm_boot(None).unwrap(); 1274 1275 if let Some(tap_name) = tap { 1276 let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); 1277 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1278 } 1279 1280 if let Some(host_mac) = tap { 1281 let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); 1282 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1283 } 1284 1285 #[cfg(target_arch = "aarch64")] 1286 let iface = "enp0s4"; 1287 #[cfg(target_arch = "x86_64")] 1288 let iface = "ens4"; 1289 1290 assert_eq!( 1291 guest 1292 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 1293 .unwrap() 1294 .trim(), 1295 "3000" 1296 ); 1297 1298 // 1 network interface + default localhost ==> 2 interfaces 1299 // It's important to note that this test is fully exercising the 1300 // vhost-user-net implementation and the associated backend since 1301 // it does not define any --net network interface. That means all 1302 // the ssh communication in that test happens through the network 1303 // interface backed by vhost-user-net. 1304 assert_eq!( 1305 guest 1306 .ssh_command("ip -o link | wc -l") 1307 .unwrap() 1308 .trim() 1309 .parse::<u32>() 1310 .unwrap_or_default(), 1311 2 1312 ); 1313 1314 // The following pci devices will appear on guest with PCI-MSI 1315 // interrupt vectors assigned. 1316 // 1 virtio-console with 3 vectors: config, Rx, Tx 1317 // 1 virtio-blk with 2 vectors: config, Request 1318 // 1 virtio-blk with 2 vectors: config, Request 1319 // 1 virtio-rng with 2 vectors: config, Request 1320 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1321 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1322 // Based on the above, the total vectors should 14. 1323 #[cfg(target_arch = "x86_64")] 1324 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1325 #[cfg(target_arch = "aarch64")] 1326 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1327 assert_eq!( 1328 guest 1329 .ssh_command(grep_cmd) 1330 .unwrap() 1331 .trim() 1332 .parse::<u32>() 1333 .unwrap_or_default(), 1334 10 + (num_queues as u32) 1335 ); 1336 1337 // ACPI feature is needed. 1338 #[cfg(target_arch = "x86_64")] 1339 { 1340 guest.enable_memory_hotplug(); 1341 1342 // Add RAM to the VM 1343 let desired_ram = 1024 << 20; 1344 resize_command(&api_socket, None, Some(desired_ram), None, None); 1345 1346 thread::sleep(std::time::Duration::new(10, 0)); 1347 1348 // Here by simply checking the size (through ssh), we validate 1349 // the connection is still working, which means vhost-user-net 1350 // keeps working after the resize. 1351 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1352 } 1353 }); 1354 1355 kill_child(&mut child); 1356 let output = child.wait_with_output().unwrap(); 1357 1358 thread::sleep(std::time::Duration::new(5, 0)); 1359 let _ = daemon_child.kill(); 1360 let _ = daemon_child.wait(); 1361 1362 handle_child_output(r, &output); 1363 } 1364 1365 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1366 1367 fn test_vhost_user_blk( 1368 num_queues: usize, 1369 readonly: bool, 1370 direct: bool, 1371 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1372 ) { 1373 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1374 let guest = Guest::new(Box::new(focal)); 1375 let api_socket = temp_api_path(&guest.tmp_dir); 1376 1377 let kernel_path = direct_kernel_boot_path(); 1378 1379 let (blk_params, daemon_child) = { 1380 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1381 // Start the daemon 1382 let (daemon_child, vubd_socket_path) = 1383 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1384 1385 ( 1386 format!( 1387 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1388 ), 1389 Some(daemon_child), 1390 ) 1391 }; 1392 1393 let mut child = GuestCommand::new(&guest) 1394 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1395 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1396 .args(["--kernel", kernel_path.to_str().unwrap()]) 1397 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1398 .args([ 1399 "--disk", 1400 format!( 1401 "path={}", 1402 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1403 ) 1404 .as_str(), 1405 format!( 1406 "path={}", 1407 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1408 ) 1409 .as_str(), 1410 blk_params.as_str(), 1411 ]) 1412 .default_net() 1413 .args(["--api-socket", &api_socket]) 1414 .capture_output() 1415 .spawn() 1416 .unwrap(); 1417 1418 let r = std::panic::catch_unwind(|| { 1419 guest.wait_vm_boot(None).unwrap(); 1420 1421 // Check both if /dev/vdc exists and if the block size is 16M. 1422 assert_eq!( 1423 guest 1424 .ssh_command("lsblk | grep vdc | grep -c 16M") 1425 .unwrap() 1426 .trim() 1427 .parse::<u32>() 1428 .unwrap_or_default(), 1429 1 1430 ); 1431 1432 // Check if this block is RO or RW. 1433 assert_eq!( 1434 guest 1435 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1436 .unwrap() 1437 .trim() 1438 .parse::<u32>() 1439 .unwrap_or_default(), 1440 readonly as u32 1441 ); 1442 1443 // Check if the number of queues in /sys/block/vdc/mq matches the 1444 // expected num_queues. 1445 assert_eq!( 1446 guest 1447 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1448 .unwrap() 1449 .trim() 1450 .parse::<u32>() 1451 .unwrap_or_default(), 1452 num_queues as u32 1453 ); 1454 1455 // Mount the device 1456 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1457 guest.ssh_command("mkdir mount_image").unwrap(); 1458 guest 1459 .ssh_command( 1460 format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), 1461 ) 1462 .unwrap(); 1463 1464 // Check the content of the block device. The file "foo" should 1465 // contain "bar". 1466 assert_eq!( 1467 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1468 "bar" 1469 ); 1470 1471 // ACPI feature is needed. 1472 #[cfg(target_arch = "x86_64")] 1473 { 1474 guest.enable_memory_hotplug(); 1475 1476 // Add RAM to the VM 1477 let desired_ram = 1024 << 20; 1478 resize_command(&api_socket, None, Some(desired_ram), None, None); 1479 1480 thread::sleep(std::time::Duration::new(10, 0)); 1481 1482 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1483 1484 // Check again the content of the block device after the resize 1485 // has been performed. 1486 assert_eq!( 1487 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1488 "bar" 1489 ); 1490 } 1491 1492 // Unmount the device 1493 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1494 guest.ssh_command("rm -r mount_image").unwrap(); 1495 }); 1496 1497 kill_child(&mut child); 1498 let output = child.wait_with_output().unwrap(); 1499 1500 if let Some(mut daemon_child) = daemon_child { 1501 thread::sleep(std::time::Duration::new(5, 0)); 1502 let _ = daemon_child.kill(); 1503 let _ = daemon_child.wait(); 1504 } 1505 1506 handle_child_output(r, &output); 1507 } 1508 1509 fn test_boot_from_vhost_user_blk( 1510 num_queues: usize, 1511 readonly: bool, 1512 direct: bool, 1513 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1514 ) { 1515 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1516 let guest = Guest::new(Box::new(focal)); 1517 1518 let kernel_path = direct_kernel_boot_path(); 1519 1520 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1521 1522 let (blk_boot_params, daemon_child) = { 1523 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1524 // Start the daemon 1525 let (daemon_child, vubd_socket_path) = prepare_daemon( 1526 &guest.tmp_dir, 1527 disk_path.as_str(), 1528 num_queues, 1529 readonly, 1530 direct, 1531 ); 1532 1533 ( 1534 format!( 1535 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1536 ), 1537 Some(daemon_child), 1538 ) 1539 }; 1540 1541 let mut child = GuestCommand::new(&guest) 1542 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1543 .args(["--memory", "size=512M,shared=on"]) 1544 .args(["--kernel", kernel_path.to_str().unwrap()]) 1545 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1546 .args([ 1547 "--disk", 1548 blk_boot_params.as_str(), 1549 format!( 1550 "path={}", 1551 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1552 ) 1553 .as_str(), 1554 ]) 1555 .default_net() 1556 .capture_output() 1557 .spawn() 1558 .unwrap(); 1559 1560 let r = std::panic::catch_unwind(|| { 1561 guest.wait_vm_boot(None).unwrap(); 1562 1563 // Just check the VM booted correctly. 1564 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1565 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1566 }); 1567 kill_child(&mut child); 1568 let output = child.wait_with_output().unwrap(); 1569 1570 if let Some(mut daemon_child) = daemon_child { 1571 thread::sleep(std::time::Duration::new(5, 0)); 1572 let _ = daemon_child.kill(); 1573 let _ = daemon_child.wait(); 1574 } 1575 1576 handle_child_output(r, &output); 1577 } 1578 1579 fn _test_virtio_fs( 1580 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1581 hotplug: bool, 1582 pci_segment: Option<u16>, 1583 ) { 1584 #[cfg(target_arch = "aarch64")] 1585 let focal_image = if hotplug { 1586 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1587 } else { 1588 FOCAL_IMAGE_NAME.to_string() 1589 }; 1590 #[cfg(target_arch = "x86_64")] 1591 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1592 let focal = UbuntuDiskConfig::new(focal_image); 1593 let guest = Guest::new(Box::new(focal)); 1594 let api_socket = temp_api_path(&guest.tmp_dir); 1595 1596 let mut workload_path = dirs::home_dir().unwrap(); 1597 workload_path.push("workloads"); 1598 1599 let mut shared_dir = workload_path; 1600 shared_dir.push("shared_dir"); 1601 1602 #[cfg(target_arch = "x86_64")] 1603 let kernel_path = direct_kernel_boot_path(); 1604 #[cfg(target_arch = "aarch64")] 1605 let kernel_path = if hotplug { 1606 edk2_path() 1607 } else { 1608 direct_kernel_boot_path() 1609 }; 1610 1611 let (mut daemon_child, virtiofsd_socket_path) = 1612 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1613 1614 let mut guest_command = GuestCommand::new(&guest); 1615 guest_command 1616 .args(["--cpus", "boot=1"]) 1617 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1618 .args(["--kernel", kernel_path.to_str().unwrap()]) 1619 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1620 .default_disks() 1621 .default_net() 1622 .args(["--api-socket", &api_socket]); 1623 if pci_segment.is_some() { 1624 guest_command.args([ 1625 "--platform", 1626 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 1627 ]); 1628 } 1629 1630 let fs_params = format!( 1631 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1632 virtiofsd_socket_path, 1633 if let Some(pci_segment) = pci_segment { 1634 format!(",pci_segment={pci_segment}") 1635 } else { 1636 "".to_owned() 1637 } 1638 ); 1639 1640 if !hotplug { 1641 guest_command.args(["--fs", fs_params.as_str()]); 1642 } 1643 1644 let mut child = guest_command.capture_output().spawn().unwrap(); 1645 1646 let r = std::panic::catch_unwind(|| { 1647 guest.wait_vm_boot(None).unwrap(); 1648 1649 if hotplug { 1650 // Add fs to the VM 1651 let (cmd_success, cmd_output) = 1652 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1653 assert!(cmd_success); 1654 1655 if let Some(pci_segment) = pci_segment { 1656 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1657 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1658 ))); 1659 } else { 1660 assert!(String::from_utf8_lossy(&cmd_output) 1661 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1662 } 1663 1664 thread::sleep(std::time::Duration::new(10, 0)); 1665 } 1666 1667 // Mount shared directory through virtio_fs filesystem 1668 guest 1669 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1670 .unwrap(); 1671 1672 // Check file1 exists and its content is "foo" 1673 assert_eq!( 1674 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1675 "foo" 1676 ); 1677 // Check file2 does not exist 1678 guest 1679 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1680 .unwrap(); 1681 1682 // Check file3 exists and its content is "bar" 1683 assert_eq!( 1684 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1685 "bar" 1686 ); 1687 1688 // ACPI feature is needed. 1689 #[cfg(target_arch = "x86_64")] 1690 { 1691 guest.enable_memory_hotplug(); 1692 1693 // Add RAM to the VM 1694 let desired_ram = 1024 << 20; 1695 resize_command(&api_socket, None, Some(desired_ram), None, None); 1696 1697 thread::sleep(std::time::Duration::new(30, 0)); 1698 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1699 1700 // After the resize, check again that file1 exists and its 1701 // content is "foo". 1702 assert_eq!( 1703 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1704 "foo" 1705 ); 1706 } 1707 1708 if hotplug { 1709 // Remove from VM 1710 guest.ssh_command("sudo umount mount_dir").unwrap(); 1711 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1712 } 1713 }); 1714 1715 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1716 thread::sleep(std::time::Duration::new(10, 0)); 1717 let (daemon_child, virtiofsd_socket_path) = 1718 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1719 1720 let r = std::panic::catch_unwind(|| { 1721 thread::sleep(std::time::Duration::new(10, 0)); 1722 let fs_params = format!( 1723 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1724 virtiofsd_socket_path, 1725 if let Some(pci_segment) = pci_segment { 1726 format!(",pci_segment={pci_segment}") 1727 } else { 1728 "".to_owned() 1729 } 1730 ); 1731 1732 // Add back and check it works 1733 let (cmd_success, cmd_output) = 1734 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1735 assert!(cmd_success); 1736 if let Some(pci_segment) = pci_segment { 1737 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1738 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1739 ))); 1740 } else { 1741 assert!(String::from_utf8_lossy(&cmd_output) 1742 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1743 } 1744 1745 thread::sleep(std::time::Duration::new(10, 0)); 1746 // Mount shared directory through virtio_fs filesystem 1747 guest 1748 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1749 .unwrap(); 1750 1751 // Check file1 exists and its content is "foo" 1752 assert_eq!( 1753 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1754 "foo" 1755 ); 1756 }); 1757 1758 (r, Some(daemon_child)) 1759 } else { 1760 (r, None) 1761 }; 1762 1763 kill_child(&mut child); 1764 let output = child.wait_with_output().unwrap(); 1765 1766 let _ = daemon_child.kill(); 1767 let _ = daemon_child.wait(); 1768 1769 if let Some(mut daemon_child) = hotplug_daemon_child { 1770 let _ = daemon_child.kill(); 1771 let _ = daemon_child.wait(); 1772 } 1773 1774 handle_child_output(r, &output); 1775 } 1776 1777 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1778 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1779 let guest = Guest::new(Box::new(focal)); 1780 1781 let kernel_path = direct_kernel_boot_path(); 1782 1783 let pmem_temp_file = TempFile::new().unwrap(); 1784 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1785 1786 std::process::Command::new("mkfs.ext4") 1787 .arg(pmem_temp_file.as_path()) 1788 .output() 1789 .expect("Expect creating disk image to succeed"); 1790 1791 let mut child = GuestCommand::new(&guest) 1792 .args(["--cpus", "boot=1"]) 1793 .args(["--memory", "size=512M"]) 1794 .args(["--kernel", kernel_path.to_str().unwrap()]) 1795 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1796 .default_disks() 1797 .default_net() 1798 .args([ 1799 "--pmem", 1800 format!( 1801 "file={}{}{}", 1802 pmem_temp_file.as_path().to_str().unwrap(), 1803 if specify_size { ",size=128M" } else { "" }, 1804 if discard_writes { 1805 ",discard_writes=on" 1806 } else { 1807 "" 1808 } 1809 ) 1810 .as_str(), 1811 ]) 1812 .capture_output() 1813 .spawn() 1814 .unwrap(); 1815 1816 let r = std::panic::catch_unwind(|| { 1817 guest.wait_vm_boot(None).unwrap(); 1818 1819 // Check for the presence of /dev/pmem0 1820 assert_eq!( 1821 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1822 "/dev/pmem0" 1823 ); 1824 1825 // Check changes persist after reboot 1826 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1827 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1828 guest 1829 .ssh_command("echo test123 | sudo tee /mnt/test") 1830 .unwrap(); 1831 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1832 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1833 1834 guest.reboot_linux(0, None); 1835 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1836 assert_eq!( 1837 guest 1838 .ssh_command("sudo cat /mnt/test || true") 1839 .unwrap() 1840 .trim(), 1841 if discard_writes { "" } else { "test123" } 1842 ); 1843 }); 1844 1845 kill_child(&mut child); 1846 let output = child.wait_with_output().unwrap(); 1847 1848 handle_child_output(r, &output); 1849 } 1850 1851 fn get_fd_count(pid: u32) -> usize { 1852 fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() 1853 } 1854 1855 fn _test_virtio_vsock(hotplug: bool) { 1856 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1857 let guest = Guest::new(Box::new(focal)); 1858 1859 #[cfg(target_arch = "x86_64")] 1860 let kernel_path = direct_kernel_boot_path(); 1861 #[cfg(target_arch = "aarch64")] 1862 let kernel_path = if hotplug { 1863 edk2_path() 1864 } else { 1865 direct_kernel_boot_path() 1866 }; 1867 1868 let socket = temp_vsock_path(&guest.tmp_dir); 1869 let api_socket = temp_api_path(&guest.tmp_dir); 1870 1871 let mut cmd = GuestCommand::new(&guest); 1872 cmd.args(["--api-socket", &api_socket]); 1873 cmd.args(["--cpus", "boot=1"]); 1874 cmd.args(["--memory", "size=512M"]); 1875 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1876 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1877 cmd.default_disks(); 1878 cmd.default_net(); 1879 1880 if !hotplug { 1881 cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); 1882 } 1883 1884 let mut child = cmd.capture_output().spawn().unwrap(); 1885 1886 let r = std::panic::catch_unwind(|| { 1887 guest.wait_vm_boot(None).unwrap(); 1888 1889 if hotplug { 1890 let (cmd_success, cmd_output) = remote_command_w_output( 1891 &api_socket, 1892 "add-vsock", 1893 Some(format!("cid=3,socket={socket},id=test0").as_str()), 1894 ); 1895 assert!(cmd_success); 1896 assert!(String::from_utf8_lossy(&cmd_output) 1897 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1898 thread::sleep(std::time::Duration::new(10, 0)); 1899 // Check adding a second one fails 1900 assert!(!remote_command( 1901 &api_socket, 1902 "add-vsock", 1903 Some("cid=1234,socket=/tmp/fail") 1904 )); 1905 } 1906 1907 // Validate vsock works as expected. 1908 guest.check_vsock(socket.as_str()); 1909 guest.reboot_linux(0, None); 1910 // Validate vsock still works after a reboot. 1911 guest.check_vsock(socket.as_str()); 1912 1913 if hotplug { 1914 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1915 } 1916 }); 1917 1918 kill_child(&mut child); 1919 let output = child.wait_with_output().unwrap(); 1920 1921 handle_child_output(r, &output); 1922 } 1923 1924 fn get_ksm_pages_shared() -> u32 { 1925 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1926 .unwrap() 1927 .trim() 1928 .parse::<u32>() 1929 .unwrap() 1930 } 1931 1932 fn test_memory_mergeable(mergeable: bool) { 1933 let memory_param = if mergeable { 1934 "mergeable=on" 1935 } else { 1936 "mergeable=off" 1937 }; 1938 1939 // We are assuming the rest of the system in our CI is not using mergeable memory 1940 let ksm_ps_init = get_ksm_pages_shared(); 1941 assert!(ksm_ps_init == 0); 1942 1943 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1944 let guest1 = Guest::new(Box::new(focal1)); 1945 let mut child1 = GuestCommand::new(&guest1) 1946 .args(["--cpus", "boot=1"]) 1947 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1948 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1949 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1950 .default_disks() 1951 .args(["--net", guest1.default_net_string().as_str()]) 1952 .args(["--serial", "tty", "--console", "off"]) 1953 .capture_output() 1954 .spawn() 1955 .unwrap(); 1956 1957 let r = std::panic::catch_unwind(|| { 1958 guest1.wait_vm_boot(None).unwrap(); 1959 }); 1960 if r.is_err() { 1961 kill_child(&mut child1); 1962 let output = child1.wait_with_output().unwrap(); 1963 handle_child_output(r, &output); 1964 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1965 } 1966 1967 let ksm_ps_guest1 = get_ksm_pages_shared(); 1968 1969 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1970 let guest2 = Guest::new(Box::new(focal2)); 1971 let mut child2 = GuestCommand::new(&guest2) 1972 .args(["--cpus", "boot=1"]) 1973 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1974 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1975 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1976 .default_disks() 1977 .args(["--net", guest2.default_net_string().as_str()]) 1978 .args(["--serial", "tty", "--console", "off"]) 1979 .capture_output() 1980 .spawn() 1981 .unwrap(); 1982 1983 let r = std::panic::catch_unwind(|| { 1984 guest2.wait_vm_boot(None).unwrap(); 1985 let ksm_ps_guest2 = get_ksm_pages_shared(); 1986 1987 if mergeable { 1988 println!( 1989 "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" 1990 ); 1991 // We are expecting the number of shared pages to increase as the number of VM increases 1992 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1993 } else { 1994 assert!(ksm_ps_guest1 == 0); 1995 assert!(ksm_ps_guest2 == 0); 1996 } 1997 }); 1998 1999 kill_child(&mut child1); 2000 kill_child(&mut child2); 2001 2002 let output = child1.wait_with_output().unwrap(); 2003 child2.wait().unwrap(); 2004 2005 handle_child_output(r, &output); 2006 } 2007 2008 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 2009 let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); 2010 let reader = io::BufReader::new(smaps); 2011 2012 let mut skip_map: bool = false; 2013 let mut region_name: String = "".to_string(); 2014 let mut region_maps = HashMap::new(); 2015 for line in reader.lines() { 2016 let l = line.unwrap(); 2017 2018 if l.contains('-') { 2019 let values: Vec<&str> = l.split_whitespace().collect(); 2020 region_name = values.last().unwrap().trim().to_string(); 2021 if region_name == "0" { 2022 region_name = "anonymous".to_string() 2023 } 2024 } 2025 2026 // Each section begins with something that looks like: 2027 // Size: 2184 kB 2028 if l.starts_with("Size:") { 2029 let values: Vec<&str> = l.split_whitespace().collect(); 2030 let map_size = values[1].parse::<u32>().unwrap(); 2031 // We skip the assigned guest RAM map, its RSS is only 2032 // dependent on the guest actual memory usage. 2033 // Everything else can be added to the VMM overhead. 2034 skip_map = map_size >= guest_memory_size; 2035 continue; 2036 } 2037 2038 // If this is a map we're taking into account, then we only 2039 // count the RSS. The sum of all counted RSS is the VMM overhead. 2040 if !skip_map && l.starts_with("Rss:") { 2041 let values: Vec<&str> = l.split_whitespace().collect(); 2042 let value = values[1].trim().parse::<u32>().unwrap(); 2043 *region_maps.entry(region_name.clone()).or_insert(0) += value; 2044 } 2045 } 2046 2047 region_maps 2048 } 2049 2050 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 2051 let mut total = 0; 2052 2053 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 2054 eprintln!("{region_name}: {value}"); 2055 total += value; 2056 } 2057 2058 total 2059 } 2060 2061 fn process_rss_kib(pid: u32) -> usize { 2062 let command = format!("ps -q {pid} -o rss="); 2063 let rss = exec_host_command_output(&command); 2064 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 2065 } 2066 2067 // 10MB is our maximum accepted overhead. 2068 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 2069 2070 #[derive(PartialEq, Eq, PartialOrd)] 2071 struct Counters { 2072 rx_bytes: u64, 2073 rx_frames: u64, 2074 tx_bytes: u64, 2075 tx_frames: u64, 2076 read_bytes: u64, 2077 write_bytes: u64, 2078 read_ops: u64, 2079 write_ops: u64, 2080 } 2081 2082 fn get_counters(api_socket: &str) -> Counters { 2083 // Get counters 2084 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 2085 assert!(cmd_success); 2086 2087 let counters: HashMap<&str, HashMap<&str, u64>> = 2088 serde_json::from_slice(&cmd_output).unwrap_or_default(); 2089 2090 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 2091 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 2092 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 2093 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 2094 2095 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 2096 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 2097 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 2098 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 2099 2100 Counters { 2101 rx_bytes, 2102 rx_frames, 2103 tx_bytes, 2104 tx_frames, 2105 read_bytes, 2106 write_bytes, 2107 read_ops, 2108 write_ops, 2109 } 2110 } 2111 2112 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 2113 let (tx, rx) = mpsc::channel::<String>(); 2114 thread::spawn(move || loop { 2115 thread::sleep(std::time::Duration::new(1, 0)); 2116 let mut buf = [0; 512]; 2117 match pty.read(&mut buf) { 2118 Ok(_bytes) => { 2119 let output = std::str::from_utf8(&buf).unwrap().to_string(); 2120 match tx.send(output) { 2121 Ok(_) => (), 2122 Err(_) => break, 2123 } 2124 } 2125 Err(_) => break, 2126 } 2127 }); 2128 rx 2129 } 2130 2131 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 2132 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2133 assert!(cmd_success); 2134 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2135 assert_eq!("Pty", info["config"][pty_type]["mode"]); 2136 PathBuf::from( 2137 info["config"][pty_type]["file"] 2138 .as_str() 2139 .expect("Missing pty path"), 2140 ) 2141 } 2142 2143 // VFIO test network setup. 2144 // We reserve a different IP class for it: 172.18.0.0/24. 2145 #[cfg(target_arch = "x86_64")] 2146 fn setup_vfio_network_interfaces() { 2147 // 'vfio-br0' 2148 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 2149 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 2150 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 2151 // 'vfio-tap0' 2152 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 2153 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 2154 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 2155 // 'vfio-tap1' 2156 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 2157 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 2158 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 2159 // 'vfio-tap2' 2160 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 2161 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 2162 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 2163 // 'vfio-tap3' 2164 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 2165 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 2166 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 2167 } 2168 2169 // Tear VFIO test network down 2170 #[cfg(target_arch = "x86_64")] 2171 fn cleanup_vfio_network_interfaces() { 2172 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 2173 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 2174 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 2175 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 2176 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 2177 } 2178 2179 fn balloon_size(api_socket: &str) -> u64 { 2180 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2181 assert!(cmd_success); 2182 2183 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2184 let total_mem = &info["config"]["memory"]["size"] 2185 .to_string() 2186 .parse::<u64>() 2187 .unwrap(); 2188 let actual_mem = &info["memory_actual_size"] 2189 .to_string() 2190 .parse::<u64>() 2191 .unwrap(); 2192 total_mem - actual_mem 2193 } 2194 2195 fn vm_state(api_socket: &str) -> String { 2196 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2197 assert!(cmd_success); 2198 2199 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2200 let state = &info["state"].as_str().unwrap(); 2201 2202 state.to_string() 2203 } 2204 2205 // This test validates that it can find the virtio-iommu device at first. 2206 // It also verifies that both disks and the network card are attached to 2207 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 2208 // The last interesting part of this test is that it exercises the network 2209 // interface attached to the virtual IOMMU since this is the one used to 2210 // send all commands through SSH. 2211 fn _test_virtio_iommu(acpi: bool) { 2212 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 2213 // Focal image is still old. 2214 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 2215 // the kernel binary has been updated. 2216 #[cfg(target_arch = "aarch64")] 2217 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 2218 #[cfg(target_arch = "x86_64")] 2219 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2220 let focal = UbuntuDiskConfig::new(focal_image); 2221 let guest = Guest::new(Box::new(focal)); 2222 2223 #[cfg(target_arch = "x86_64")] 2224 let kernel_path = direct_kernel_boot_path(); 2225 #[cfg(target_arch = "aarch64")] 2226 let kernel_path = if acpi { 2227 edk2_path() 2228 } else { 2229 direct_kernel_boot_path() 2230 }; 2231 2232 let mut child = GuestCommand::new(&guest) 2233 .args(["--cpus", "boot=1"]) 2234 .args(["--memory", "size=512M"]) 2235 .args(["--kernel", kernel_path.to_str().unwrap()]) 2236 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2237 .args([ 2238 "--disk", 2239 format!( 2240 "path={},iommu=on", 2241 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2242 ) 2243 .as_str(), 2244 format!( 2245 "path={},iommu=on", 2246 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2247 ) 2248 .as_str(), 2249 ]) 2250 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 2251 .capture_output() 2252 .spawn() 2253 .unwrap(); 2254 2255 let r = std::panic::catch_unwind(|| { 2256 guest.wait_vm_boot(None).unwrap(); 2257 2258 // Verify the virtio-iommu device is present. 2259 assert!(guest 2260 .does_device_vendor_pair_match("0x1057", "0x1af4") 2261 .unwrap_or_default()); 2262 2263 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 2264 // different with ACPI. 2265 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 2266 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 2267 // and appear under folder '/sys/kernel/iommu_groups/'. 2268 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 2269 // which is the console. The first disk "0000:00:02.0" is in group '1'. 2270 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 2271 // contains "0000:00:02.0" which is the first disk. 2272 // 2273 // Verify the iommu group of the first disk. 2274 let iommu_group = !acpi as i32; 2275 assert_eq!( 2276 guest 2277 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2278 .unwrap() 2279 .trim(), 2280 "0000:00:02.0" 2281 ); 2282 2283 // Verify the iommu group of the second disk. 2284 let iommu_group = if acpi { 1 } else { 2 }; 2285 assert_eq!( 2286 guest 2287 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2288 .unwrap() 2289 .trim(), 2290 "0000:00:03.0" 2291 ); 2292 2293 // Verify the iommu group of the network card. 2294 let iommu_group = if acpi { 2 } else { 3 }; 2295 assert_eq!( 2296 guest 2297 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2298 .unwrap() 2299 .trim(), 2300 "0000:00:04.0" 2301 ); 2302 }); 2303 2304 kill_child(&mut child); 2305 let output = child.wait_with_output().unwrap(); 2306 2307 handle_child_output(r, &output); 2308 } 2309 2310 fn get_reboot_count(guest: &Guest) -> u32 { 2311 guest 2312 .ssh_command("sudo last | grep -c reboot") 2313 .unwrap() 2314 .trim() 2315 .parse::<u32>() 2316 .unwrap_or_default() 2317 } 2318 2319 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 2320 // Check for PCI device 2321 assert!(guest 2322 .does_device_vendor_pair_match("0x1063", "0x1af4") 2323 .unwrap_or_default()); 2324 2325 // Enable systemd watchdog 2326 guest 2327 .ssh_command(&format!( 2328 "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" 2329 )) 2330 .unwrap(); 2331 2332 guest.ssh_command("sudo systemctl daemon-reexec").unwrap(); 2333 } 2334 2335 fn make_guest_panic(guest: &Guest) { 2336 // Check for pvpanic device 2337 assert!(guest 2338 .does_device_vendor_pair_match("0x0011", "0x1b36") 2339 .unwrap_or_default()); 2340 2341 // Trigger guest a panic 2342 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 2343 } 2344 2345 mod common_parallel { 2346 use std::fs::OpenOptions; 2347 use std::io::SeekFrom; 2348 2349 use crate::*; 2350 2351 #[test] 2352 #[cfg(target_arch = "x86_64")] 2353 fn test_focal_hypervisor_fw() { 2354 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 2355 } 2356 2357 #[test] 2358 #[cfg(target_arch = "x86_64")] 2359 fn test_focal_ovmf() { 2360 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 2361 } 2362 2363 #[cfg(target_arch = "x86_64")] 2364 fn test_simple_launch(fw_path: String, disk_path: &str) { 2365 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 2366 let guest = Guest::new(disk_config); 2367 let event_path = temp_event_monitor_path(&guest.tmp_dir); 2368 2369 let mut child = GuestCommand::new(&guest) 2370 .args(["--cpus", "boot=1"]) 2371 .args(["--memory", "size=512M"]) 2372 .args(["--kernel", fw_path.as_str()]) 2373 .default_disks() 2374 .default_net() 2375 .args(["--serial", "tty", "--console", "off"]) 2376 .args(["--event-monitor", format!("path={event_path}").as_str()]) 2377 .capture_output() 2378 .spawn() 2379 .unwrap(); 2380 2381 let r = std::panic::catch_unwind(|| { 2382 guest.wait_vm_boot(Some(120)).unwrap(); 2383 2384 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2385 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2386 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2387 2388 let expected_sequential_events = [ 2389 &MetaEvent { 2390 event: "starting".to_string(), 2391 device_id: None, 2392 }, 2393 &MetaEvent { 2394 event: "booting".to_string(), 2395 device_id: None, 2396 }, 2397 &MetaEvent { 2398 event: "booted".to_string(), 2399 device_id: None, 2400 }, 2401 &MetaEvent { 2402 event: "activated".to_string(), 2403 device_id: Some("_disk0".to_string()), 2404 }, 2405 &MetaEvent { 2406 event: "reset".to_string(), 2407 device_id: Some("_disk0".to_string()), 2408 }, 2409 ]; 2410 assert!(check_sequential_events( 2411 &expected_sequential_events, 2412 &event_path 2413 )); 2414 2415 // It's been observed on the Bionic image that udev and snapd 2416 // services can cause some delay in the VM's shutdown. Disabling 2417 // them improves the reliability of this test. 2418 let _ = guest.ssh_command("sudo systemctl disable udev"); 2419 let _ = guest.ssh_command("sudo systemctl stop udev"); 2420 let _ = guest.ssh_command("sudo systemctl disable snapd"); 2421 let _ = guest.ssh_command("sudo systemctl stop snapd"); 2422 2423 guest.ssh_command("sudo poweroff").unwrap(); 2424 thread::sleep(std::time::Duration::new(20, 0)); 2425 let latest_events = [ 2426 &MetaEvent { 2427 event: "shutdown".to_string(), 2428 device_id: None, 2429 }, 2430 &MetaEvent { 2431 event: "deleted".to_string(), 2432 device_id: None, 2433 }, 2434 &MetaEvent { 2435 event: "shutdown".to_string(), 2436 device_id: None, 2437 }, 2438 ]; 2439 assert!(check_latest_events_exact(&latest_events, &event_path)); 2440 }); 2441 2442 kill_child(&mut child); 2443 let output = child.wait_with_output().unwrap(); 2444 2445 handle_child_output(r, &output); 2446 } 2447 2448 #[test] 2449 fn test_multi_cpu() { 2450 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2451 let jammy = UbuntuDiskConfig::new(jammy_image); 2452 let guest = Guest::new(Box::new(jammy)); 2453 2454 let mut cmd = GuestCommand::new(&guest); 2455 cmd.args(["--cpus", "boot=2,max=4"]) 2456 .args(["--memory", "size=512M"]) 2457 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2458 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2459 .capture_output() 2460 .default_disks() 2461 .default_net(); 2462 2463 let mut child = cmd.spawn().unwrap(); 2464 2465 let r = std::panic::catch_unwind(|| { 2466 guest.wait_vm_boot(Some(120)).unwrap(); 2467 2468 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2469 2470 assert_eq!( 2471 guest 2472 .ssh_command( 2473 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2474 ) 2475 .unwrap() 2476 .trim(), 2477 "smp: Brought up 1 node, 2 CPUs" 2478 ); 2479 }); 2480 2481 kill_child(&mut child); 2482 let output = child.wait_with_output().unwrap(); 2483 2484 handle_child_output(r, &output); 2485 } 2486 2487 #[test] 2488 fn test_cpu_topology_421() { 2489 test_cpu_topology(4, 2, 1, false); 2490 } 2491 2492 #[test] 2493 fn test_cpu_topology_142() { 2494 test_cpu_topology(1, 4, 2, false); 2495 } 2496 2497 #[test] 2498 fn test_cpu_topology_262() { 2499 test_cpu_topology(2, 6, 2, false); 2500 } 2501 2502 #[test] 2503 #[cfg(target_arch = "x86_64")] 2504 #[cfg(not(feature = "mshv"))] 2505 fn test_cpu_physical_bits() { 2506 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2507 let guest = Guest::new(Box::new(focal)); 2508 let max_phys_bits: u8 = 36; 2509 let mut child = GuestCommand::new(&guest) 2510 .args(["--cpus", &format!("max_phys_bits={max_phys_bits}")]) 2511 .args(["--memory", "size=512M"]) 2512 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2513 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2514 .default_disks() 2515 .default_net() 2516 .capture_output() 2517 .spawn() 2518 .unwrap(); 2519 2520 let r = std::panic::catch_unwind(|| { 2521 guest.wait_vm_boot(None).unwrap(); 2522 2523 assert!( 2524 guest 2525 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2526 .unwrap() 2527 .trim() 2528 .parse::<u8>() 2529 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2530 ); 2531 }); 2532 2533 kill_child(&mut child); 2534 let output = child.wait_with_output().unwrap(); 2535 2536 handle_child_output(r, &output); 2537 } 2538 2539 #[test] 2540 fn test_cpu_affinity() { 2541 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2542 let guest = Guest::new(Box::new(focal)); 2543 2544 // We need the host to have at least 4 CPUs if we want to be able 2545 // to run this test. 2546 let host_cpus_count = exec_host_command_output("nproc"); 2547 assert!( 2548 String::from_utf8_lossy(&host_cpus_count.stdout) 2549 .trim() 2550 .parse::<u16>() 2551 .unwrap_or(0) 2552 >= 4 2553 ); 2554 2555 let mut child = GuestCommand::new(&guest) 2556 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2557 .args(["--memory", "size=512M"]) 2558 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2559 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2560 .default_disks() 2561 .default_net() 2562 .capture_output() 2563 .spawn() 2564 .unwrap(); 2565 2566 let r = std::panic::catch_unwind(|| { 2567 guest.wait_vm_boot(None).unwrap(); 2568 let pid = child.id(); 2569 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2570 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2571 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2572 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2573 }); 2574 2575 kill_child(&mut child); 2576 let output = child.wait_with_output().unwrap(); 2577 handle_child_output(r, &output); 2578 } 2579 2580 #[test] 2581 fn test_virtio_queue_affinity() { 2582 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2583 let guest = Guest::new(Box::new(focal)); 2584 2585 // We need the host to have at least 4 CPUs if we want to be able 2586 // to run this test. 2587 let host_cpus_count = exec_host_command_output("nproc"); 2588 assert!( 2589 String::from_utf8_lossy(&host_cpus_count.stdout) 2590 .trim() 2591 .parse::<u16>() 2592 .unwrap_or(0) 2593 >= 4 2594 ); 2595 2596 let mut child = GuestCommand::new(&guest) 2597 .args(["--cpus", "boot=4"]) 2598 .args(["--memory", "size=512M"]) 2599 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2600 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2601 .args([ 2602 "--disk", 2603 format!( 2604 "path={}", 2605 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2606 ) 2607 .as_str(), 2608 format!( 2609 "path={},num_queues=4,queue_affinity=[0@[0,2],1@[1,3],2@[1],3@[3]]", 2610 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2611 ) 2612 .as_str(), 2613 ]) 2614 .default_net() 2615 .capture_output() 2616 .spawn() 2617 .unwrap(); 2618 2619 let r = std::panic::catch_unwind(|| { 2620 guest.wait_vm_boot(None).unwrap(); 2621 let pid = child.id(); 2622 let taskset_q0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2623 assert_eq!(String::from_utf8_lossy(&taskset_q0.stdout).trim(), "0,2"); 2624 let taskset_q1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2625 assert_eq!(String::from_utf8_lossy(&taskset_q1.stdout).trim(), "1,3"); 2626 let taskset_q2 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q2 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2627 assert_eq!(String::from_utf8_lossy(&taskset_q2.stdout).trim(), "1"); 2628 let taskset_q3 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q3 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2629 assert_eq!(String::from_utf8_lossy(&taskset_q3.stdout).trim(), "3"); 2630 }); 2631 2632 kill_child(&mut child); 2633 let output = child.wait_with_output().unwrap(); 2634 handle_child_output(r, &output); 2635 } 2636 2637 #[test] 2638 #[cfg(not(feature = "mshv"))] 2639 fn test_large_vm() { 2640 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2641 let guest = Guest::new(Box::new(focal)); 2642 let mut cmd = GuestCommand::new(&guest); 2643 cmd.args(["--cpus", "boot=48"]) 2644 .args(["--memory", "size=5120M"]) 2645 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2646 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2647 .args(["--serial", "tty"]) 2648 .args(["--console", "off"]) 2649 .capture_output() 2650 .default_disks() 2651 .default_net(); 2652 2653 let mut child = cmd.spawn().unwrap(); 2654 2655 guest.wait_vm_boot(None).unwrap(); 2656 2657 let r = std::panic::catch_unwind(|| { 2658 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2659 assert_eq!( 2660 guest 2661 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2662 .unwrap() 2663 .trim(), 2664 "0-47" 2665 ); 2666 2667 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2668 }); 2669 2670 kill_child(&mut child); 2671 let output = child.wait_with_output().unwrap(); 2672 2673 handle_child_output(r, &output); 2674 } 2675 2676 #[test] 2677 #[cfg(not(feature = "mshv"))] 2678 fn test_huge_memory() { 2679 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2680 let guest = Guest::new(Box::new(focal)); 2681 let mut cmd = GuestCommand::new(&guest); 2682 cmd.args(["--cpus", "boot=1"]) 2683 .args(["--memory", "size=128G"]) 2684 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2685 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2686 .capture_output() 2687 .default_disks() 2688 .default_net(); 2689 2690 let mut child = cmd.spawn().unwrap(); 2691 2692 guest.wait_vm_boot(Some(120)).unwrap(); 2693 2694 let r = std::panic::catch_unwind(|| { 2695 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2696 }); 2697 2698 kill_child(&mut child); 2699 let output = child.wait_with_output().unwrap(); 2700 2701 handle_child_output(r, &output); 2702 } 2703 2704 #[test] 2705 fn test_power_button() { 2706 _test_power_button(false); 2707 } 2708 2709 #[test] 2710 #[cfg(not(feature = "mshv"))] 2711 fn test_user_defined_memory_regions() { 2712 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2713 let guest = Guest::new(Box::new(focal)); 2714 let api_socket = temp_api_path(&guest.tmp_dir); 2715 2716 let kernel_path = direct_kernel_boot_path(); 2717 2718 let mut child = GuestCommand::new(&guest) 2719 .args(["--cpus", "boot=1"]) 2720 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2721 .args([ 2722 "--memory-zone", 2723 "id=mem0,size=1G,hotplug_size=2G", 2724 "id=mem1,size=1G,shared=on", 2725 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2726 ]) 2727 .args(["--kernel", kernel_path.to_str().unwrap()]) 2728 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2729 .args(["--api-socket", &api_socket]) 2730 .capture_output() 2731 .default_disks() 2732 .default_net() 2733 .spawn() 2734 .unwrap(); 2735 2736 let r = std::panic::catch_unwind(|| { 2737 guest.wait_vm_boot(None).unwrap(); 2738 2739 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2740 2741 guest.enable_memory_hotplug(); 2742 2743 resize_zone_command(&api_socket, "mem0", "3G"); 2744 thread::sleep(std::time::Duration::new(5, 0)); 2745 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2746 resize_zone_command(&api_socket, "mem2", "3G"); 2747 thread::sleep(std::time::Duration::new(5, 0)); 2748 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2749 resize_zone_command(&api_socket, "mem0", "2G"); 2750 thread::sleep(std::time::Duration::new(5, 0)); 2751 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2752 resize_zone_command(&api_socket, "mem2", "2G"); 2753 thread::sleep(std::time::Duration::new(5, 0)); 2754 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2755 2756 guest.reboot_linux(0, None); 2757 2758 // Check the amount of RAM after reboot 2759 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2760 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2761 2762 // Check if we can still resize down to the initial 'boot'size 2763 resize_zone_command(&api_socket, "mem0", "1G"); 2764 thread::sleep(std::time::Duration::new(5, 0)); 2765 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2766 resize_zone_command(&api_socket, "mem2", "1G"); 2767 thread::sleep(std::time::Duration::new(5, 0)); 2768 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2769 }); 2770 2771 kill_child(&mut child); 2772 let output = child.wait_with_output().unwrap(); 2773 2774 handle_child_output(r, &output); 2775 } 2776 2777 #[test] 2778 #[cfg(not(feature = "mshv"))] 2779 fn test_guest_numa_nodes() { 2780 _test_guest_numa_nodes(false); 2781 } 2782 2783 #[test] 2784 #[cfg(target_arch = "x86_64")] 2785 fn test_iommu_segments() { 2786 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2787 let focal = UbuntuDiskConfig::new(focal_image); 2788 let guest = Guest::new(Box::new(focal)); 2789 2790 // Prepare another disk file for the virtio-disk device 2791 let test_disk_path = String::from( 2792 guest 2793 .tmp_dir 2794 .as_path() 2795 .join("test-disk.raw") 2796 .to_str() 2797 .unwrap(), 2798 ); 2799 assert!( 2800 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2801 ); 2802 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2803 2804 let api_socket = temp_api_path(&guest.tmp_dir); 2805 let mut cmd = GuestCommand::new(&guest); 2806 2807 cmd.args(["--cpus", "boot=1"]) 2808 .args(["--api-socket", &api_socket]) 2809 .args(["--memory", "size=512M"]) 2810 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2811 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2812 .args([ 2813 "--platform", 2814 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS},iommu_segments=[1]"), 2815 ]) 2816 .default_disks() 2817 .capture_output() 2818 .default_net(); 2819 2820 let mut child = cmd.spawn().unwrap(); 2821 2822 guest.wait_vm_boot(None).unwrap(); 2823 2824 let r = std::panic::catch_unwind(|| { 2825 let (cmd_success, cmd_output) = remote_command_w_output( 2826 &api_socket, 2827 "add-disk", 2828 Some( 2829 format!( 2830 "path={},id=test0,pci_segment=1,iommu=on", 2831 test_disk_path.as_str() 2832 ) 2833 .as_str(), 2834 ), 2835 ); 2836 assert!(cmd_success); 2837 assert!(String::from_utf8_lossy(&cmd_output) 2838 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2839 2840 // Check IOMMU setup 2841 assert!(guest 2842 .does_device_vendor_pair_match("0x1057", "0x1af4") 2843 .unwrap_or_default()); 2844 assert_eq!( 2845 guest 2846 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2847 .unwrap() 2848 .trim(), 2849 "0001:00:01.0" 2850 ); 2851 }); 2852 2853 kill_child(&mut child); 2854 let output = child.wait_with_output().unwrap(); 2855 2856 handle_child_output(r, &output); 2857 } 2858 2859 #[test] 2860 fn test_pci_msi() { 2861 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2862 let guest = Guest::new(Box::new(focal)); 2863 let mut cmd = GuestCommand::new(&guest); 2864 cmd.args(["--cpus", "boot=1"]) 2865 .args(["--memory", "size=512M"]) 2866 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2867 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2868 .capture_output() 2869 .default_disks() 2870 .default_net(); 2871 2872 let mut child = cmd.spawn().unwrap(); 2873 2874 guest.wait_vm_boot(None).unwrap(); 2875 2876 #[cfg(target_arch = "x86_64")] 2877 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2878 #[cfg(target_arch = "aarch64")] 2879 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2880 2881 let r = std::panic::catch_unwind(|| { 2882 assert_eq!( 2883 guest 2884 .ssh_command(grep_cmd) 2885 .unwrap() 2886 .trim() 2887 .parse::<u32>() 2888 .unwrap_or_default(), 2889 12 2890 ); 2891 }); 2892 2893 kill_child(&mut child); 2894 let output = child.wait_with_output().unwrap(); 2895 2896 handle_child_output(r, &output); 2897 } 2898 2899 #[test] 2900 fn test_virtio_net_ctrl_queue() { 2901 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2902 let guest = Guest::new(Box::new(focal)); 2903 let mut cmd = GuestCommand::new(&guest); 2904 cmd.args(["--cpus", "boot=1"]) 2905 .args(["--memory", "size=512M"]) 2906 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2907 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2908 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2909 .capture_output() 2910 .default_disks(); 2911 2912 let mut child = cmd.spawn().unwrap(); 2913 2914 guest.wait_vm_boot(None).unwrap(); 2915 2916 #[cfg(target_arch = "aarch64")] 2917 let iface = "enp0s4"; 2918 #[cfg(target_arch = "x86_64")] 2919 let iface = "ens4"; 2920 2921 let r = std::panic::catch_unwind(|| { 2922 assert_eq!( 2923 guest 2924 .ssh_command( 2925 format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() 2926 ) 2927 .unwrap() 2928 .trim(), 2929 "success" 2930 ); 2931 assert_eq!( 2932 guest 2933 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 2934 .unwrap() 2935 .trim(), 2936 "3000" 2937 ); 2938 }); 2939 2940 kill_child(&mut child); 2941 let output = child.wait_with_output().unwrap(); 2942 2943 handle_child_output(r, &output); 2944 } 2945 2946 #[test] 2947 fn test_pci_multiple_segments() { 2948 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2949 let guest = Guest::new(Box::new(focal)); 2950 2951 // Prepare another disk file for the virtio-disk device 2952 let test_disk_path = String::from( 2953 guest 2954 .tmp_dir 2955 .as_path() 2956 .join("test-disk.raw") 2957 .to_str() 2958 .unwrap(), 2959 ); 2960 assert!( 2961 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2962 ); 2963 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2964 2965 let mut cmd = GuestCommand::new(&guest); 2966 cmd.args(["--cpus", "boot=1"]) 2967 .args(["--memory", "size=512M"]) 2968 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2969 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2970 .args([ 2971 "--platform", 2972 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 2973 ]) 2974 .args([ 2975 "--disk", 2976 format!( 2977 "path={}", 2978 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2979 ) 2980 .as_str(), 2981 format!( 2982 "path={}", 2983 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2984 ) 2985 .as_str(), 2986 format!("path={test_disk_path},pci_segment=15").as_str(), 2987 ]) 2988 .capture_output() 2989 .default_net(); 2990 2991 let mut child = cmd.spawn().unwrap(); 2992 2993 guest.wait_vm_boot(None).unwrap(); 2994 2995 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2996 2997 let r = std::panic::catch_unwind(|| { 2998 // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. 2999 assert_eq!( 3000 guest 3001 .ssh_command(grep_cmd) 3002 .unwrap() 3003 .trim() 3004 .parse::<u16>() 3005 .unwrap_or_default(), 3006 MAX_NUM_PCI_SEGMENTS 3007 ); 3008 3009 // Check both if /dev/vdc exists and if the block size is 4M. 3010 assert_eq!( 3011 guest 3012 .ssh_command("lsblk | grep vdc | grep -c 4M") 3013 .unwrap() 3014 .trim() 3015 .parse::<u32>() 3016 .unwrap_or_default(), 3017 1 3018 ); 3019 3020 // Mount the device. 3021 guest.ssh_command("mkdir mount_image").unwrap(); 3022 guest 3023 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 3024 .unwrap(); 3025 // Grant all users with write permission. 3026 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 3027 3028 // Write something to the device. 3029 guest 3030 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 3031 .unwrap(); 3032 3033 // Check the content of the block device. The file "foo" should 3034 // contain "bar". 3035 assert_eq!( 3036 guest 3037 .ssh_command("sudo cat mount_image/foo") 3038 .unwrap() 3039 .trim(), 3040 "bar" 3041 ); 3042 }); 3043 3044 kill_child(&mut child); 3045 let output = child.wait_with_output().unwrap(); 3046 3047 handle_child_output(r, &output); 3048 } 3049 3050 #[test] 3051 fn test_pci_multiple_segments_numa_node() { 3052 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3053 let guest = Guest::new(Box::new(focal)); 3054 let api_socket = temp_api_path(&guest.tmp_dir); 3055 #[cfg(target_arch = "x86_64")] 3056 let kernel_path = direct_kernel_boot_path(); 3057 #[cfg(target_arch = "aarch64")] 3058 let kernel_path = edk2_path(); 3059 3060 // Prepare another disk file for the virtio-disk device 3061 let test_disk_path = String::from( 3062 guest 3063 .tmp_dir 3064 .as_path() 3065 .join("test-disk.raw") 3066 .to_str() 3067 .unwrap(), 3068 ); 3069 assert!( 3070 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 3071 ); 3072 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 3073 const TEST_DISK_NODE: u16 = 1; 3074 3075 let mut child = GuestCommand::new(&guest) 3076 .args(["--platform", "num_pci_segments=2"]) 3077 .args(["--cpus", "boot=2"]) 3078 .args(["--memory", "size=0"]) 3079 .args(["--memory-zone", "id=mem0,size=256M", "id=mem1,size=256M"]) 3080 .args([ 3081 "--numa", 3082 "guest_numa_id=0,cpus=[0],distances=[1@20],memory_zones=mem0,pci_segments=[0]", 3083 "guest_numa_id=1,cpus=[1],distances=[0@20],memory_zones=mem1,pci_segments=[1]", 3084 ]) 3085 .args(["--kernel", kernel_path.to_str().unwrap()]) 3086 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3087 .args(["--api-socket", &api_socket]) 3088 .capture_output() 3089 .args([ 3090 "--disk", 3091 format!( 3092 "path={}", 3093 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3094 ) 3095 .as_str(), 3096 format!( 3097 "path={}", 3098 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3099 ) 3100 .as_str(), 3101 format!("path={test_disk_path},pci_segment={TEST_DISK_NODE}").as_str(), 3102 ]) 3103 .default_net() 3104 .spawn() 3105 .unwrap(); 3106 3107 let cmd = "cat /sys/block/vdc/device/../numa_node"; 3108 3109 let r = std::panic::catch_unwind(|| { 3110 guest.wait_vm_boot(None).unwrap(); 3111 3112 assert_eq!( 3113 guest 3114 .ssh_command(cmd) 3115 .unwrap() 3116 .trim() 3117 .parse::<u16>() 3118 .unwrap_or_default(), 3119 TEST_DISK_NODE 3120 ); 3121 }); 3122 3123 kill_child(&mut child); 3124 let output = child.wait_with_output().unwrap(); 3125 3126 handle_child_output(r, &output); 3127 } 3128 3129 #[test] 3130 fn test_direct_kernel_boot() { 3131 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3132 let guest = Guest::new(Box::new(focal)); 3133 3134 let kernel_path = direct_kernel_boot_path(); 3135 3136 let mut child = GuestCommand::new(&guest) 3137 .args(["--cpus", "boot=1"]) 3138 .args(["--memory", "size=512M"]) 3139 .args(["--kernel", kernel_path.to_str().unwrap()]) 3140 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3141 .default_disks() 3142 .default_net() 3143 .capture_output() 3144 .spawn() 3145 .unwrap(); 3146 3147 let r = std::panic::catch_unwind(|| { 3148 guest.wait_vm_boot(None).unwrap(); 3149 3150 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3151 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3152 3153 let grep_cmd = if cfg!(target_arch = "x86_64") { 3154 "grep -c PCI-MSI /proc/interrupts" 3155 } else { 3156 "grep -c ITS-MSI /proc/interrupts" 3157 }; 3158 assert_eq!( 3159 guest 3160 .ssh_command(grep_cmd) 3161 .unwrap() 3162 .trim() 3163 .parse::<u32>() 3164 .unwrap_or_default(), 3165 12 3166 ); 3167 }); 3168 3169 kill_child(&mut child); 3170 let output = child.wait_with_output().unwrap(); 3171 3172 handle_child_output(r, &output); 3173 } 3174 3175 #[test] 3176 #[cfg(target_arch = "x86_64")] 3177 fn test_direct_kernel_boot_bzimage() { 3178 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3179 let guest = Guest::new(Box::new(focal)); 3180 3181 let mut kernel_path = direct_kernel_boot_path(); 3182 // Replace the default kernel with the bzImage. 3183 kernel_path.pop(); 3184 kernel_path.push("bzImage"); 3185 3186 let mut child = GuestCommand::new(&guest) 3187 .args(["--cpus", "boot=1"]) 3188 .args(["--memory", "size=512M"]) 3189 .args(["--kernel", kernel_path.to_str().unwrap()]) 3190 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3191 .default_disks() 3192 .default_net() 3193 .capture_output() 3194 .spawn() 3195 .unwrap(); 3196 3197 let r = std::panic::catch_unwind(|| { 3198 guest.wait_vm_boot(None).unwrap(); 3199 3200 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3201 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3202 3203 let grep_cmd = if cfg!(target_arch = "x86_64") { 3204 "grep -c PCI-MSI /proc/interrupts" 3205 } else { 3206 "grep -c ITS-MSI /proc/interrupts" 3207 }; 3208 assert_eq!( 3209 guest 3210 .ssh_command(grep_cmd) 3211 .unwrap() 3212 .trim() 3213 .parse::<u32>() 3214 .unwrap_or_default(), 3215 12 3216 ); 3217 }); 3218 3219 kill_child(&mut child); 3220 let output = child.wait_with_output().unwrap(); 3221 3222 handle_child_output(r, &output); 3223 } 3224 3225 fn _test_virtio_block(image_name: &str, disable_io_uring: bool, disable_aio: bool) { 3226 let focal = UbuntuDiskConfig::new(image_name.to_string()); 3227 let guest = Guest::new(Box::new(focal)); 3228 3229 let mut workload_path = dirs::home_dir().unwrap(); 3230 workload_path.push("workloads"); 3231 3232 let mut blk_file_path = workload_path; 3233 blk_file_path.push("blk.img"); 3234 3235 let kernel_path = direct_kernel_boot_path(); 3236 3237 let mut cloud_child = GuestCommand::new(&guest) 3238 .args(["--cpus", "boot=4"]) 3239 .args(["--memory", "size=512M,shared=on"]) 3240 .args(["--kernel", kernel_path.to_str().unwrap()]) 3241 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3242 .args([ 3243 "--disk", 3244 format!( 3245 "path={}", 3246 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3247 ) 3248 .as_str(), 3249 format!( 3250 "path={}", 3251 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3252 ) 3253 .as_str(), 3254 format!( 3255 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", 3256 blk_file_path.to_str().unwrap(), 3257 disable_io_uring, 3258 disable_aio, 3259 ) 3260 .as_str(), 3261 ]) 3262 .default_net() 3263 .capture_output() 3264 .spawn() 3265 .unwrap(); 3266 3267 let r = std::panic::catch_unwind(|| { 3268 guest.wait_vm_boot(None).unwrap(); 3269 3270 // Check both if /dev/vdc exists and if the block size is 16M. 3271 assert_eq!( 3272 guest 3273 .ssh_command("lsblk | grep vdc | grep -c 16M") 3274 .unwrap() 3275 .trim() 3276 .parse::<u32>() 3277 .unwrap_or_default(), 3278 1 3279 ); 3280 3281 // Check both if /dev/vdc exists and if this block is RO. 3282 assert_eq!( 3283 guest 3284 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 3285 .unwrap() 3286 .trim() 3287 .parse::<u32>() 3288 .unwrap_or_default(), 3289 1 3290 ); 3291 3292 // Check if the number of queues is 4. 3293 assert_eq!( 3294 guest 3295 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 3296 .unwrap() 3297 .trim() 3298 .parse::<u32>() 3299 .unwrap_or_default(), 3300 4 3301 ); 3302 }); 3303 3304 let _ = cloud_child.kill(); 3305 let output = cloud_child.wait_with_output().unwrap(); 3306 3307 handle_child_output(r, &output); 3308 } 3309 3310 #[test] 3311 fn test_virtio_block_io_uring() { 3312 _test_virtio_block(FOCAL_IMAGE_NAME, false, true) 3313 } 3314 3315 #[test] 3316 fn test_virtio_block_aio() { 3317 _test_virtio_block(FOCAL_IMAGE_NAME, true, false) 3318 } 3319 3320 #[test] 3321 fn test_virtio_block_sync() { 3322 _test_virtio_block(FOCAL_IMAGE_NAME, true, true) 3323 } 3324 3325 #[test] 3326 fn test_virtio_block_qcow2() { 3327 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false, false) 3328 } 3329 3330 #[test] 3331 fn test_virtio_block_qcow2_backing_file() { 3332 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE, false, false) 3333 } 3334 3335 #[test] 3336 fn test_virtio_block_vhd() { 3337 let mut workload_path = dirs::home_dir().unwrap(); 3338 workload_path.push("workloads"); 3339 3340 let mut raw_file_path = workload_path.clone(); 3341 let mut vhd_file_path = workload_path; 3342 raw_file_path.push(FOCAL_IMAGE_NAME); 3343 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 3344 3345 // Generate VHD file from RAW file 3346 std::process::Command::new("qemu-img") 3347 .arg("convert") 3348 .arg("-p") 3349 .args(["-f", "raw"]) 3350 .args(["-O", "vpc"]) 3351 .args(["-o", "subformat=fixed"]) 3352 .arg(raw_file_path.to_str().unwrap()) 3353 .arg(vhd_file_path.to_str().unwrap()) 3354 .output() 3355 .expect("Expect generating VHD image from RAW image"); 3356 3357 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false, false) 3358 } 3359 3360 #[test] 3361 fn test_virtio_block_vhdx() { 3362 let mut workload_path = dirs::home_dir().unwrap(); 3363 workload_path.push("workloads"); 3364 3365 let mut raw_file_path = workload_path.clone(); 3366 let mut vhdx_file_path = workload_path; 3367 raw_file_path.push(FOCAL_IMAGE_NAME); 3368 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 3369 3370 // Generate dynamic VHDX file from RAW file 3371 std::process::Command::new("qemu-img") 3372 .arg("convert") 3373 .arg("-p") 3374 .args(["-f", "raw"]) 3375 .args(["-O", "vhdx"]) 3376 .arg(raw_file_path.to_str().unwrap()) 3377 .arg(vhdx_file_path.to_str().unwrap()) 3378 .output() 3379 .expect("Expect generating dynamic VHDx image from RAW image"); 3380 3381 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false, false) 3382 } 3383 3384 #[test] 3385 fn test_virtio_block_dynamic_vhdx_expand() { 3386 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 3387 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 3388 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 3389 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 3390 3391 let mut workload_path = dirs::home_dir().unwrap(); 3392 workload_path.push("workloads"); 3393 3394 let mut vhdx_file_path = workload_path; 3395 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 3396 let vhdx_path = vhdx_file_path.to_str().unwrap(); 3397 3398 // Generate a 100 MiB dynamic VHDX file 3399 std::process::Command::new("qemu-img") 3400 .arg("create") 3401 .args(["-f", "vhdx"]) 3402 .arg(vhdx_path) 3403 .arg(VIRTUAL_DISK_SIZE.to_string()) 3404 .output() 3405 .expect("Expect generating dynamic VHDx image from RAW image"); 3406 3407 // Check if the size matches with empty VHDx file size 3408 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 3409 3410 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3411 let guest = Guest::new(Box::new(focal)); 3412 let kernel_path = direct_kernel_boot_path(); 3413 3414 let mut cloud_child = GuestCommand::new(&guest) 3415 .args(["--cpus", "boot=1"]) 3416 .args(["--memory", "size=512M"]) 3417 .args(["--kernel", kernel_path.to_str().unwrap()]) 3418 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3419 .args([ 3420 "--disk", 3421 format!( 3422 "path={}", 3423 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3424 ) 3425 .as_str(), 3426 format!( 3427 "path={}", 3428 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3429 ) 3430 .as_str(), 3431 format!("path={vhdx_path}").as_str(), 3432 ]) 3433 .default_net() 3434 .capture_output() 3435 .spawn() 3436 .unwrap(); 3437 3438 let r = std::panic::catch_unwind(|| { 3439 guest.wait_vm_boot(None).unwrap(); 3440 3441 // Check both if /dev/vdc exists and if the block size is 100 MiB. 3442 assert_eq!( 3443 guest 3444 .ssh_command("lsblk | grep vdc | grep -c 100M") 3445 .unwrap() 3446 .trim() 3447 .parse::<u32>() 3448 .unwrap_or_default(), 3449 1 3450 ); 3451 3452 // Write 100 MB of data to the VHDx disk 3453 guest 3454 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 3455 .unwrap(); 3456 }); 3457 3458 // Check if the size matches with expected expanded VHDx file size 3459 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 3460 3461 kill_child(&mut cloud_child); 3462 let output = cloud_child.wait_with_output().unwrap(); 3463 3464 handle_child_output(r, &output); 3465 } 3466 3467 fn vhdx_image_size(disk_name: &str) -> u64 { 3468 std::fs::File::open(disk_name) 3469 .unwrap() 3470 .seek(SeekFrom::End(0)) 3471 .unwrap() 3472 } 3473 3474 #[test] 3475 fn test_virtio_block_direct_and_firmware() { 3476 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3477 let guest = Guest::new(Box::new(focal)); 3478 3479 // The OS disk must be copied to a location that is not backed by 3480 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 3481 // with EINVAL because tmpfs doesn't support this flag. 3482 let mut workloads_path = dirs::home_dir().unwrap(); 3483 workloads_path.push("workloads"); 3484 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 3485 let mut os_path = os_dir.as_path().to_path_buf(); 3486 os_path.push("osdisk.img"); 3487 rate_limited_copy( 3488 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3489 os_path.as_path(), 3490 ) 3491 .expect("copying of OS disk failed"); 3492 3493 let mut child = GuestCommand::new(&guest) 3494 .args(["--cpus", "boot=1"]) 3495 .args(["--memory", "size=512M"]) 3496 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 3497 .args([ 3498 "--disk", 3499 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 3500 format!( 3501 "path={}", 3502 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3503 ) 3504 .as_str(), 3505 ]) 3506 .default_net() 3507 .capture_output() 3508 .spawn() 3509 .unwrap(); 3510 3511 let r = std::panic::catch_unwind(|| { 3512 guest.wait_vm_boot(Some(120)).unwrap(); 3513 }); 3514 3515 kill_child(&mut child); 3516 let output = child.wait_with_output().unwrap(); 3517 3518 handle_child_output(r, &output); 3519 } 3520 3521 #[test] 3522 fn test_vhost_user_net_default() { 3523 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 3524 } 3525 3526 #[test] 3527 fn test_vhost_user_net_named_tap() { 3528 test_vhost_user_net( 3529 Some("mytap0"), 3530 2, 3531 &prepare_vhost_user_net_daemon, 3532 false, 3533 false, 3534 ) 3535 } 3536 3537 #[test] 3538 fn test_vhost_user_net_existing_tap() { 3539 test_vhost_user_net( 3540 Some("vunet-tap0"), 3541 2, 3542 &prepare_vhost_user_net_daemon, 3543 false, 3544 false, 3545 ) 3546 } 3547 3548 #[test] 3549 fn test_vhost_user_net_multiple_queues() { 3550 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 3551 } 3552 3553 #[test] 3554 fn test_vhost_user_net_tap_multiple_queues() { 3555 test_vhost_user_net( 3556 Some("vunet-tap1"), 3557 4, 3558 &prepare_vhost_user_net_daemon, 3559 false, 3560 false, 3561 ) 3562 } 3563 3564 #[test] 3565 fn test_vhost_user_net_host_mac() { 3566 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 3567 } 3568 3569 #[test] 3570 fn test_vhost_user_net_client_mode() { 3571 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 3572 } 3573 3574 #[test] 3575 #[cfg(not(target_arch = "aarch64"))] 3576 fn test_vhost_user_blk_default() { 3577 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 3578 } 3579 3580 #[test] 3581 #[cfg(not(target_arch = "aarch64"))] 3582 fn test_vhost_user_blk_readonly() { 3583 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3584 } 3585 3586 #[test] 3587 #[cfg(not(target_arch = "aarch64"))] 3588 fn test_vhost_user_blk_direct() { 3589 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3590 } 3591 3592 #[test] 3593 fn test_boot_from_vhost_user_blk_default() { 3594 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3595 } 3596 3597 #[test] 3598 #[cfg(target_arch = "x86_64")] 3599 fn test_split_irqchip() { 3600 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3601 let guest = Guest::new(Box::new(focal)); 3602 3603 let mut child = GuestCommand::new(&guest) 3604 .args(["--cpus", "boot=1"]) 3605 .args(["--memory", "size=512M"]) 3606 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3607 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3608 .default_disks() 3609 .default_net() 3610 .capture_output() 3611 .spawn() 3612 .unwrap(); 3613 3614 let r = std::panic::catch_unwind(|| { 3615 guest.wait_vm_boot(None).unwrap(); 3616 3617 assert_eq!( 3618 guest 3619 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3620 .unwrap() 3621 .trim() 3622 .parse::<u32>() 3623 .unwrap_or(1), 3624 0 3625 ); 3626 assert_eq!( 3627 guest 3628 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3629 .unwrap() 3630 .trim() 3631 .parse::<u32>() 3632 .unwrap_or(1), 3633 0 3634 ); 3635 }); 3636 3637 kill_child(&mut child); 3638 let output = child.wait_with_output().unwrap(); 3639 3640 handle_child_output(r, &output); 3641 } 3642 3643 #[test] 3644 #[cfg(target_arch = "x86_64")] 3645 fn test_dmi_serial_number() { 3646 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3647 let guest = Guest::new(Box::new(focal)); 3648 3649 let mut child = GuestCommand::new(&guest) 3650 .args(["--cpus", "boot=1"]) 3651 .args(["--memory", "size=512M"]) 3652 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3653 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3654 .args(["--platform", "serial_number=a=b;c=d"]) 3655 .default_disks() 3656 .default_net() 3657 .capture_output() 3658 .spawn() 3659 .unwrap(); 3660 3661 let r = std::panic::catch_unwind(|| { 3662 guest.wait_vm_boot(None).unwrap(); 3663 3664 assert_eq!( 3665 guest 3666 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3667 .unwrap() 3668 .trim(), 3669 "a=b;c=d" 3670 ); 3671 }); 3672 3673 kill_child(&mut child); 3674 let output = child.wait_with_output().unwrap(); 3675 3676 handle_child_output(r, &output); 3677 } 3678 3679 #[test] 3680 #[cfg(target_arch = "x86_64")] 3681 fn test_dmi_uuid() { 3682 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3683 let guest = Guest::new(Box::new(focal)); 3684 3685 let mut child = GuestCommand::new(&guest) 3686 .args(["--cpus", "boot=1"]) 3687 .args(["--memory", "size=512M"]) 3688 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3689 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3690 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3691 .default_disks() 3692 .default_net() 3693 .capture_output() 3694 .spawn() 3695 .unwrap(); 3696 3697 let r = std::panic::catch_unwind(|| { 3698 guest.wait_vm_boot(None).unwrap(); 3699 3700 assert_eq!( 3701 guest 3702 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3703 .unwrap() 3704 .trim(), 3705 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3706 ); 3707 }); 3708 3709 kill_child(&mut child); 3710 let output = child.wait_with_output().unwrap(); 3711 3712 handle_child_output(r, &output); 3713 } 3714 3715 #[test] 3716 #[cfg(target_arch = "x86_64")] 3717 fn test_dmi_oem_strings() { 3718 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3719 let guest = Guest::new(Box::new(focal)); 3720 3721 let s1 = "io.systemd.credential:xx=yy"; 3722 let s2 = "This is a test string"; 3723 3724 let oem_strings = format!("oem_strings=[{s1},{s2}]"); 3725 3726 let mut child = GuestCommand::new(&guest) 3727 .args(["--cpus", "boot=1"]) 3728 .args(["--memory", "size=512M"]) 3729 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3730 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3731 .args(["--platform", &oem_strings]) 3732 .default_disks() 3733 .default_net() 3734 .capture_output() 3735 .spawn() 3736 .unwrap(); 3737 3738 let r = std::panic::catch_unwind(|| { 3739 guest.wait_vm_boot(None).unwrap(); 3740 3741 assert_eq!( 3742 guest 3743 .ssh_command("sudo dmidecode --oem-string count") 3744 .unwrap() 3745 .trim(), 3746 "2" 3747 ); 3748 3749 assert_eq!( 3750 guest 3751 .ssh_command("sudo dmidecode --oem-string 1") 3752 .unwrap() 3753 .trim(), 3754 s1 3755 ); 3756 3757 assert_eq!( 3758 guest 3759 .ssh_command("sudo dmidecode --oem-string 2") 3760 .unwrap() 3761 .trim(), 3762 s2 3763 ); 3764 }); 3765 3766 kill_child(&mut child); 3767 let output = child.wait_with_output().unwrap(); 3768 3769 handle_child_output(r, &output); 3770 } 3771 3772 #[test] 3773 fn test_virtio_fs() { 3774 _test_virtio_fs(&prepare_virtiofsd, false, None) 3775 } 3776 3777 #[test] 3778 fn test_virtio_fs_hotplug() { 3779 _test_virtio_fs(&prepare_virtiofsd, true, None) 3780 } 3781 3782 #[test] 3783 #[cfg(not(feature = "mshv"))] 3784 fn test_virtio_fs_multi_segment_hotplug() { 3785 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3786 } 3787 3788 #[test] 3789 #[cfg(not(feature = "mshv"))] 3790 fn test_virtio_fs_multi_segment() { 3791 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3792 } 3793 3794 #[test] 3795 fn test_virtio_pmem_persist_writes() { 3796 test_virtio_pmem(false, false) 3797 } 3798 3799 #[test] 3800 fn test_virtio_pmem_discard_writes() { 3801 test_virtio_pmem(true, false) 3802 } 3803 3804 #[test] 3805 fn test_virtio_pmem_with_size() { 3806 test_virtio_pmem(true, true) 3807 } 3808 3809 #[test] 3810 fn test_boot_from_virtio_pmem() { 3811 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3812 let guest = Guest::new(Box::new(focal)); 3813 3814 let kernel_path = direct_kernel_boot_path(); 3815 3816 let mut child = GuestCommand::new(&guest) 3817 .args(["--cpus", "boot=1"]) 3818 .args(["--memory", "size=512M"]) 3819 .args(["--kernel", kernel_path.to_str().unwrap()]) 3820 .args([ 3821 "--disk", 3822 format!( 3823 "path={}", 3824 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3825 ) 3826 .as_str(), 3827 ]) 3828 .default_net() 3829 .args([ 3830 "--pmem", 3831 format!( 3832 "file={},size={}", 3833 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3834 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3835 .unwrap() 3836 .len() 3837 ) 3838 .as_str(), 3839 ]) 3840 .args([ 3841 "--cmdline", 3842 DIRECT_KERNEL_BOOT_CMDLINE 3843 .replace("vda1", "pmem0p1") 3844 .as_str(), 3845 ]) 3846 .capture_output() 3847 .spawn() 3848 .unwrap(); 3849 3850 let r = std::panic::catch_unwind(|| { 3851 guest.wait_vm_boot(None).unwrap(); 3852 3853 // Simple checks to validate the VM booted properly 3854 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3855 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3856 }); 3857 3858 kill_child(&mut child); 3859 let output = child.wait_with_output().unwrap(); 3860 3861 handle_child_output(r, &output); 3862 } 3863 3864 #[test] 3865 fn test_multiple_network_interfaces() { 3866 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3867 let guest = Guest::new(Box::new(focal)); 3868 3869 let kernel_path = direct_kernel_boot_path(); 3870 3871 let mut child = GuestCommand::new(&guest) 3872 .args(["--cpus", "boot=1"]) 3873 .args(["--memory", "size=512M"]) 3874 .args(["--kernel", kernel_path.to_str().unwrap()]) 3875 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3876 .default_disks() 3877 .args([ 3878 "--net", 3879 guest.default_net_string().as_str(), 3880 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3881 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3882 ]) 3883 .capture_output() 3884 .spawn() 3885 .unwrap(); 3886 3887 let r = std::panic::catch_unwind(|| { 3888 guest.wait_vm_boot(None).unwrap(); 3889 3890 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3891 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3892 3893 // 3 network interfaces + default localhost ==> 4 interfaces 3894 assert_eq!( 3895 guest 3896 .ssh_command("ip -o link | wc -l") 3897 .unwrap() 3898 .trim() 3899 .parse::<u32>() 3900 .unwrap_or_default(), 3901 4 3902 ); 3903 }); 3904 3905 kill_child(&mut child); 3906 let output = child.wait_with_output().unwrap(); 3907 3908 handle_child_output(r, &output); 3909 } 3910 3911 #[test] 3912 #[cfg(target_arch = "aarch64")] 3913 fn test_pmu_on() { 3914 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3915 let guest = Guest::new(Box::new(focal)); 3916 let mut child = GuestCommand::new(&guest) 3917 .args(["--cpus", "boot=1"]) 3918 .args(["--memory", "size=512M"]) 3919 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3920 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3921 .default_disks() 3922 .default_net() 3923 .capture_output() 3924 .spawn() 3925 .unwrap(); 3926 3927 let r = std::panic::catch_unwind(|| { 3928 guest.wait_vm_boot(None).unwrap(); 3929 3930 // Test that PMU exists. 3931 assert_eq!( 3932 guest 3933 .ssh_command(GREP_PMU_IRQ_CMD) 3934 .unwrap() 3935 .trim() 3936 .parse::<u32>() 3937 .unwrap_or_default(), 3938 1 3939 ); 3940 }); 3941 3942 kill_child(&mut child); 3943 let output = child.wait_with_output().unwrap(); 3944 3945 handle_child_output(r, &output); 3946 } 3947 3948 #[test] 3949 fn test_serial_off() { 3950 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3951 let guest = Guest::new(Box::new(focal)); 3952 let mut child = GuestCommand::new(&guest) 3953 .args(["--cpus", "boot=1"]) 3954 .args(["--memory", "size=512M"]) 3955 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3956 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3957 .default_disks() 3958 .default_net() 3959 .args(["--serial", "off"]) 3960 .capture_output() 3961 .spawn() 3962 .unwrap(); 3963 3964 let r = std::panic::catch_unwind(|| { 3965 guest.wait_vm_boot(None).unwrap(); 3966 3967 // Test that there is no ttyS0 3968 assert_eq!( 3969 guest 3970 .ssh_command(GREP_SERIAL_IRQ_CMD) 3971 .unwrap() 3972 .trim() 3973 .parse::<u32>() 3974 .unwrap_or(1), 3975 0 3976 ); 3977 }); 3978 3979 kill_child(&mut child); 3980 let output = child.wait_with_output().unwrap(); 3981 3982 handle_child_output(r, &output); 3983 } 3984 3985 #[test] 3986 fn test_serial_null() { 3987 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3988 let guest = Guest::new(Box::new(focal)); 3989 let mut cmd = GuestCommand::new(&guest); 3990 #[cfg(target_arch = "x86_64")] 3991 let console_str: &str = "console=ttyS0"; 3992 #[cfg(target_arch = "aarch64")] 3993 let console_str: &str = "console=ttyAMA0"; 3994 3995 cmd.args(["--cpus", "boot=1"]) 3996 .args(["--memory", "size=512M"]) 3997 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3998 .args([ 3999 "--cmdline", 4000 DIRECT_KERNEL_BOOT_CMDLINE 4001 .replace("console=hvc0 ", console_str) 4002 .as_str(), 4003 ]) 4004 .default_disks() 4005 .default_net() 4006 .args(["--serial", "null"]) 4007 .args(["--console", "off"]) 4008 .capture_output(); 4009 4010 let mut child = cmd.spawn().unwrap(); 4011 4012 let r = std::panic::catch_unwind(|| { 4013 guest.wait_vm_boot(None).unwrap(); 4014 4015 // Test that there is a ttyS0 4016 assert_eq!( 4017 guest 4018 .ssh_command(GREP_SERIAL_IRQ_CMD) 4019 .unwrap() 4020 .trim() 4021 .parse::<u32>() 4022 .unwrap_or_default(), 4023 1 4024 ); 4025 }); 4026 4027 kill_child(&mut child); 4028 let output = child.wait_with_output().unwrap(); 4029 handle_child_output(r, &output); 4030 4031 let r = std::panic::catch_unwind(|| { 4032 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4033 }); 4034 4035 handle_child_output(r, &output); 4036 } 4037 4038 #[test] 4039 fn test_serial_tty() { 4040 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4041 let guest = Guest::new(Box::new(focal)); 4042 4043 let kernel_path = direct_kernel_boot_path(); 4044 4045 #[cfg(target_arch = "x86_64")] 4046 let console_str: &str = "console=ttyS0"; 4047 #[cfg(target_arch = "aarch64")] 4048 let console_str: &str = "console=ttyAMA0"; 4049 4050 let mut child = GuestCommand::new(&guest) 4051 .args(["--cpus", "boot=1"]) 4052 .args(["--memory", "size=512M"]) 4053 .args(["--kernel", kernel_path.to_str().unwrap()]) 4054 .args([ 4055 "--cmdline", 4056 DIRECT_KERNEL_BOOT_CMDLINE 4057 .replace("console=hvc0 ", console_str) 4058 .as_str(), 4059 ]) 4060 .default_disks() 4061 .default_net() 4062 .args(["--serial", "tty"]) 4063 .args(["--console", "off"]) 4064 .capture_output() 4065 .spawn() 4066 .unwrap(); 4067 4068 let r = std::panic::catch_unwind(|| { 4069 guest.wait_vm_boot(None).unwrap(); 4070 4071 // Test that there is a ttyS0 4072 assert_eq!( 4073 guest 4074 .ssh_command(GREP_SERIAL_IRQ_CMD) 4075 .unwrap() 4076 .trim() 4077 .parse::<u32>() 4078 .unwrap_or_default(), 4079 1 4080 ); 4081 }); 4082 4083 // This sleep is needed to wait for the login prompt 4084 thread::sleep(std::time::Duration::new(2, 0)); 4085 4086 kill_child(&mut child); 4087 let output = child.wait_with_output().unwrap(); 4088 handle_child_output(r, &output); 4089 4090 let r = std::panic::catch_unwind(|| { 4091 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4092 }); 4093 4094 handle_child_output(r, &output); 4095 } 4096 4097 #[test] 4098 fn test_serial_file() { 4099 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4100 let guest = Guest::new(Box::new(focal)); 4101 4102 let serial_path = guest.tmp_dir.as_path().join("serial-output"); 4103 #[cfg(target_arch = "x86_64")] 4104 let console_str: &str = "console=ttyS0"; 4105 #[cfg(target_arch = "aarch64")] 4106 let console_str: &str = "console=ttyAMA0"; 4107 4108 let mut child = GuestCommand::new(&guest) 4109 .args(["--cpus", "boot=1"]) 4110 .args(["--memory", "size=512M"]) 4111 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4112 .args([ 4113 "--cmdline", 4114 DIRECT_KERNEL_BOOT_CMDLINE 4115 .replace("console=hvc0 ", console_str) 4116 .as_str(), 4117 ]) 4118 .default_disks() 4119 .default_net() 4120 .args([ 4121 "--serial", 4122 format!("file={}", serial_path.to_str().unwrap()).as_str(), 4123 ]) 4124 .capture_output() 4125 .spawn() 4126 .unwrap(); 4127 4128 let r = std::panic::catch_unwind(|| { 4129 guest.wait_vm_boot(None).unwrap(); 4130 4131 // Test that there is a ttyS0 4132 assert_eq!( 4133 guest 4134 .ssh_command(GREP_SERIAL_IRQ_CMD) 4135 .unwrap() 4136 .trim() 4137 .parse::<u32>() 4138 .unwrap_or_default(), 4139 1 4140 ); 4141 4142 guest.ssh_command("sudo shutdown -h now").unwrap(); 4143 }); 4144 4145 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4146 kill_child(&mut child); 4147 let output = child.wait_with_output().unwrap(); 4148 handle_child_output(r, &output); 4149 4150 let r = std::panic::catch_unwind(|| { 4151 // Check that the cloud-hypervisor binary actually terminated 4152 assert!(output.status.success()); 4153 4154 // Do this check after shutdown of the VM as an easy way to ensure 4155 // all writes are flushed to disk 4156 let mut f = std::fs::File::open(serial_path).unwrap(); 4157 let mut buf = String::new(); 4158 f.read_to_string(&mut buf).unwrap(); 4159 assert!(buf.contains(CONSOLE_TEST_STRING)); 4160 }); 4161 4162 handle_child_output(r, &output); 4163 } 4164 4165 #[test] 4166 fn test_pty_interaction() { 4167 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4168 let guest = Guest::new(Box::new(focal)); 4169 let api_socket = temp_api_path(&guest.tmp_dir); 4170 let serial_option = if cfg!(target_arch = "x86_64") { 4171 " console=ttyS0" 4172 } else { 4173 " console=ttyAMA0" 4174 }; 4175 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4176 4177 let mut child = GuestCommand::new(&guest) 4178 .args(["--cpus", "boot=1"]) 4179 .args(["--memory", "size=512M"]) 4180 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4181 .args(["--cmdline", &cmdline]) 4182 .default_disks() 4183 .default_net() 4184 .args(["--serial", "null"]) 4185 .args(["--console", "pty"]) 4186 .args(["--api-socket", &api_socket]) 4187 .spawn() 4188 .unwrap(); 4189 4190 let r = std::panic::catch_unwind(|| { 4191 guest.wait_vm_boot(None).unwrap(); 4192 // Get pty fd for console 4193 let console_path = get_pty_path(&api_socket, "console"); 4194 _test_pty_interaction(console_path); 4195 4196 guest.ssh_command("sudo shutdown -h now").unwrap(); 4197 }); 4198 4199 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4200 let _ = child.kill(); 4201 let output = child.wait_with_output().unwrap(); 4202 handle_child_output(r, &output); 4203 4204 let r = std::panic::catch_unwind(|| { 4205 // Check that the cloud-hypervisor binary actually terminated 4206 assert!(output.status.success()) 4207 }); 4208 handle_child_output(r, &output); 4209 } 4210 4211 #[test] 4212 fn test_serial_socket_interaction() { 4213 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4214 let guest = Guest::new(Box::new(focal)); 4215 let serial_socket = guest.tmp_dir.as_path().join("serial.socket"); 4216 let serial_socket_pty = guest.tmp_dir.as_path().join("serial.pty"); 4217 let serial_option = if cfg!(target_arch = "x86_64") { 4218 " console=ttyS0" 4219 } else { 4220 " console=ttyAMA0" 4221 }; 4222 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4223 4224 let mut child = GuestCommand::new(&guest) 4225 .args(["--cpus", "boot=1"]) 4226 .args(["--memory", "size=512M"]) 4227 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4228 .args(["--cmdline", &cmdline]) 4229 .default_disks() 4230 .default_net() 4231 .args(["--console", "null"]) 4232 .args([ 4233 "--serial", 4234 format!("socket={}", serial_socket.to_str().unwrap()).as_str(), 4235 ]) 4236 .spawn() 4237 .unwrap(); 4238 4239 let _ = std::panic::catch_unwind(|| { 4240 guest.wait_vm_boot(None).unwrap(); 4241 }); 4242 4243 let mut socat_command = Command::new("socat"); 4244 let socat_args = [ 4245 &format!("pty,link={},raw", serial_socket_pty.display()), 4246 &format!("UNIX-CONNECT:{}", serial_socket.display()), 4247 ]; 4248 socat_command.args(socat_args); 4249 4250 let mut socat_child = socat_command.spawn().unwrap(); 4251 thread::sleep(std::time::Duration::new(1, 0)); 4252 4253 let _ = std::panic::catch_unwind(|| { 4254 _test_pty_interaction(serial_socket_pty); 4255 }); 4256 4257 let _ = socat_child.kill(); 4258 4259 let r = std::panic::catch_unwind(|| { 4260 guest.ssh_command("sudo shutdown -h now").unwrap(); 4261 }); 4262 4263 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4264 kill_child(&mut child); 4265 let output = child.wait_with_output().unwrap(); 4266 handle_child_output(r, &output); 4267 4268 let r = std::panic::catch_unwind(|| { 4269 // Check that the cloud-hypervisor binary actually terminated 4270 if !output.status.success() { 4271 panic!( 4272 "Cloud Hypervisor process failed to terminate gracefully: {:?}", 4273 output.status 4274 ); 4275 } 4276 }); 4277 handle_child_output(r, &output); 4278 } 4279 4280 #[test] 4281 fn test_virtio_console() { 4282 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4283 let guest = Guest::new(Box::new(focal)); 4284 4285 let kernel_path = direct_kernel_boot_path(); 4286 4287 let mut child = GuestCommand::new(&guest) 4288 .args(["--cpus", "boot=1"]) 4289 .args(["--memory", "size=512M"]) 4290 .args(["--kernel", kernel_path.to_str().unwrap()]) 4291 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4292 .default_disks() 4293 .default_net() 4294 .args(["--console", "tty"]) 4295 .args(["--serial", "null"]) 4296 .capture_output() 4297 .spawn() 4298 .unwrap(); 4299 4300 let text = String::from("On a branch floating down river a cricket, singing."); 4301 let cmd = format!("echo {text} | sudo tee /dev/hvc0"); 4302 4303 let r = std::panic::catch_unwind(|| { 4304 guest.wait_vm_boot(None).unwrap(); 4305 4306 assert!(guest 4307 .does_device_vendor_pair_match("0x1043", "0x1af4") 4308 .unwrap_or_default()); 4309 4310 guest.ssh_command(&cmd).unwrap(); 4311 }); 4312 4313 kill_child(&mut child); 4314 let output = child.wait_with_output().unwrap(); 4315 handle_child_output(r, &output); 4316 4317 let r = std::panic::catch_unwind(|| { 4318 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 4319 }); 4320 4321 handle_child_output(r, &output); 4322 } 4323 4324 #[test] 4325 fn test_console_file() { 4326 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4327 let guest = Guest::new(Box::new(focal)); 4328 4329 let console_path = guest.tmp_dir.as_path().join("console-output"); 4330 let mut child = GuestCommand::new(&guest) 4331 .args(["--cpus", "boot=1"]) 4332 .args(["--memory", "size=512M"]) 4333 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4334 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4335 .default_disks() 4336 .default_net() 4337 .args([ 4338 "--console", 4339 format!("file={}", console_path.to_str().unwrap()).as_str(), 4340 ]) 4341 .capture_output() 4342 .spawn() 4343 .unwrap(); 4344 4345 guest.wait_vm_boot(None).unwrap(); 4346 4347 guest.ssh_command("sudo shutdown -h now").unwrap(); 4348 4349 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4350 kill_child(&mut child); 4351 let output = child.wait_with_output().unwrap(); 4352 4353 let r = std::panic::catch_unwind(|| { 4354 // Check that the cloud-hypervisor binary actually terminated 4355 assert!(output.status.success()); 4356 4357 // Do this check after shutdown of the VM as an easy way to ensure 4358 // all writes are flushed to disk 4359 let mut f = std::fs::File::open(console_path).unwrap(); 4360 let mut buf = String::new(); 4361 f.read_to_string(&mut buf).unwrap(); 4362 4363 if !buf.contains(CONSOLE_TEST_STRING) { 4364 eprintln!( 4365 "\n\n==== Console file output ====\n\n{buf}\n\n==== End console file output ====" 4366 ); 4367 } 4368 assert!(buf.contains(CONSOLE_TEST_STRING)); 4369 }); 4370 4371 handle_child_output(r, &output); 4372 } 4373 4374 #[test] 4375 #[cfg(target_arch = "x86_64")] 4376 #[cfg(not(feature = "mshv"))] 4377 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 4378 // backed networking interfaces, bound through a simple bridge on the host. 4379 // So if the nested cloud-hypervisor succeeds in getting a directly 4380 // assigned interface from its cloud-hypervisor host, we should be able to 4381 // ssh into it, and verify that it's running with the right kernel command 4382 // line (We tag the command line from cloud-hypervisor for that purpose). 4383 // The third device is added to validate that hotplug works correctly since 4384 // it is being added to the L2 VM through hotplugging mechanism. 4385 // Also, we pass-through a virtio-blk device to the L2 VM to test the 32-bit 4386 // vfio device support 4387 fn test_vfio() { 4388 setup_vfio_network_interfaces(); 4389 4390 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 4391 let guest = Guest::new_from_ip_range(Box::new(jammy), "172.18", 0); 4392 4393 let mut workload_path = dirs::home_dir().unwrap(); 4394 workload_path.push("workloads"); 4395 4396 let kernel_path = direct_kernel_boot_path(); 4397 4398 let mut vfio_path = workload_path.clone(); 4399 vfio_path.push("vfio"); 4400 4401 let mut cloud_init_vfio_base_path = vfio_path.clone(); 4402 cloud_init_vfio_base_path.push("cloudinit.img"); 4403 4404 // We copy our cloudinit into the vfio mount point, for the nested 4405 // cloud-hypervisor guest to use. 4406 rate_limited_copy( 4407 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 4408 &cloud_init_vfio_base_path, 4409 ) 4410 .expect("copying of cloud-init disk failed"); 4411 4412 let mut vfio_disk_path = workload_path.clone(); 4413 vfio_disk_path.push("vfio.img"); 4414 4415 // Create the vfio disk image 4416 let output = Command::new("mkfs.ext4") 4417 .arg("-d") 4418 .arg(vfio_path.to_str().unwrap()) 4419 .arg(vfio_disk_path.to_str().unwrap()) 4420 .arg("2g") 4421 .output() 4422 .unwrap(); 4423 if !output.status.success() { 4424 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 4425 panic!("mkfs.ext4 command generated an error"); 4426 } 4427 4428 let mut blk_file_path = workload_path; 4429 blk_file_path.push("blk.img"); 4430 4431 let vfio_tap0 = "vfio-tap0"; 4432 let vfio_tap1 = "vfio-tap1"; 4433 let vfio_tap2 = "vfio-tap2"; 4434 let vfio_tap3 = "vfio-tap3"; 4435 4436 let mut child = GuestCommand::new(&guest) 4437 .args(["--cpus", "boot=4"]) 4438 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 4439 .args(["--kernel", kernel_path.to_str().unwrap()]) 4440 .args([ 4441 "--disk", 4442 format!( 4443 "path={}", 4444 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4445 ) 4446 .as_str(), 4447 format!( 4448 "path={}", 4449 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4450 ) 4451 .as_str(), 4452 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 4453 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 4454 ]) 4455 .args([ 4456 "--cmdline", 4457 format!( 4458 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 4459 ) 4460 .as_str(), 4461 ]) 4462 .args([ 4463 "--net", 4464 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 4465 format!( 4466 "tap={},mac={},iommu=on", 4467 vfio_tap1, guest.network.l2_guest_mac1 4468 ) 4469 .as_str(), 4470 format!( 4471 "tap={},mac={},iommu=on", 4472 vfio_tap2, guest.network.l2_guest_mac2 4473 ) 4474 .as_str(), 4475 format!( 4476 "tap={},mac={},iommu=on", 4477 vfio_tap3, guest.network.l2_guest_mac3 4478 ) 4479 .as_str(), 4480 ]) 4481 .capture_output() 4482 .spawn() 4483 .unwrap(); 4484 4485 thread::sleep(std::time::Duration::new(30, 0)); 4486 4487 let r = std::panic::catch_unwind(|| { 4488 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 4489 thread::sleep(std::time::Duration::new(120, 0)); 4490 4491 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 4492 // added to its kernel command line. 4493 // Let's ssh into it and verify that it's there. If it is it means 4494 // we're in the right guest (The L2 one) because the QEMU L1 guest 4495 // does not have this command line tag. 4496 assert!(check_matched_lines_count( 4497 guest.ssh_command_l2_1("cat /proc/cmdline").unwrap().trim(), 4498 vec!["VFIOTAG"], 4499 1 4500 )); 4501 4502 // Let's also verify from the second virtio-net device passed to 4503 // the L2 VM. 4504 assert!(check_matched_lines_count( 4505 guest.ssh_command_l2_2("cat /proc/cmdline").unwrap().trim(), 4506 vec!["VFIOTAG"], 4507 1 4508 )); 4509 4510 // Check the amount of PCI devices appearing in L2 VM. 4511 assert!(check_lines_count( 4512 guest 4513 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4514 .unwrap() 4515 .trim(), 4516 8 4517 )); 4518 4519 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 4520 assert!(check_matched_lines_count( 4521 guest.ssh_command_l2_1("lsblk").unwrap().trim(), 4522 vec!["vdc", "16M"], 4523 1 4524 )); 4525 4526 // Hotplug an extra virtio-net device through L2 VM. 4527 guest 4528 .ssh_command_l1( 4529 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 4530 ) 4531 .unwrap(); 4532 guest 4533 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 4534 .unwrap(); 4535 let vfio_hotplug_output = guest 4536 .ssh_command_l1( 4537 "sudo /mnt/ch-remote \ 4538 --api-socket=/tmp/ch_api.sock \ 4539 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 4540 ) 4541 .unwrap(); 4542 assert!(check_matched_lines_count( 4543 vfio_hotplug_output.trim(), 4544 vec!["{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}"], 4545 1 4546 )); 4547 4548 thread::sleep(std::time::Duration::new(10, 0)); 4549 4550 // Let's also verify from the third virtio-net device passed to 4551 // the L2 VM. This third device has been hotplugged through the L2 4552 // VM, so this is our way to validate hotplug works for VFIO PCI. 4553 assert!(check_matched_lines_count( 4554 guest.ssh_command_l2_3("cat /proc/cmdline").unwrap().trim(), 4555 vec!["VFIOTAG"], 4556 1 4557 )); 4558 4559 // Check the amount of PCI devices appearing in L2 VM. 4560 // There should be one more device than before, raising the count 4561 // up to 9 PCI devices. 4562 assert!(check_lines_count( 4563 guest 4564 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4565 .unwrap() 4566 .trim(), 4567 9 4568 )); 4569 4570 // Let's now verify that we can correctly remove the virtio-net 4571 // device through the "remove-device" command responsible for 4572 // unplugging VFIO devices. 4573 guest 4574 .ssh_command_l1( 4575 "sudo /mnt/ch-remote \ 4576 --api-socket=/tmp/ch_api.sock \ 4577 remove-device vfio123", 4578 ) 4579 .unwrap(); 4580 thread::sleep(std::time::Duration::new(10, 0)); 4581 4582 // Check the amount of PCI devices appearing in L2 VM is back down 4583 // to 8 devices. 4584 assert!(check_lines_count( 4585 guest 4586 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4587 .unwrap() 4588 .trim(), 4589 8 4590 )); 4591 4592 // Perform memory hotplug in L2 and validate the memory is showing 4593 // up as expected. In order to check, we will use the virtio-net 4594 // device already passed through L2 as a VFIO device, this will 4595 // verify that VFIO devices are functional with memory hotplug. 4596 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 4597 guest 4598 .ssh_command_l2_1( 4599 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 4600 ) 4601 .unwrap(); 4602 guest 4603 .ssh_command_l1( 4604 "sudo /mnt/ch-remote \ 4605 --api-socket=/tmp/ch_api.sock \ 4606 resize --memory=1073741824", 4607 ) 4608 .unwrap(); 4609 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 4610 }); 4611 4612 kill_child(&mut child); 4613 let output = child.wait_with_output().unwrap(); 4614 4615 cleanup_vfio_network_interfaces(); 4616 4617 handle_child_output(r, &output); 4618 } 4619 4620 #[test] 4621 fn test_direct_kernel_boot_noacpi() { 4622 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4623 let guest = Guest::new(Box::new(focal)); 4624 4625 let kernel_path = direct_kernel_boot_path(); 4626 4627 let mut child = GuestCommand::new(&guest) 4628 .args(["--cpus", "boot=1"]) 4629 .args(["--memory", "size=512M"]) 4630 .args(["--kernel", kernel_path.to_str().unwrap()]) 4631 .args([ 4632 "--cmdline", 4633 format!("{DIRECT_KERNEL_BOOT_CMDLINE} acpi=off").as_str(), 4634 ]) 4635 .default_disks() 4636 .default_net() 4637 .capture_output() 4638 .spawn() 4639 .unwrap(); 4640 4641 let r = std::panic::catch_unwind(|| { 4642 guest.wait_vm_boot(None).unwrap(); 4643 4644 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4645 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4646 }); 4647 4648 kill_child(&mut child); 4649 let output = child.wait_with_output().unwrap(); 4650 4651 handle_child_output(r, &output); 4652 } 4653 4654 #[test] 4655 fn test_virtio_vsock() { 4656 _test_virtio_vsock(false) 4657 } 4658 4659 #[test] 4660 fn test_virtio_vsock_hotplug() { 4661 _test_virtio_vsock(true); 4662 } 4663 4664 #[test] 4665 fn test_api_http_shutdown() { 4666 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4667 let guest = Guest::new(Box::new(focal)); 4668 4669 _test_api_shutdown(TargetApi::new_http_api(&guest.tmp_dir), guest) 4670 } 4671 4672 #[test] 4673 fn test_api_http_delete() { 4674 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4675 let guest = Guest::new(Box::new(focal)); 4676 4677 _test_api_delete(TargetApi::new_http_api(&guest.tmp_dir), guest); 4678 } 4679 4680 #[test] 4681 fn test_api_http_pause_resume() { 4682 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4683 let guest = Guest::new(Box::new(focal)); 4684 4685 _test_api_pause_resume(TargetApi::new_http_api(&guest.tmp_dir), guest) 4686 } 4687 4688 #[test] 4689 fn test_api_http_create_boot() { 4690 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4691 let guest = Guest::new(Box::new(focal)); 4692 4693 _test_api_create_boot(TargetApi::new_http_api(&guest.tmp_dir), guest) 4694 } 4695 4696 #[test] 4697 fn test_virtio_iommu() { 4698 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4699 } 4700 4701 #[test] 4702 // We cannot force the software running in the guest to reprogram the BAR 4703 // with some different addresses, but we have a reliable way of testing it 4704 // with a standard Linux kernel. 4705 // By removing a device from the PCI tree, and then rescanning the tree, 4706 // Linux consistently chooses to reorganize the PCI device BARs to other 4707 // locations in the guest address space. 4708 // This test creates a dedicated PCI network device to be checked as being 4709 // properly probed first, then removing it, and adding it again by doing a 4710 // rescan. 4711 fn test_pci_bar_reprogramming() { 4712 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4713 let guest = Guest::new(Box::new(focal)); 4714 4715 #[cfg(target_arch = "x86_64")] 4716 let kernel_path = direct_kernel_boot_path(); 4717 #[cfg(target_arch = "aarch64")] 4718 let kernel_path = edk2_path(); 4719 4720 let mut child = GuestCommand::new(&guest) 4721 .args(["--cpus", "boot=1"]) 4722 .args(["--memory", "size=512M"]) 4723 .args(["--kernel", kernel_path.to_str().unwrap()]) 4724 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4725 .default_disks() 4726 .args([ 4727 "--net", 4728 guest.default_net_string().as_str(), 4729 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4730 ]) 4731 .capture_output() 4732 .spawn() 4733 .unwrap(); 4734 4735 let r = std::panic::catch_unwind(|| { 4736 guest.wait_vm_boot(None).unwrap(); 4737 4738 // 2 network interfaces + default localhost ==> 3 interfaces 4739 assert_eq!( 4740 guest 4741 .ssh_command("ip -o link | wc -l") 4742 .unwrap() 4743 .trim() 4744 .parse::<u32>() 4745 .unwrap_or_default(), 4746 3 4747 ); 4748 4749 let init_bar_addr = guest 4750 .ssh_command( 4751 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4752 ) 4753 .unwrap(); 4754 4755 // Remove the PCI device 4756 guest 4757 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4758 .unwrap(); 4759 4760 // Only 1 network interface left + default localhost ==> 2 interfaces 4761 assert_eq!( 4762 guest 4763 .ssh_command("ip -o link | wc -l") 4764 .unwrap() 4765 .trim() 4766 .parse::<u32>() 4767 .unwrap_or_default(), 4768 2 4769 ); 4770 4771 // Remove the PCI device 4772 guest 4773 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4774 .unwrap(); 4775 4776 // Back to 2 network interface + default localhost ==> 3 interfaces 4777 assert_eq!( 4778 guest 4779 .ssh_command("ip -o link | wc -l") 4780 .unwrap() 4781 .trim() 4782 .parse::<u32>() 4783 .unwrap_or_default(), 4784 3 4785 ); 4786 4787 let new_bar_addr = guest 4788 .ssh_command( 4789 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4790 ) 4791 .unwrap(); 4792 4793 // Let's compare the BAR addresses for our virtio-net device. 4794 // They should be different as we expect the BAR reprogramming 4795 // to have happened. 4796 assert_ne!(init_bar_addr, new_bar_addr); 4797 }); 4798 4799 kill_child(&mut child); 4800 let output = child.wait_with_output().unwrap(); 4801 4802 handle_child_output(r, &output); 4803 } 4804 4805 #[test] 4806 fn test_memory_mergeable_off() { 4807 test_memory_mergeable(false) 4808 } 4809 4810 #[test] 4811 #[cfg(target_arch = "x86_64")] 4812 fn test_cpu_hotplug() { 4813 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4814 let guest = Guest::new(Box::new(focal)); 4815 let api_socket = temp_api_path(&guest.tmp_dir); 4816 4817 let kernel_path = direct_kernel_boot_path(); 4818 4819 let mut child = GuestCommand::new(&guest) 4820 .args(["--cpus", "boot=2,max=4"]) 4821 .args(["--memory", "size=512M"]) 4822 .args(["--kernel", kernel_path.to_str().unwrap()]) 4823 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4824 .default_disks() 4825 .default_net() 4826 .args(["--api-socket", &api_socket]) 4827 .capture_output() 4828 .spawn() 4829 .unwrap(); 4830 4831 let r = std::panic::catch_unwind(|| { 4832 guest.wait_vm_boot(None).unwrap(); 4833 4834 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4835 4836 // Resize the VM 4837 let desired_vcpus = 4; 4838 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4839 4840 guest 4841 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4842 .unwrap(); 4843 guest 4844 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4845 .unwrap(); 4846 thread::sleep(std::time::Duration::new(10, 0)); 4847 assert_eq!( 4848 guest.get_cpu_count().unwrap_or_default(), 4849 u32::from(desired_vcpus) 4850 ); 4851 4852 guest.reboot_linux(0, None); 4853 4854 assert_eq!( 4855 guest.get_cpu_count().unwrap_or_default(), 4856 u32::from(desired_vcpus) 4857 ); 4858 4859 // Resize the VM 4860 let desired_vcpus = 2; 4861 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4862 4863 thread::sleep(std::time::Duration::new(10, 0)); 4864 assert_eq!( 4865 guest.get_cpu_count().unwrap_or_default(), 4866 u32::from(desired_vcpus) 4867 ); 4868 4869 // Resize the VM back up to 4 4870 let desired_vcpus = 4; 4871 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4872 4873 guest 4874 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4875 .unwrap(); 4876 guest 4877 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4878 .unwrap(); 4879 thread::sleep(std::time::Duration::new(10, 0)); 4880 assert_eq!( 4881 guest.get_cpu_count().unwrap_or_default(), 4882 u32::from(desired_vcpus) 4883 ); 4884 }); 4885 4886 kill_child(&mut child); 4887 let output = child.wait_with_output().unwrap(); 4888 4889 handle_child_output(r, &output); 4890 } 4891 4892 #[test] 4893 fn test_memory_hotplug() { 4894 #[cfg(target_arch = "aarch64")] 4895 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4896 #[cfg(target_arch = "x86_64")] 4897 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4898 let focal = UbuntuDiskConfig::new(focal_image); 4899 let guest = Guest::new(Box::new(focal)); 4900 let api_socket = temp_api_path(&guest.tmp_dir); 4901 4902 #[cfg(target_arch = "aarch64")] 4903 let kernel_path = edk2_path(); 4904 #[cfg(target_arch = "x86_64")] 4905 let kernel_path = direct_kernel_boot_path(); 4906 4907 let mut child = GuestCommand::new(&guest) 4908 .args(["--cpus", "boot=2,max=4"]) 4909 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4910 .args(["--kernel", kernel_path.to_str().unwrap()]) 4911 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4912 .default_disks() 4913 .default_net() 4914 .args(["--balloon", "size=0"]) 4915 .args(["--api-socket", &api_socket]) 4916 .capture_output() 4917 .spawn() 4918 .unwrap(); 4919 4920 let r = std::panic::catch_unwind(|| { 4921 guest.wait_vm_boot(None).unwrap(); 4922 4923 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4924 4925 guest.enable_memory_hotplug(); 4926 4927 // Add RAM to the VM 4928 let desired_ram = 1024 << 20; 4929 resize_command(&api_socket, None, Some(desired_ram), None, None); 4930 4931 thread::sleep(std::time::Duration::new(10, 0)); 4932 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4933 4934 // Use balloon to remove RAM from the VM 4935 let desired_balloon = 512 << 20; 4936 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4937 4938 thread::sleep(std::time::Duration::new(10, 0)); 4939 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4940 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4941 4942 guest.reboot_linux(0, None); 4943 4944 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4945 4946 // Use balloon add RAM to the VM 4947 let desired_balloon = 0; 4948 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4949 4950 thread::sleep(std::time::Duration::new(10, 0)); 4951 4952 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4953 4954 guest.enable_memory_hotplug(); 4955 4956 // Add RAM to the VM 4957 let desired_ram = 2048 << 20; 4958 resize_command(&api_socket, None, Some(desired_ram), None, None); 4959 4960 thread::sleep(std::time::Duration::new(10, 0)); 4961 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4962 4963 // Remove RAM to the VM (only applies after reboot) 4964 let desired_ram = 1024 << 20; 4965 resize_command(&api_socket, None, Some(desired_ram), None, None); 4966 4967 guest.reboot_linux(1, None); 4968 4969 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4970 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4971 }); 4972 4973 kill_child(&mut child); 4974 let output = child.wait_with_output().unwrap(); 4975 4976 handle_child_output(r, &output); 4977 } 4978 4979 #[test] 4980 #[cfg(not(feature = "mshv"))] 4981 fn test_virtio_mem() { 4982 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4983 let guest = Guest::new(Box::new(focal)); 4984 let api_socket = temp_api_path(&guest.tmp_dir); 4985 4986 let kernel_path = direct_kernel_boot_path(); 4987 4988 let mut child = GuestCommand::new(&guest) 4989 .args(["--cpus", "boot=2,max=4"]) 4990 .args([ 4991 "--memory", 4992 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4993 ]) 4994 .args(["--kernel", kernel_path.to_str().unwrap()]) 4995 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4996 .default_disks() 4997 .default_net() 4998 .args(["--api-socket", &api_socket]) 4999 .capture_output() 5000 .spawn() 5001 .unwrap(); 5002 5003 let r = std::panic::catch_unwind(|| { 5004 guest.wait_vm_boot(None).unwrap(); 5005 5006 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5007 5008 guest.enable_memory_hotplug(); 5009 5010 // Add RAM to the VM 5011 let desired_ram = 1024 << 20; 5012 resize_command(&api_socket, None, Some(desired_ram), None, None); 5013 5014 thread::sleep(std::time::Duration::new(10, 0)); 5015 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5016 5017 // Add RAM to the VM 5018 let desired_ram = 2048 << 20; 5019 resize_command(&api_socket, None, Some(desired_ram), None, None); 5020 5021 thread::sleep(std::time::Duration::new(10, 0)); 5022 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 5023 5024 // Remove RAM from the VM 5025 let desired_ram = 1024 << 20; 5026 resize_command(&api_socket, None, Some(desired_ram), None, None); 5027 5028 thread::sleep(std::time::Duration::new(10, 0)); 5029 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5030 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5031 5032 guest.reboot_linux(0, None); 5033 5034 // Check the amount of memory after reboot is 1GiB 5035 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5036 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5037 5038 // Check we can still resize to 512MiB 5039 let desired_ram = 512 << 20; 5040 resize_command(&api_socket, None, Some(desired_ram), None, None); 5041 thread::sleep(std::time::Duration::new(10, 0)); 5042 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5043 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 5044 }); 5045 5046 kill_child(&mut child); 5047 let output = child.wait_with_output().unwrap(); 5048 5049 handle_child_output(r, &output); 5050 } 5051 5052 #[test] 5053 #[cfg(target_arch = "x86_64")] 5054 #[cfg(not(feature = "mshv"))] 5055 // Test both vCPU and memory resizing together 5056 fn test_resize() { 5057 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5058 let guest = Guest::new(Box::new(focal)); 5059 let api_socket = temp_api_path(&guest.tmp_dir); 5060 5061 let kernel_path = direct_kernel_boot_path(); 5062 5063 let mut child = GuestCommand::new(&guest) 5064 .args(["--cpus", "boot=2,max=4"]) 5065 .args(["--memory", "size=512M,hotplug_size=8192M"]) 5066 .args(["--kernel", kernel_path.to_str().unwrap()]) 5067 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5068 .default_disks() 5069 .default_net() 5070 .args(["--api-socket", &api_socket]) 5071 .capture_output() 5072 .spawn() 5073 .unwrap(); 5074 5075 let r = std::panic::catch_unwind(|| { 5076 guest.wait_vm_boot(None).unwrap(); 5077 5078 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 5079 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5080 5081 guest.enable_memory_hotplug(); 5082 5083 // Resize the VM 5084 let desired_vcpus = 4; 5085 let desired_ram = 1024 << 20; 5086 resize_command( 5087 &api_socket, 5088 Some(desired_vcpus), 5089 Some(desired_ram), 5090 None, 5091 None, 5092 ); 5093 5094 guest 5095 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 5096 .unwrap(); 5097 guest 5098 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 5099 .unwrap(); 5100 thread::sleep(std::time::Duration::new(10, 0)); 5101 assert_eq!( 5102 guest.get_cpu_count().unwrap_or_default(), 5103 u32::from(desired_vcpus) 5104 ); 5105 5106 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5107 }); 5108 5109 kill_child(&mut child); 5110 let output = child.wait_with_output().unwrap(); 5111 5112 handle_child_output(r, &output); 5113 } 5114 5115 #[test] 5116 fn test_memory_overhead() { 5117 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5118 let guest = Guest::new(Box::new(focal)); 5119 5120 let kernel_path = direct_kernel_boot_path(); 5121 5122 let guest_memory_size_kb = 512 * 1024; 5123 5124 let mut child = GuestCommand::new(&guest) 5125 .args(["--cpus", "boot=1"]) 5126 .args(["--memory", format!("size={guest_memory_size_kb}K").as_str()]) 5127 .args(["--kernel", kernel_path.to_str().unwrap()]) 5128 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5129 .default_net() 5130 .default_disks() 5131 .capture_output() 5132 .spawn() 5133 .unwrap(); 5134 5135 guest.wait_vm_boot(None).unwrap(); 5136 5137 let r = std::panic::catch_unwind(|| { 5138 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 5139 eprintln!("Guest memory overhead: {overhead} vs {MAXIMUM_VMM_OVERHEAD_KB}"); 5140 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 5141 }); 5142 5143 kill_child(&mut child); 5144 let output = child.wait_with_output().unwrap(); 5145 5146 handle_child_output(r, &output); 5147 } 5148 5149 #[test] 5150 #[cfg(target_arch = "x86_64")] 5151 // This test runs a guest with Landlock enabled and hotplugs a new disk. As 5152 // the path for the hotplug disk is not pre-added to Landlock rules, this 5153 // the test will result in a failure. 5154 fn test_landlock() { 5155 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5156 let guest = Guest::new(Box::new(focal)); 5157 5158 #[cfg(target_arch = "x86_64")] 5159 let kernel_path = direct_kernel_boot_path(); 5160 #[cfg(target_arch = "aarch64")] 5161 let kernel_path = edk2_path(); 5162 5163 let api_socket = temp_api_path(&guest.tmp_dir); 5164 5165 let mut child = GuestCommand::new(&guest) 5166 .args(["--api-socket", &api_socket]) 5167 .args(["--cpus", "boot=1"]) 5168 .args(["--memory", "size=512M"]) 5169 .args(["--kernel", kernel_path.to_str().unwrap()]) 5170 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5171 .args(["--landlock"]) 5172 .default_disks() 5173 .default_net() 5174 .capture_output() 5175 .spawn() 5176 .unwrap(); 5177 5178 let r = std::panic::catch_unwind(|| { 5179 guest.wait_vm_boot(None).unwrap(); 5180 5181 // Check /dev/vdc is not there 5182 assert_eq!( 5183 guest 5184 .ssh_command("lsblk | grep -c vdc.*16M || true") 5185 .unwrap() 5186 .trim() 5187 .parse::<u32>() 5188 .unwrap_or(1), 5189 0 5190 ); 5191 5192 // Now let's add the extra disk. 5193 let mut blk_file_path = dirs::home_dir().unwrap(); 5194 blk_file_path.push("workloads"); 5195 blk_file_path.push("blk.img"); 5196 // As the path to the hotplug disk is not pre-added, this remote 5197 // command will fail. 5198 assert!(!remote_command( 5199 &api_socket, 5200 "add-disk", 5201 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5202 )); 5203 }); 5204 5205 let _ = child.kill(); 5206 let output = child.wait_with_output().unwrap(); 5207 5208 handle_child_output(r, &output); 5209 } 5210 5211 fn _test_disk_hotplug(landlock_enabled: bool) { 5212 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5213 let guest = Guest::new(Box::new(focal)); 5214 5215 #[cfg(target_arch = "x86_64")] 5216 let kernel_path = direct_kernel_boot_path(); 5217 #[cfg(target_arch = "aarch64")] 5218 let kernel_path = edk2_path(); 5219 5220 let api_socket = temp_api_path(&guest.tmp_dir); 5221 5222 let mut blk_file_path = dirs::home_dir().unwrap(); 5223 blk_file_path.push("workloads"); 5224 blk_file_path.push("blk.img"); 5225 5226 let mut cmd = GuestCommand::new(&guest); 5227 if landlock_enabled { 5228 cmd.args(["--landlock"]).args([ 5229 "--landlock-rules", 5230 format!("path={:?},access=rw", blk_file_path).as_str(), 5231 ]); 5232 } 5233 5234 cmd.args(["--api-socket", &api_socket]) 5235 .args(["--cpus", "boot=1"]) 5236 .args(["--memory", "size=512M"]) 5237 .args(["--kernel", kernel_path.to_str().unwrap()]) 5238 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5239 .default_disks() 5240 .default_net() 5241 .capture_output(); 5242 5243 let mut child = cmd.spawn().unwrap(); 5244 5245 let r = std::panic::catch_unwind(|| { 5246 guest.wait_vm_boot(None).unwrap(); 5247 5248 // Check /dev/vdc is not there 5249 assert_eq!( 5250 guest 5251 .ssh_command("lsblk | grep -c vdc.*16M || true") 5252 .unwrap() 5253 .trim() 5254 .parse::<u32>() 5255 .unwrap_or(1), 5256 0 5257 ); 5258 5259 // Now let's add the extra disk. 5260 let (cmd_success, cmd_output) = remote_command_w_output( 5261 &api_socket, 5262 "add-disk", 5263 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5264 ); 5265 assert!(cmd_success); 5266 assert!(String::from_utf8_lossy(&cmd_output) 5267 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5268 5269 thread::sleep(std::time::Duration::new(10, 0)); 5270 5271 // Check that /dev/vdc exists and the block size is 16M. 5272 assert_eq!( 5273 guest 5274 .ssh_command("lsblk | grep vdc | grep -c 16M") 5275 .unwrap() 5276 .trim() 5277 .parse::<u32>() 5278 .unwrap_or_default(), 5279 1 5280 ); 5281 // And check the block device can be read. 5282 guest 5283 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5284 .unwrap(); 5285 5286 // Let's remove it the extra disk. 5287 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5288 thread::sleep(std::time::Duration::new(5, 0)); 5289 // And check /dev/vdc is not there 5290 assert_eq!( 5291 guest 5292 .ssh_command("lsblk | grep -c vdc.*16M || true") 5293 .unwrap() 5294 .trim() 5295 .parse::<u32>() 5296 .unwrap_or(1), 5297 0 5298 ); 5299 5300 // And add it back to validate unplug did work correctly. 5301 let (cmd_success, cmd_output) = remote_command_w_output( 5302 &api_socket, 5303 "add-disk", 5304 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5305 ); 5306 assert!(cmd_success); 5307 assert!(String::from_utf8_lossy(&cmd_output) 5308 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5309 5310 thread::sleep(std::time::Duration::new(10, 0)); 5311 5312 // Check that /dev/vdc exists and the block size is 16M. 5313 assert_eq!( 5314 guest 5315 .ssh_command("lsblk | grep vdc | grep -c 16M") 5316 .unwrap() 5317 .trim() 5318 .parse::<u32>() 5319 .unwrap_or_default(), 5320 1 5321 ); 5322 // And check the block device can be read. 5323 guest 5324 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5325 .unwrap(); 5326 5327 // Reboot the VM. 5328 guest.reboot_linux(0, None); 5329 5330 // Check still there after reboot 5331 assert_eq!( 5332 guest 5333 .ssh_command("lsblk | grep vdc | grep -c 16M") 5334 .unwrap() 5335 .trim() 5336 .parse::<u32>() 5337 .unwrap_or_default(), 5338 1 5339 ); 5340 5341 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5342 5343 thread::sleep(std::time::Duration::new(20, 0)); 5344 5345 // Check device has gone away 5346 assert_eq!( 5347 guest 5348 .ssh_command("lsblk | grep -c vdc.*16M || true") 5349 .unwrap() 5350 .trim() 5351 .parse::<u32>() 5352 .unwrap_or(1), 5353 0 5354 ); 5355 5356 guest.reboot_linux(1, None); 5357 5358 // Check device still absent 5359 assert_eq!( 5360 guest 5361 .ssh_command("lsblk | grep -c vdc.*16M || true") 5362 .unwrap() 5363 .trim() 5364 .parse::<u32>() 5365 .unwrap_or(1), 5366 0 5367 ); 5368 }); 5369 5370 kill_child(&mut child); 5371 let output = child.wait_with_output().unwrap(); 5372 5373 handle_child_output(r, &output); 5374 } 5375 5376 #[test] 5377 fn test_disk_hotplug() { 5378 _test_disk_hotplug(false) 5379 } 5380 5381 #[test] 5382 #[cfg(target_arch = "x86_64")] 5383 fn test_disk_hotplug_with_landlock() { 5384 _test_disk_hotplug(true) 5385 } 5386 5387 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 5388 const LOOP_CONFIGURE: u64 = 0x4c0a; 5389 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 5390 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 5391 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 5392 5393 #[repr(C)] 5394 struct LoopInfo64 { 5395 lo_device: u64, 5396 lo_inode: u64, 5397 lo_rdevice: u64, 5398 lo_offset: u64, 5399 lo_sizelimit: u64, 5400 lo_number: u32, 5401 lo_encrypt_type: u32, 5402 lo_encrypt_key_size: u32, 5403 lo_flags: u32, 5404 lo_file_name: [u8; 64], 5405 lo_crypt_name: [u8; 64], 5406 lo_encrypt_key: [u8; 32], 5407 lo_init: [u64; 2], 5408 } 5409 5410 impl Default for LoopInfo64 { 5411 fn default() -> Self { 5412 LoopInfo64 { 5413 lo_device: 0, 5414 lo_inode: 0, 5415 lo_rdevice: 0, 5416 lo_offset: 0, 5417 lo_sizelimit: 0, 5418 lo_number: 0, 5419 lo_encrypt_type: 0, 5420 lo_encrypt_key_size: 0, 5421 lo_flags: 0, 5422 lo_file_name: [0; 64], 5423 lo_crypt_name: [0; 64], 5424 lo_encrypt_key: [0; 32], 5425 lo_init: [0; 2], 5426 } 5427 } 5428 } 5429 5430 #[derive(Default)] 5431 #[repr(C)] 5432 struct LoopConfig { 5433 fd: u32, 5434 block_size: u32, 5435 info: LoopInfo64, 5436 _reserved: [u64; 8], 5437 } 5438 5439 // Open loop-control device 5440 let loop_ctl_file = OpenOptions::new() 5441 .read(true) 5442 .write(true) 5443 .open(LOOP_CTL_PATH) 5444 .unwrap(); 5445 5446 // Request a free loop device 5447 let loop_device_number = 5448 unsafe { libc::ioctl(loop_ctl_file.as_raw_fd(), LOOP_CTL_GET_FREE as _) }; 5449 5450 if loop_device_number < 0 { 5451 panic!("Couldn't find a free loop device"); 5452 } 5453 5454 // Create loop device path 5455 let loop_device_path = format!("{LOOP_DEVICE_PREFIX}{loop_device_number}"); 5456 5457 // Open loop device 5458 let loop_device_file = OpenOptions::new() 5459 .read(true) 5460 .write(true) 5461 .open(&loop_device_path) 5462 .unwrap(); 5463 5464 // Open backing file 5465 let backing_file = OpenOptions::new() 5466 .read(true) 5467 .write(true) 5468 .open(backing_file_path) 5469 .unwrap(); 5470 5471 let loop_config = LoopConfig { 5472 fd: backing_file.as_raw_fd() as u32, 5473 block_size, 5474 ..Default::default() 5475 }; 5476 5477 for i in 0..num_retries { 5478 let ret = unsafe { 5479 libc::ioctl( 5480 loop_device_file.as_raw_fd(), 5481 LOOP_CONFIGURE as _, 5482 &loop_config, 5483 ) 5484 }; 5485 if ret != 0 { 5486 if i < num_retries - 1 { 5487 println!( 5488 "Iteration {}: Failed to configure the loop device {}: {}", 5489 i, 5490 loop_device_path, 5491 std::io::Error::last_os_error() 5492 ); 5493 } else { 5494 panic!( 5495 "Failed {} times trying to configure the loop device {}: {}", 5496 num_retries, 5497 loop_device_path, 5498 std::io::Error::last_os_error() 5499 ); 5500 } 5501 } else { 5502 break; 5503 } 5504 5505 // Wait for a bit before retrying 5506 thread::sleep(std::time::Duration::new(5, 0)); 5507 } 5508 5509 loop_device_path 5510 } 5511 5512 #[test] 5513 fn test_virtio_block_topology() { 5514 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5515 let guest = Guest::new(Box::new(focal)); 5516 5517 let kernel_path = direct_kernel_boot_path(); 5518 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5519 5520 let output = exec_host_command_output( 5521 format!( 5522 "qemu-img create -f raw {} 16M", 5523 test_disk_path.to_str().unwrap() 5524 ) 5525 .as_str(), 5526 ); 5527 if !output.status.success() { 5528 let stdout = String::from_utf8_lossy(&output.stdout); 5529 let stderr = String::from_utf8_lossy(&output.stderr); 5530 panic!("qemu-img command failed\nstdout\n{stdout}\nstderr\n{stderr}"); 5531 } 5532 5533 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5534 5535 let mut child = GuestCommand::new(&guest) 5536 .args(["--cpus", "boot=1"]) 5537 .args(["--memory", "size=512M"]) 5538 .args(["--kernel", kernel_path.to_str().unwrap()]) 5539 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5540 .args([ 5541 "--disk", 5542 format!( 5543 "path={}", 5544 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5545 ) 5546 .as_str(), 5547 format!( 5548 "path={}", 5549 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5550 ) 5551 .as_str(), 5552 format!("path={}", &loop_dev).as_str(), 5553 ]) 5554 .default_net() 5555 .capture_output() 5556 .spawn() 5557 .unwrap(); 5558 5559 let r = std::panic::catch_unwind(|| { 5560 guest.wait_vm_boot(None).unwrap(); 5561 5562 // MIN-IO column 5563 assert_eq!( 5564 guest 5565 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5566 .unwrap() 5567 .trim() 5568 .parse::<u32>() 5569 .unwrap_or_default(), 5570 4096 5571 ); 5572 // PHY-SEC column 5573 assert_eq!( 5574 guest 5575 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5576 .unwrap() 5577 .trim() 5578 .parse::<u32>() 5579 .unwrap_or_default(), 5580 4096 5581 ); 5582 // LOG-SEC column 5583 assert_eq!( 5584 guest 5585 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5586 .unwrap() 5587 .trim() 5588 .parse::<u32>() 5589 .unwrap_or_default(), 5590 4096 5591 ); 5592 }); 5593 5594 kill_child(&mut child); 5595 let output = child.wait_with_output().unwrap(); 5596 5597 handle_child_output(r, &output); 5598 5599 Command::new("losetup") 5600 .args(["-d", &loop_dev]) 5601 .output() 5602 .expect("loop device not found"); 5603 } 5604 5605 #[test] 5606 fn test_virtio_balloon_deflate_on_oom() { 5607 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5608 let guest = Guest::new(Box::new(focal)); 5609 5610 let kernel_path = direct_kernel_boot_path(); 5611 5612 let api_socket = temp_api_path(&guest.tmp_dir); 5613 5614 //Let's start a 4G guest with balloon occupied 2G memory 5615 let mut child = GuestCommand::new(&guest) 5616 .args(["--api-socket", &api_socket]) 5617 .args(["--cpus", "boot=1"]) 5618 .args(["--memory", "size=4G"]) 5619 .args(["--kernel", kernel_path.to_str().unwrap()]) 5620 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5621 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5622 .default_disks() 5623 .default_net() 5624 .capture_output() 5625 .spawn() 5626 .unwrap(); 5627 5628 let r = std::panic::catch_unwind(|| { 5629 guest.wait_vm_boot(None).unwrap(); 5630 5631 // Wait for balloon memory's initialization and check its size. 5632 // The virtio-balloon driver might take a few seconds to report the 5633 // balloon effective size back to the VMM. 5634 thread::sleep(std::time::Duration::new(20, 0)); 5635 5636 let orig_balloon = balloon_size(&api_socket); 5637 println!("The original balloon memory size is {orig_balloon} bytes"); 5638 assert!(orig_balloon == 2147483648); 5639 5640 // Two steps to verify if the 'deflate_on_oom' parameter works. 5641 // 1st: run a command to trigger an OOM in the guest. 5642 guest 5643 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5644 .unwrap(); 5645 5646 // Give some time for the OOM to happen in the guest and be reported 5647 // back to the host. 5648 thread::sleep(std::time::Duration::new(20, 0)); 5649 5650 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5651 let deflated_balloon = balloon_size(&api_socket); 5652 println!("After deflating, balloon memory size is {deflated_balloon} bytes"); 5653 // Verify the balloon size deflated 5654 assert!(deflated_balloon < 2147483648); 5655 }); 5656 5657 kill_child(&mut child); 5658 let output = child.wait_with_output().unwrap(); 5659 5660 handle_child_output(r, &output); 5661 } 5662 5663 #[test] 5664 #[cfg(not(feature = "mshv"))] 5665 fn test_virtio_balloon_free_page_reporting() { 5666 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5667 let guest = Guest::new(Box::new(focal)); 5668 5669 //Let's start a 4G guest with balloon occupied 2G memory 5670 let mut child = GuestCommand::new(&guest) 5671 .args(["--cpus", "boot=1"]) 5672 .args(["--memory", "size=4G"]) 5673 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5674 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5675 .args(["--balloon", "size=0,free_page_reporting=on"]) 5676 .default_disks() 5677 .default_net() 5678 .capture_output() 5679 .spawn() 5680 .unwrap(); 5681 5682 let pid = child.id(); 5683 let r = std::panic::catch_unwind(|| { 5684 guest.wait_vm_boot(None).unwrap(); 5685 5686 // Check the initial RSS is less than 1GiB 5687 let rss = process_rss_kib(pid); 5688 println!("RSS {rss} < 1048576"); 5689 assert!(rss < 1048576); 5690 5691 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5692 // seconds 5693 let guest_ip = guest.network.guest_ip.clone(); 5694 thread::spawn(move || { 5695 ssh_command_ip( 5696 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5697 &guest_ip, 5698 DEFAULT_SSH_RETRIES, 5699 DEFAULT_SSH_TIMEOUT, 5700 ) 5701 .unwrap(); 5702 }); 5703 5704 // Wait for 50 seconds to make sure the stress command is consuming 5705 // the expected amount of memory. 5706 thread::sleep(std::time::Duration::new(50, 0)); 5707 let rss = process_rss_kib(pid); 5708 println!("RSS {rss} >= 2097152"); 5709 assert!(rss >= 2097152); 5710 5711 // Wait for an extra minute to make sure the stress command has 5712 // completed and that the guest reported the free pages to the VMM 5713 // through the virtio-balloon device. We expect the RSS to be under 5714 // 2GiB. 5715 thread::sleep(std::time::Duration::new(60, 0)); 5716 let rss = process_rss_kib(pid); 5717 println!("RSS {rss} < 2097152"); 5718 assert!(rss < 2097152); 5719 }); 5720 5721 kill_child(&mut child); 5722 let output = child.wait_with_output().unwrap(); 5723 5724 handle_child_output(r, &output); 5725 } 5726 5727 #[test] 5728 fn test_pmem_hotplug() { 5729 _test_pmem_hotplug(None) 5730 } 5731 5732 #[test] 5733 fn test_pmem_multi_segment_hotplug() { 5734 _test_pmem_hotplug(Some(15)) 5735 } 5736 5737 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5738 #[cfg(target_arch = "aarch64")] 5739 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5740 #[cfg(target_arch = "x86_64")] 5741 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5742 let focal = UbuntuDiskConfig::new(focal_image); 5743 let guest = Guest::new(Box::new(focal)); 5744 5745 #[cfg(target_arch = "x86_64")] 5746 let kernel_path = direct_kernel_boot_path(); 5747 #[cfg(target_arch = "aarch64")] 5748 let kernel_path = edk2_path(); 5749 5750 let api_socket = temp_api_path(&guest.tmp_dir); 5751 5752 let mut cmd = GuestCommand::new(&guest); 5753 5754 cmd.args(["--api-socket", &api_socket]) 5755 .args(["--cpus", "boot=1"]) 5756 .args(["--memory", "size=512M"]) 5757 .args(["--kernel", kernel_path.to_str().unwrap()]) 5758 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5759 .default_disks() 5760 .default_net() 5761 .capture_output(); 5762 5763 if pci_segment.is_some() { 5764 cmd.args([ 5765 "--platform", 5766 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5767 ]); 5768 } 5769 5770 let mut child = cmd.spawn().unwrap(); 5771 5772 let r = std::panic::catch_unwind(|| { 5773 guest.wait_vm_boot(None).unwrap(); 5774 5775 // Check /dev/pmem0 is not there 5776 assert_eq!( 5777 guest 5778 .ssh_command("lsblk | grep -c pmem0 || true") 5779 .unwrap() 5780 .trim() 5781 .parse::<u32>() 5782 .unwrap_or(1), 5783 0 5784 ); 5785 5786 let pmem_temp_file = TempFile::new().unwrap(); 5787 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5788 let (cmd_success, cmd_output) = remote_command_w_output( 5789 &api_socket, 5790 "add-pmem", 5791 Some(&format!( 5792 "file={},id=test0{}", 5793 pmem_temp_file.as_path().to_str().unwrap(), 5794 if let Some(pci_segment) = pci_segment { 5795 format!(",pci_segment={pci_segment}") 5796 } else { 5797 "".to_owned() 5798 } 5799 )), 5800 ); 5801 assert!(cmd_success); 5802 if let Some(pci_segment) = pci_segment { 5803 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5804 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5805 ))); 5806 } else { 5807 assert!(String::from_utf8_lossy(&cmd_output) 5808 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5809 } 5810 5811 // Check that /dev/pmem0 exists and the block size is 128M 5812 assert_eq!( 5813 guest 5814 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5815 .unwrap() 5816 .trim() 5817 .parse::<u32>() 5818 .unwrap_or_default(), 5819 1 5820 ); 5821 5822 guest.reboot_linux(0, None); 5823 5824 // Check still there after reboot 5825 assert_eq!( 5826 guest 5827 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5828 .unwrap() 5829 .trim() 5830 .parse::<u32>() 5831 .unwrap_or_default(), 5832 1 5833 ); 5834 5835 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5836 5837 thread::sleep(std::time::Duration::new(20, 0)); 5838 5839 // Check device has gone away 5840 assert_eq!( 5841 guest 5842 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5843 .unwrap() 5844 .trim() 5845 .parse::<u32>() 5846 .unwrap_or(1), 5847 0 5848 ); 5849 5850 guest.reboot_linux(1, None); 5851 5852 // Check still absent after reboot 5853 assert_eq!( 5854 guest 5855 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5856 .unwrap() 5857 .trim() 5858 .parse::<u32>() 5859 .unwrap_or(1), 5860 0 5861 ); 5862 }); 5863 5864 kill_child(&mut child); 5865 let output = child.wait_with_output().unwrap(); 5866 5867 handle_child_output(r, &output); 5868 } 5869 5870 #[test] 5871 fn test_net_hotplug() { 5872 _test_net_hotplug(None) 5873 } 5874 5875 #[test] 5876 fn test_net_multi_segment_hotplug() { 5877 _test_net_hotplug(Some(15)) 5878 } 5879 5880 fn _test_net_hotplug(pci_segment: Option<u16>) { 5881 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5882 let guest = Guest::new(Box::new(focal)); 5883 5884 #[cfg(target_arch = "x86_64")] 5885 let kernel_path = direct_kernel_boot_path(); 5886 #[cfg(target_arch = "aarch64")] 5887 let kernel_path = edk2_path(); 5888 5889 let api_socket = temp_api_path(&guest.tmp_dir); 5890 5891 // Boot without network 5892 let mut cmd = GuestCommand::new(&guest); 5893 5894 cmd.args(["--api-socket", &api_socket]) 5895 .args(["--cpus", "boot=1"]) 5896 .args(["--memory", "size=512M"]) 5897 .args(["--kernel", kernel_path.to_str().unwrap()]) 5898 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5899 .default_disks() 5900 .capture_output(); 5901 5902 if pci_segment.is_some() { 5903 cmd.args([ 5904 "--platform", 5905 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5906 ]); 5907 } 5908 5909 let mut child = cmd.spawn().unwrap(); 5910 5911 thread::sleep(std::time::Duration::new(20, 0)); 5912 5913 let r = std::panic::catch_unwind(|| { 5914 // Add network 5915 let (cmd_success, cmd_output) = remote_command_w_output( 5916 &api_socket, 5917 "add-net", 5918 Some( 5919 format!( 5920 "{}{},id=test0", 5921 guest.default_net_string(), 5922 if let Some(pci_segment) = pci_segment { 5923 format!(",pci_segment={pci_segment}") 5924 } else { 5925 "".to_owned() 5926 } 5927 ) 5928 .as_str(), 5929 ), 5930 ); 5931 assert!(cmd_success); 5932 5933 if let Some(pci_segment) = pci_segment { 5934 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5935 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5936 ))); 5937 } else { 5938 assert!(String::from_utf8_lossy(&cmd_output) 5939 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5940 } 5941 5942 thread::sleep(std::time::Duration::new(5, 0)); 5943 5944 // 1 network interfaces + default localhost ==> 2 interfaces 5945 assert_eq!( 5946 guest 5947 .ssh_command("ip -o link | wc -l") 5948 .unwrap() 5949 .trim() 5950 .parse::<u32>() 5951 .unwrap_or_default(), 5952 2 5953 ); 5954 5955 // Remove network 5956 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5957 thread::sleep(std::time::Duration::new(5, 0)); 5958 5959 let (cmd_success, cmd_output) = remote_command_w_output( 5960 &api_socket, 5961 "add-net", 5962 Some( 5963 format!( 5964 "{}{},id=test1", 5965 guest.default_net_string(), 5966 if let Some(pci_segment) = pci_segment { 5967 format!(",pci_segment={pci_segment}") 5968 } else { 5969 "".to_owned() 5970 } 5971 ) 5972 .as_str(), 5973 ), 5974 ); 5975 assert!(cmd_success); 5976 5977 if let Some(pci_segment) = pci_segment { 5978 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5979 "{{\"id\":\"test1\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5980 ))); 5981 } else { 5982 assert!(String::from_utf8_lossy(&cmd_output) 5983 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5984 } 5985 5986 thread::sleep(std::time::Duration::new(5, 0)); 5987 5988 // 1 network interfaces + default localhost ==> 2 interfaces 5989 assert_eq!( 5990 guest 5991 .ssh_command("ip -o link | wc -l") 5992 .unwrap() 5993 .trim() 5994 .parse::<u32>() 5995 .unwrap_or_default(), 5996 2 5997 ); 5998 5999 guest.reboot_linux(0, None); 6000 6001 // Check still there after reboot 6002 // 1 network interfaces + default localhost ==> 2 interfaces 6003 assert_eq!( 6004 guest 6005 .ssh_command("ip -o link | wc -l") 6006 .unwrap() 6007 .trim() 6008 .parse::<u32>() 6009 .unwrap_or_default(), 6010 2 6011 ); 6012 }); 6013 6014 kill_child(&mut child); 6015 let output = child.wait_with_output().unwrap(); 6016 6017 handle_child_output(r, &output); 6018 } 6019 6020 #[test] 6021 fn test_initramfs() { 6022 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6023 let guest = Guest::new(Box::new(focal)); 6024 let mut workload_path = dirs::home_dir().unwrap(); 6025 workload_path.push("workloads"); 6026 6027 #[cfg(target_arch = "x86_64")] 6028 let mut kernels = vec![direct_kernel_boot_path()]; 6029 #[cfg(target_arch = "aarch64")] 6030 let kernels = [direct_kernel_boot_path()]; 6031 6032 #[cfg(target_arch = "x86_64")] 6033 { 6034 let mut pvh_kernel_path = workload_path.clone(); 6035 pvh_kernel_path.push("vmlinux"); 6036 kernels.push(pvh_kernel_path); 6037 } 6038 6039 let mut initramfs_path = workload_path; 6040 initramfs_path.push("alpine_initramfs.img"); 6041 6042 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 6043 let cmdline = format!("console=hvc0 quiet TEST_STRING={test_string}"); 6044 6045 kernels.iter().for_each(|k_path| { 6046 let mut child = GuestCommand::new(&guest) 6047 .args(["--kernel", k_path.to_str().unwrap()]) 6048 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 6049 .args(["--cmdline", &cmdline]) 6050 .capture_output() 6051 .spawn() 6052 .unwrap(); 6053 6054 thread::sleep(std::time::Duration::new(20, 0)); 6055 6056 kill_child(&mut child); 6057 let output = child.wait_with_output().unwrap(); 6058 6059 let r = std::panic::catch_unwind(|| { 6060 let s = String::from_utf8_lossy(&output.stdout); 6061 6062 assert_ne!(s.lines().position(|line| line == test_string), None); 6063 }); 6064 6065 handle_child_output(r, &output); 6066 }); 6067 } 6068 6069 #[test] 6070 fn test_counters() { 6071 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6072 let guest = Guest::new(Box::new(focal)); 6073 let api_socket = temp_api_path(&guest.tmp_dir); 6074 6075 let mut cmd = GuestCommand::new(&guest); 6076 cmd.args(["--cpus", "boot=1"]) 6077 .args(["--memory", "size=512M"]) 6078 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 6079 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6080 .default_disks() 6081 .args(["--net", guest.default_net_string().as_str()]) 6082 .args(["--api-socket", &api_socket]) 6083 .capture_output(); 6084 6085 let mut child = cmd.spawn().unwrap(); 6086 6087 let r = std::panic::catch_unwind(|| { 6088 guest.wait_vm_boot(None).unwrap(); 6089 6090 let orig_counters = get_counters(&api_socket); 6091 guest 6092 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 6093 .unwrap(); 6094 6095 let new_counters = get_counters(&api_socket); 6096 6097 // Check that all the counters have increased 6098 assert!(new_counters > orig_counters); 6099 }); 6100 6101 kill_child(&mut child); 6102 let output = child.wait_with_output().unwrap(); 6103 6104 handle_child_output(r, &output); 6105 } 6106 6107 #[test] 6108 #[cfg(feature = "guest_debug")] 6109 fn test_coredump() { 6110 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6111 let guest = Guest::new(Box::new(focal)); 6112 let api_socket = temp_api_path(&guest.tmp_dir); 6113 6114 let mut cmd = GuestCommand::new(&guest); 6115 cmd.args(["--cpus", "boot=4"]) 6116 .args(["--memory", "size=4G"]) 6117 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6118 .default_disks() 6119 .args(["--net", guest.default_net_string().as_str()]) 6120 .args(["--api-socket", &api_socket]) 6121 .capture_output(); 6122 6123 let mut child = cmd.spawn().unwrap(); 6124 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6125 6126 let r = std::panic::catch_unwind(|| { 6127 guest.wait_vm_boot(None).unwrap(); 6128 6129 assert!(remote_command(&api_socket, "pause", None)); 6130 6131 assert!(remote_command( 6132 &api_socket, 6133 "coredump", 6134 Some(format!("file://{vmcore_file}").as_str()), 6135 )); 6136 6137 // the num of CORE notes should equals to vcpu 6138 let readelf_core_num_cmd = 6139 format!("readelf --all {vmcore_file} |grep CORE |grep -v Type |wc -l"); 6140 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 6141 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 6142 6143 // the num of QEMU notes should equals to vcpu 6144 let readelf_vmm_num_cmd = format!("readelf --all {vmcore_file} |grep QEMU |wc -l"); 6145 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 6146 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 6147 }); 6148 6149 kill_child(&mut child); 6150 let output = child.wait_with_output().unwrap(); 6151 6152 handle_child_output(r, &output); 6153 } 6154 6155 #[test] 6156 #[cfg(feature = "guest_debug")] 6157 fn test_coredump_no_pause() { 6158 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6159 let guest = Guest::new(Box::new(focal)); 6160 let api_socket = temp_api_path(&guest.tmp_dir); 6161 6162 let mut cmd = GuestCommand::new(&guest); 6163 cmd.args(["--cpus", "boot=4"]) 6164 .args(["--memory", "size=4G"]) 6165 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6166 .default_disks() 6167 .args(["--net", guest.default_net_string().as_str()]) 6168 .args(["--api-socket", &api_socket]) 6169 .capture_output(); 6170 6171 let mut child = cmd.spawn().unwrap(); 6172 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6173 6174 let r = std::panic::catch_unwind(|| { 6175 guest.wait_vm_boot(None).unwrap(); 6176 6177 assert!(remote_command( 6178 &api_socket, 6179 "coredump", 6180 Some(format!("file://{vmcore_file}").as_str()), 6181 )); 6182 6183 assert_eq!(vm_state(&api_socket), "Running"); 6184 }); 6185 6186 kill_child(&mut child); 6187 let output = child.wait_with_output().unwrap(); 6188 6189 handle_child_output(r, &output); 6190 } 6191 6192 #[test] 6193 fn test_watchdog() { 6194 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6195 let guest = Guest::new(Box::new(focal)); 6196 let api_socket = temp_api_path(&guest.tmp_dir); 6197 6198 let kernel_path = direct_kernel_boot_path(); 6199 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6200 6201 let mut cmd = GuestCommand::new(&guest); 6202 cmd.args(["--cpus", "boot=1"]) 6203 .args(["--memory", "size=512M"]) 6204 .args(["--kernel", kernel_path.to_str().unwrap()]) 6205 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6206 .default_disks() 6207 .args(["--net", guest.default_net_string().as_str()]) 6208 .args(["--watchdog"]) 6209 .args(["--api-socket", &api_socket]) 6210 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6211 .capture_output(); 6212 6213 let mut child = cmd.spawn().unwrap(); 6214 6215 let r = std::panic::catch_unwind(|| { 6216 guest.wait_vm_boot(None).unwrap(); 6217 6218 let mut expected_reboot_count = 1; 6219 6220 // Enable the watchdog with a 15s timeout 6221 enable_guest_watchdog(&guest, 15); 6222 6223 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6224 assert_eq!( 6225 guest 6226 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6227 .unwrap() 6228 .trim() 6229 .parse::<u32>() 6230 .unwrap_or_default(), 6231 1 6232 ); 6233 6234 // Allow some normal time to elapse to check we don't get spurious reboots 6235 thread::sleep(std::time::Duration::new(40, 0)); 6236 // Check no reboot 6237 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6238 6239 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6240 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6241 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6242 guest.wait_vm_boot(Some(50)).unwrap(); 6243 // Check a reboot is triggered by the watchdog 6244 expected_reboot_count += 1; 6245 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6246 6247 #[cfg(target_arch = "x86_64")] 6248 { 6249 // Now pause the VM and remain offline for 30s 6250 assert!(remote_command(&api_socket, "pause", None)); 6251 let latest_events = [ 6252 &MetaEvent { 6253 event: "pausing".to_string(), 6254 device_id: None, 6255 }, 6256 &MetaEvent { 6257 event: "paused".to_string(), 6258 device_id: None, 6259 }, 6260 ]; 6261 assert!(check_latest_events_exact(&latest_events, &event_path)); 6262 assert!(remote_command(&api_socket, "resume", None)); 6263 6264 // Check no reboot 6265 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6266 } 6267 }); 6268 6269 kill_child(&mut child); 6270 let output = child.wait_with_output().unwrap(); 6271 6272 handle_child_output(r, &output); 6273 } 6274 6275 #[test] 6276 fn test_pvpanic() { 6277 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 6278 let guest = Guest::new(Box::new(jammy)); 6279 let api_socket = temp_api_path(&guest.tmp_dir); 6280 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6281 6282 let kernel_path = direct_kernel_boot_path(); 6283 6284 let mut cmd = GuestCommand::new(&guest); 6285 cmd.args(["--cpus", "boot=1"]) 6286 .args(["--memory", "size=512M"]) 6287 .args(["--kernel", kernel_path.to_str().unwrap()]) 6288 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6289 .default_disks() 6290 .args(["--net", guest.default_net_string().as_str()]) 6291 .args(["--pvpanic"]) 6292 .args(["--api-socket", &api_socket]) 6293 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6294 .capture_output(); 6295 6296 let mut child = cmd.spawn().unwrap(); 6297 6298 let r = std::panic::catch_unwind(|| { 6299 guest.wait_vm_boot(None).unwrap(); 6300 6301 // Trigger guest a panic 6302 make_guest_panic(&guest); 6303 6304 // Wait a while for guest 6305 thread::sleep(std::time::Duration::new(10, 0)); 6306 6307 let expected_sequential_events = [&MetaEvent { 6308 event: "panic".to_string(), 6309 device_id: None, 6310 }]; 6311 assert!(check_latest_events_exact( 6312 &expected_sequential_events, 6313 &event_path 6314 )); 6315 }); 6316 6317 kill_child(&mut child); 6318 let output = child.wait_with_output().unwrap(); 6319 6320 handle_child_output(r, &output); 6321 } 6322 6323 #[test] 6324 fn test_tap_from_fd() { 6325 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6326 let guest = Guest::new(Box::new(focal)); 6327 let kernel_path = direct_kernel_boot_path(); 6328 6329 // Create a TAP interface with multi-queue enabled 6330 let num_queue_pairs: usize = 2; 6331 6332 use std::str::FromStr; 6333 let taps = net_util::open_tap( 6334 Some("chtap0"), 6335 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6336 None, 6337 &mut None, 6338 None, 6339 num_queue_pairs, 6340 Some(libc::O_RDWR | libc::O_NONBLOCK), 6341 ) 6342 .unwrap(); 6343 6344 let mut child = GuestCommand::new(&guest) 6345 .args(["--cpus", &format!("boot={num_queue_pairs}")]) 6346 .args(["--memory", "size=512M"]) 6347 .args(["--kernel", kernel_path.to_str().unwrap()]) 6348 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6349 .default_disks() 6350 .args([ 6351 "--net", 6352 &format!( 6353 "fd=[{},{}],mac={},num_queues={}", 6354 taps[0].as_raw_fd(), 6355 taps[1].as_raw_fd(), 6356 guest.network.guest_mac, 6357 num_queue_pairs * 2 6358 ), 6359 ]) 6360 .capture_output() 6361 .spawn() 6362 .unwrap(); 6363 6364 let r = std::panic::catch_unwind(|| { 6365 guest.wait_vm_boot(None).unwrap(); 6366 6367 assert_eq!( 6368 guest 6369 .ssh_command("ip -o link | wc -l") 6370 .unwrap() 6371 .trim() 6372 .parse::<u32>() 6373 .unwrap_or_default(), 6374 2 6375 ); 6376 6377 guest.reboot_linux(0, None); 6378 6379 assert_eq!( 6380 guest 6381 .ssh_command("ip -o link | wc -l") 6382 .unwrap() 6383 .trim() 6384 .parse::<u32>() 6385 .unwrap_or_default(), 6386 2 6387 ); 6388 }); 6389 6390 kill_child(&mut child); 6391 let output = child.wait_with_output().unwrap(); 6392 6393 handle_child_output(r, &output); 6394 } 6395 6396 // By design, a guest VM won't be able to connect to the host 6397 // machine when using a macvtap network interface (while it can 6398 // communicate externally). As a workaround, this integration 6399 // test creates two macvtap interfaces in 'bridge' mode on the 6400 // same physical net interface, one for the guest and one for 6401 // the host. With additional setup on the IP address and the 6402 // routing table, it enables the communications between the 6403 // guest VM and the host machine. 6404 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6405 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6406 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6407 let guest = Guest::new(Box::new(focal)); 6408 let api_socket = temp_api_path(&guest.tmp_dir); 6409 6410 #[cfg(target_arch = "x86_64")] 6411 let kernel_path = direct_kernel_boot_path(); 6412 #[cfg(target_arch = "aarch64")] 6413 let kernel_path = edk2_path(); 6414 6415 let phy_net = "eth0"; 6416 6417 // Create a macvtap interface for the guest VM to use 6418 assert!(exec_host_command_status(&format!( 6419 "sudo ip link add link {phy_net} name {guest_macvtap_name} type macvtap mod bridge" 6420 )) 6421 .success()); 6422 assert!(exec_host_command_status(&format!( 6423 "sudo ip link set {} address {} up", 6424 guest_macvtap_name, guest.network.guest_mac 6425 )) 6426 .success()); 6427 assert!( 6428 exec_host_command_status(&format!("sudo ip link show {guest_macvtap_name}")).success() 6429 ); 6430 6431 let tap_index = 6432 fs::read_to_string(format!("/sys/class/net/{guest_macvtap_name}/ifindex")).unwrap(); 6433 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6434 6435 assert!(exec_host_command_status(&format!("sudo chown $UID.$UID {tap_device}")).success()); 6436 6437 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6438 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6439 assert!(tap_fd1 > 0); 6440 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6441 assert!(tap_fd2 > 0); 6442 6443 // Create a macvtap on the same physical net interface for 6444 // the host machine to use 6445 assert!(exec_host_command_status(&format!( 6446 "sudo ip link add link {phy_net} name {host_macvtap_name} type macvtap mod bridge" 6447 )) 6448 .success()); 6449 // Use default mask "255.255.255.0" 6450 assert!(exec_host_command_status(&format!( 6451 "sudo ip address add {}/24 dev {}", 6452 guest.network.host_ip, host_macvtap_name 6453 )) 6454 .success()); 6455 assert!( 6456 exec_host_command_status(&format!("sudo ip link set dev {host_macvtap_name} up")) 6457 .success() 6458 ); 6459 6460 let mut guest_command = GuestCommand::new(&guest); 6461 guest_command 6462 .args(["--cpus", "boot=2"]) 6463 .args(["--memory", "size=512M"]) 6464 .args(["--kernel", kernel_path.to_str().unwrap()]) 6465 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6466 .default_disks() 6467 .args(["--api-socket", &api_socket]); 6468 6469 let net_params = format!( 6470 "fd=[{},{}],mac={},num_queues=4", 6471 tap_fd1, tap_fd2, guest.network.guest_mac 6472 ); 6473 6474 if !hotplug { 6475 guest_command.args(["--net", &net_params]); 6476 } 6477 6478 let mut child = guest_command.capture_output().spawn().unwrap(); 6479 6480 if hotplug { 6481 // Give some time to the VMM process to listen to the API 6482 // socket. This is the only requirement to avoid the following 6483 // call to ch-remote from failing. 6484 thread::sleep(std::time::Duration::new(10, 0)); 6485 // Hotplug the virtio-net device 6486 let (cmd_success, cmd_output) = 6487 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6488 assert!(cmd_success); 6489 #[cfg(target_arch = "x86_64")] 6490 assert!(String::from_utf8_lossy(&cmd_output) 6491 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6492 #[cfg(target_arch = "aarch64")] 6493 assert!(String::from_utf8_lossy(&cmd_output) 6494 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6495 } 6496 6497 // The functional connectivity provided by the virtio-net device 6498 // gets tested through wait_vm_boot() as it expects to receive a 6499 // HTTP request, and through the SSH command as well. 6500 let r = std::panic::catch_unwind(|| { 6501 guest.wait_vm_boot(None).unwrap(); 6502 6503 assert_eq!( 6504 guest 6505 .ssh_command("ip -o link | wc -l") 6506 .unwrap() 6507 .trim() 6508 .parse::<u32>() 6509 .unwrap_or_default(), 6510 2 6511 ); 6512 6513 guest.reboot_linux(0, None); 6514 6515 assert_eq!( 6516 guest 6517 .ssh_command("ip -o link | wc -l") 6518 .unwrap() 6519 .trim() 6520 .parse::<u32>() 6521 .unwrap_or_default(), 6522 2 6523 ); 6524 }); 6525 6526 kill_child(&mut child); 6527 6528 exec_host_command_status(&format!("sudo ip link del {guest_macvtap_name}")); 6529 exec_host_command_status(&format!("sudo ip link del {host_macvtap_name}")); 6530 6531 let output = child.wait_with_output().unwrap(); 6532 6533 handle_child_output(r, &output); 6534 } 6535 6536 #[test] 6537 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6538 fn test_macvtap() { 6539 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6540 } 6541 6542 #[test] 6543 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6544 fn test_macvtap_hotplug() { 6545 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6546 } 6547 6548 #[test] 6549 #[cfg(not(feature = "mshv"))] 6550 fn test_ovs_dpdk() { 6551 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6552 let guest1 = Guest::new(Box::new(focal1)); 6553 6554 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6555 let guest2 = Guest::new(Box::new(focal2)); 6556 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6557 6558 let (mut child1, mut child2) = 6559 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6560 6561 // Create the snapshot directory 6562 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6563 6564 let r = std::panic::catch_unwind(|| { 6565 // Remove one of the two ports from the OVS bridge 6566 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6567 6568 // Spawn a new netcat listener in the first VM 6569 let guest_ip = guest1.network.guest_ip.clone(); 6570 thread::spawn(move || { 6571 ssh_command_ip( 6572 "nc -l 12345", 6573 &guest_ip, 6574 DEFAULT_SSH_RETRIES, 6575 DEFAULT_SSH_TIMEOUT, 6576 ) 6577 .unwrap(); 6578 }); 6579 6580 // Wait for the server to be listening 6581 thread::sleep(std::time::Duration::new(5, 0)); 6582 6583 // Check the connection fails this time 6584 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap_err(); 6585 6586 // Add the OVS port back 6587 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6588 6589 // And finally check the connection is functional again 6590 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6591 6592 // Pause the VM 6593 assert!(remote_command(&api_socket_source, "pause", None)); 6594 6595 // Take a snapshot from the VM 6596 assert!(remote_command( 6597 &api_socket_source, 6598 "snapshot", 6599 Some(format!("file://{snapshot_dir}").as_str()), 6600 )); 6601 6602 // Wait to make sure the snapshot is completed 6603 thread::sleep(std::time::Duration::new(10, 0)); 6604 }); 6605 6606 // Shutdown the source VM 6607 kill_child(&mut child2); 6608 let output = child2.wait_with_output().unwrap(); 6609 handle_child_output(r, &output); 6610 6611 // Remove the vhost-user socket file. 6612 Command::new("rm") 6613 .arg("-f") 6614 .arg("/tmp/dpdkvhostclient2") 6615 .output() 6616 .unwrap(); 6617 6618 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6619 // Restore the VM from the snapshot 6620 let mut child2 = GuestCommand::new(&guest2) 6621 .args(["--api-socket", &api_socket_restored]) 6622 .args([ 6623 "--restore", 6624 format!("source_url=file://{snapshot_dir}").as_str(), 6625 ]) 6626 .capture_output() 6627 .spawn() 6628 .unwrap(); 6629 6630 // Wait for the VM to be restored 6631 thread::sleep(std::time::Duration::new(10, 0)); 6632 6633 let r = std::panic::catch_unwind(|| { 6634 // Resume the VM 6635 assert!(remote_command(&api_socket_restored, "resume", None)); 6636 6637 // Spawn a new netcat listener in the first VM 6638 let guest_ip = guest1.network.guest_ip.clone(); 6639 thread::spawn(move || { 6640 ssh_command_ip( 6641 "nc -l 12345", 6642 &guest_ip, 6643 DEFAULT_SSH_RETRIES, 6644 DEFAULT_SSH_TIMEOUT, 6645 ) 6646 .unwrap(); 6647 }); 6648 6649 // Wait for the server to be listening 6650 thread::sleep(std::time::Duration::new(5, 0)); 6651 6652 // And check the connection is still functional after restore 6653 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6654 }); 6655 6656 kill_child(&mut child1); 6657 kill_child(&mut child2); 6658 6659 let output = child1.wait_with_output().unwrap(); 6660 child2.wait().unwrap(); 6661 6662 cleanup_ovs_dpdk(); 6663 6664 handle_child_output(r, &output); 6665 } 6666 6667 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6668 cleanup_spdk_nvme(); 6669 6670 assert!(exec_host_command_status(&format!( 6671 "mkdir -p {}", 6672 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6673 )) 6674 .success()); 6675 assert!(exec_host_command_status(&format!( 6676 "truncate {} -s 128M", 6677 nvme_dir.join("test-disk.raw").to_str().unwrap() 6678 )) 6679 .success()); 6680 assert!(exec_host_command_status(&format!( 6681 "mkfs.ext4 {}", 6682 nvme_dir.join("test-disk.raw").to_str().unwrap() 6683 )) 6684 .success()); 6685 6686 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6687 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6688 .args(["-i", "0", "-m", "0x1"]) 6689 .spawn() 6690 .unwrap(); 6691 thread::sleep(std::time::Duration::new(2, 0)); 6692 6693 assert!(exec_host_command_with_retries( 6694 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER", 6695 3, 6696 std::time::Duration::new(5, 0), 6697 )); 6698 assert!(exec_host_command_status(&format!( 6699 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6700 nvme_dir.join("test-disk.raw").to_str().unwrap() 6701 )) 6702 .success()); 6703 assert!(exec_host_command_status( 6704 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6705 ) 6706 .success()); 6707 assert!(exec_host_command_status( 6708 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6709 ) 6710 .success()); 6711 assert!(exec_host_command_status(&format!( 6712 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6713 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6714 )) 6715 .success()); 6716 } 6717 6718 fn cleanup_spdk_nvme() { 6719 exec_host_command_status("pkill -f nvmf_tgt"); 6720 } 6721 6722 #[test] 6723 fn test_vfio_user() { 6724 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6725 let jammy = UbuntuDiskConfig::new(jammy_image); 6726 let guest = Guest::new(Box::new(jammy)); 6727 6728 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6729 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6730 6731 let api_socket = temp_api_path(&guest.tmp_dir); 6732 let mut child = GuestCommand::new(&guest) 6733 .args(["--api-socket", &api_socket]) 6734 .args(["--cpus", "boot=1"]) 6735 .args(["--memory", "size=512M,shared=on,hugepages=on"]) 6736 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6737 .args(["--serial", "tty", "--console", "off"]) 6738 .default_disks() 6739 .default_net() 6740 .capture_output() 6741 .spawn() 6742 .unwrap(); 6743 6744 let r = std::panic::catch_unwind(|| { 6745 guest.wait_vm_boot(None).unwrap(); 6746 6747 // Hotplug the SPDK-NVMe device to the VM 6748 let (cmd_success, cmd_output) = remote_command_w_output( 6749 &api_socket, 6750 "add-user-device", 6751 Some(&format!( 6752 "socket={},id=vfio_user0", 6753 spdk_nvme_dir 6754 .as_path() 6755 .join("nvme-vfio-user/cntrl") 6756 .to_str() 6757 .unwrap(), 6758 )), 6759 ); 6760 assert!(cmd_success); 6761 assert!(String::from_utf8_lossy(&cmd_output) 6762 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6763 6764 thread::sleep(std::time::Duration::new(10, 0)); 6765 6766 // Check both if /dev/nvme exists and if the block size is 128M. 6767 assert_eq!( 6768 guest 6769 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6770 .unwrap() 6771 .trim() 6772 .parse::<u32>() 6773 .unwrap_or_default(), 6774 1 6775 ); 6776 6777 // Check changes persist after reboot 6778 assert_eq!( 6779 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6780 "" 6781 ); 6782 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6783 guest 6784 .ssh_command("echo test123 | sudo tee /mnt/test") 6785 .unwrap(); 6786 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6787 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6788 6789 guest.reboot_linux(0, None); 6790 assert_eq!( 6791 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6792 "" 6793 ); 6794 assert_eq!( 6795 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6796 "test123" 6797 ); 6798 }); 6799 6800 cleanup_spdk_nvme(); 6801 6802 kill_child(&mut child); 6803 let output = child.wait_with_output().unwrap(); 6804 6805 handle_child_output(r, &output); 6806 } 6807 6808 #[test] 6809 #[cfg(target_arch = "x86_64")] 6810 fn test_vdpa_block() { 6811 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6812 assert!(exec_host_command_status("lsmod | grep vdpa_sim_blk").success()); 6813 6814 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6815 let guest = Guest::new(Box::new(focal)); 6816 let api_socket = temp_api_path(&guest.tmp_dir); 6817 6818 let kernel_path = direct_kernel_boot_path(); 6819 6820 let mut child = GuestCommand::new(&guest) 6821 .args(["--cpus", "boot=2"]) 6822 .args(["--memory", "size=512M,hugepages=on"]) 6823 .args(["--kernel", kernel_path.to_str().unwrap()]) 6824 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6825 .default_disks() 6826 .default_net() 6827 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6828 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 6829 .args(["--api-socket", &api_socket]) 6830 .capture_output() 6831 .spawn() 6832 .unwrap(); 6833 6834 let r = std::panic::catch_unwind(|| { 6835 guest.wait_vm_boot(None).unwrap(); 6836 6837 // Check both if /dev/vdc exists and if the block size is 128M. 6838 assert_eq!( 6839 guest 6840 .ssh_command("lsblk | grep vdc | grep -c 128M") 6841 .unwrap() 6842 .trim() 6843 .parse::<u32>() 6844 .unwrap_or_default(), 6845 1 6846 ); 6847 6848 // Check the content of the block device after we wrote to it. 6849 // The vpda-sim-blk should let us read what we previously wrote. 6850 guest 6851 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6852 .unwrap(); 6853 assert_eq!( 6854 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6855 "foobar" 6856 ); 6857 6858 // Hotplug an extra vDPA block device behind the vIOMMU 6859 // Add a new vDPA device to the VM 6860 let (cmd_success, cmd_output) = remote_command_w_output( 6861 &api_socket, 6862 "add-vdpa", 6863 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6864 ); 6865 assert!(cmd_success); 6866 assert!(String::from_utf8_lossy(&cmd_output) 6867 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6868 6869 thread::sleep(std::time::Duration::new(10, 0)); 6870 6871 // Check IOMMU setup 6872 assert!(guest 6873 .does_device_vendor_pair_match("0x1057", "0x1af4") 6874 .unwrap_or_default()); 6875 assert_eq!( 6876 guest 6877 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6878 .unwrap() 6879 .trim(), 6880 "0001:00:01.0" 6881 ); 6882 6883 // Check both if /dev/vdd exists and if the block size is 128M. 6884 assert_eq!( 6885 guest 6886 .ssh_command("lsblk | grep vdd | grep -c 128M") 6887 .unwrap() 6888 .trim() 6889 .parse::<u32>() 6890 .unwrap_or_default(), 6891 1 6892 ); 6893 6894 // Write some content to the block device we've just plugged. 6895 guest 6896 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6897 .unwrap(); 6898 6899 // Check we can read the content back. 6900 assert_eq!( 6901 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6902 "foobar" 6903 ); 6904 6905 // Unplug the device 6906 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6907 assert!(cmd_success); 6908 thread::sleep(std::time::Duration::new(10, 0)); 6909 6910 // Check /dev/vdd doesn't exist anymore 6911 assert_eq!( 6912 guest 6913 .ssh_command("lsblk | grep -c vdd || true") 6914 .unwrap() 6915 .trim() 6916 .parse::<u32>() 6917 .unwrap_or(1), 6918 0 6919 ); 6920 }); 6921 6922 kill_child(&mut child); 6923 let output = child.wait_with_output().unwrap(); 6924 6925 handle_child_output(r, &output); 6926 } 6927 6928 #[test] 6929 #[cfg(target_arch = "x86_64")] 6930 #[ignore = "See #5756"] 6931 fn test_vdpa_net() { 6932 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6933 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6934 return; 6935 } 6936 6937 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6938 let guest = Guest::new(Box::new(focal)); 6939 6940 let kernel_path = direct_kernel_boot_path(); 6941 6942 let mut child = GuestCommand::new(&guest) 6943 .args(["--cpus", "boot=2"]) 6944 .args(["--memory", "size=512M,hugepages=on"]) 6945 .args(["--kernel", kernel_path.to_str().unwrap()]) 6946 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6947 .default_disks() 6948 .default_net() 6949 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 6950 .capture_output() 6951 .spawn() 6952 .unwrap(); 6953 6954 let r = std::panic::catch_unwind(|| { 6955 guest.wait_vm_boot(None).unwrap(); 6956 6957 // Check we can find network interface related to vDPA device 6958 assert_eq!( 6959 guest 6960 .ssh_command("ip -o link | grep -c ens6") 6961 .unwrap() 6962 .trim() 6963 .parse::<u32>() 6964 .unwrap_or(0), 6965 1 6966 ); 6967 6968 guest 6969 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 6970 .unwrap(); 6971 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 6972 6973 // Check there is no packet yet on both TX/RX of the network interface 6974 assert_eq!( 6975 guest 6976 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 6977 .unwrap() 6978 .trim() 6979 .parse::<u32>() 6980 .unwrap_or(0), 6981 2 6982 ); 6983 6984 // Send 6 packets with ping command 6985 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 6986 6987 // Check we can find 6 packets on both TX/RX of the network interface 6988 assert_eq!( 6989 guest 6990 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 6991 .unwrap() 6992 .trim() 6993 .parse::<u32>() 6994 .unwrap_or(0), 6995 2 6996 ); 6997 6998 // No need to check for hotplug as we already tested it through 6999 // test_vdpa_block() 7000 }); 7001 7002 kill_child(&mut child); 7003 let output = child.wait_with_output().unwrap(); 7004 7005 handle_child_output(r, &output); 7006 } 7007 7008 #[test] 7009 #[cfg(target_arch = "x86_64")] 7010 fn test_tpm() { 7011 let focal = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7012 let guest = Guest::new(Box::new(focal)); 7013 7014 let (mut swtpm_command, swtpm_socket_path) = prepare_swtpm_daemon(&guest.tmp_dir); 7015 7016 let mut guest_cmd = GuestCommand::new(&guest); 7017 guest_cmd 7018 .args(["--cpus", "boot=1"]) 7019 .args(["--memory", "size=512M"]) 7020 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7021 .args(["--tpm", &format!("socket={swtpm_socket_path}")]) 7022 .capture_output() 7023 .default_disks() 7024 .default_net(); 7025 7026 // Start swtpm daemon 7027 let mut swtpm_child = swtpm_command.spawn().unwrap(); 7028 thread::sleep(std::time::Duration::new(10, 0)); 7029 let mut child = guest_cmd.spawn().unwrap(); 7030 let r = std::panic::catch_unwind(|| { 7031 guest.wait_vm_boot(None).unwrap(); 7032 assert_eq!( 7033 guest.ssh_command("ls /dev/tpm0").unwrap().trim(), 7034 "/dev/tpm0" 7035 ); 7036 guest.ssh_command("sudo tpm2_selftest -f").unwrap(); 7037 guest 7038 .ssh_command("echo 'hello' > /tmp/checksum_test; ") 7039 .unwrap(); 7040 guest.ssh_command("cmp <(sudo tpm2_pcrevent /tmp/checksum_test | grep sha256 | awk '{print $2}') <(sha256sum /tmp/checksum_test| awk '{print $1}')").unwrap(); 7041 }); 7042 7043 let _ = swtpm_child.kill(); 7044 let _d_out = swtpm_child.wait_with_output().unwrap(); 7045 7046 kill_child(&mut child); 7047 let output = child.wait_with_output().unwrap(); 7048 7049 handle_child_output(r, &output); 7050 } 7051 7052 #[test] 7053 #[cfg(target_arch = "x86_64")] 7054 fn test_double_tty() { 7055 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7056 let guest = Guest::new(Box::new(focal)); 7057 let mut cmd = GuestCommand::new(&guest); 7058 let api_socket = temp_api_path(&guest.tmp_dir); 7059 let tty_str: &str = "console=hvc0 earlyprintk=ttyS0 "; 7060 // linux printk module enable console log. 7061 let con_dis_str: &str = "console [hvc0] enabled"; 7062 // linux printk module disable console log. 7063 let con_enb_str: &str = "bootconsole [earlyser0] disabled"; 7064 7065 let kernel_path = direct_kernel_boot_path(); 7066 7067 cmd.args(["--cpus", "boot=1"]) 7068 .args(["--memory", "size=512M"]) 7069 .args(["--kernel", kernel_path.to_str().unwrap()]) 7070 .args([ 7071 "--cmdline", 7072 DIRECT_KERNEL_BOOT_CMDLINE 7073 .replace("console=hvc0 ", tty_str) 7074 .as_str(), 7075 ]) 7076 .capture_output() 7077 .default_disks() 7078 .default_net() 7079 .args(["--serial", "tty"]) 7080 .args(["--console", "tty"]) 7081 .args(["--api-socket", &api_socket]); 7082 7083 let mut child = cmd.spawn().unwrap(); 7084 7085 let mut r = std::panic::catch_unwind(|| { 7086 guest.wait_vm_boot(None).unwrap(); 7087 }); 7088 7089 kill_child(&mut child); 7090 let output = child.wait_with_output().unwrap(); 7091 7092 if r.is_ok() { 7093 r = std::panic::catch_unwind(|| { 7094 let s = String::from_utf8_lossy(&output.stdout); 7095 assert!(s.contains(tty_str)); 7096 assert!(s.contains(con_dis_str)); 7097 assert!(s.contains(con_enb_str)); 7098 }); 7099 } 7100 7101 handle_child_output(r, &output); 7102 } 7103 7104 #[test] 7105 #[cfg(target_arch = "x86_64")] 7106 fn test_nmi() { 7107 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7108 let guest = Guest::new(Box::new(jammy)); 7109 let api_socket = temp_api_path(&guest.tmp_dir); 7110 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7111 7112 let kernel_path = direct_kernel_boot_path(); 7113 let cmd_line = format!("{} {}", DIRECT_KERNEL_BOOT_CMDLINE, "unknown_nmi_panic=1"); 7114 7115 let mut cmd = GuestCommand::new(&guest); 7116 cmd.args(["--cpus", "boot=4"]) 7117 .args(["--memory", "size=512M"]) 7118 .args(["--kernel", kernel_path.to_str().unwrap()]) 7119 .args(["--cmdline", cmd_line.as_str()]) 7120 .default_disks() 7121 .args(["--net", guest.default_net_string().as_str()]) 7122 .args(["--pvpanic"]) 7123 .args(["--api-socket", &api_socket]) 7124 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7125 .capture_output(); 7126 7127 let mut child = cmd.spawn().unwrap(); 7128 7129 let r = std::panic::catch_unwind(|| { 7130 guest.wait_vm_boot(None).unwrap(); 7131 7132 assert!(remote_command(&api_socket, "nmi", None)); 7133 7134 // Wait a while for guest 7135 thread::sleep(std::time::Duration::new(3, 0)); 7136 7137 let expected_sequential_events = [&MetaEvent { 7138 event: "panic".to_string(), 7139 device_id: None, 7140 }]; 7141 assert!(check_latest_events_exact( 7142 &expected_sequential_events, 7143 &event_path 7144 )); 7145 }); 7146 7147 kill_child(&mut child); 7148 let output = child.wait_with_output().unwrap(); 7149 7150 handle_child_output(r, &output); 7151 } 7152 } 7153 7154 mod dbus_api { 7155 use crate::*; 7156 7157 // Start cloud-hypervisor with no VM parameters, running both the HTTP 7158 // and DBus APIs. Alternate calls to the external APIs (HTTP and DBus) 7159 // to create a VM, boot it, and verify that it can be shut down and then 7160 // booted again. 7161 #[test] 7162 fn test_api_dbus_and_http_interleaved() { 7163 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7164 let guest = Guest::new(Box::new(focal)); 7165 let dbus_api = TargetApi::new_dbus_api(&guest.tmp_dir); 7166 let http_api = TargetApi::new_http_api(&guest.tmp_dir); 7167 7168 let mut child = GuestCommand::new(&guest) 7169 .args(dbus_api.guest_args()) 7170 .args(http_api.guest_args()) 7171 .capture_output() 7172 .spawn() 7173 .unwrap(); 7174 7175 thread::sleep(std::time::Duration::new(1, 0)); 7176 7177 // Verify API servers are running 7178 assert!(dbus_api.remote_command("ping", None)); 7179 assert!(http_api.remote_command("ping", None)); 7180 7181 // Create the VM first 7182 let cpu_count: u8 = 4; 7183 let request_body = guest.api_create_body( 7184 cpu_count, 7185 direct_kernel_boot_path().to_str().unwrap(), 7186 DIRECT_KERNEL_BOOT_CMDLINE, 7187 ); 7188 7189 let temp_config_path = guest.tmp_dir.as_path().join("config"); 7190 std::fs::write(&temp_config_path, request_body).unwrap(); 7191 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 7192 7193 let r = std::panic::catch_unwind(|| { 7194 // Create the VM 7195 assert!(dbus_api.remote_command("create", Some(create_config),)); 7196 7197 // Then boot it 7198 assert!(http_api.remote_command("boot", None)); 7199 guest.wait_vm_boot(None).unwrap(); 7200 7201 // Check that the VM booted as expected 7202 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7203 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7204 7205 // Sync and shutdown without powering off to prevent filesystem 7206 // corruption. 7207 guest.ssh_command("sync").unwrap(); 7208 guest.ssh_command("sudo shutdown -H now").unwrap(); 7209 7210 // Wait for the guest to be fully shutdown 7211 thread::sleep(std::time::Duration::new(20, 0)); 7212 7213 // Then shutdown the VM 7214 assert!(dbus_api.remote_command("shutdown", None)); 7215 7216 // Then boot it again 7217 assert!(http_api.remote_command("boot", None)); 7218 guest.wait_vm_boot(None).unwrap(); 7219 7220 // Check that the VM booted as expected 7221 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7222 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7223 }); 7224 7225 kill_child(&mut child); 7226 let output = child.wait_with_output().unwrap(); 7227 7228 handle_child_output(r, &output); 7229 } 7230 7231 #[test] 7232 fn test_api_dbus_create_boot() { 7233 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7234 let guest = Guest::new(Box::new(focal)); 7235 7236 _test_api_create_boot(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7237 } 7238 7239 #[test] 7240 fn test_api_dbus_shutdown() { 7241 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7242 let guest = Guest::new(Box::new(focal)); 7243 7244 _test_api_shutdown(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7245 } 7246 7247 #[test] 7248 fn test_api_dbus_delete() { 7249 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7250 let guest = Guest::new(Box::new(focal)); 7251 7252 _test_api_delete(TargetApi::new_dbus_api(&guest.tmp_dir), guest); 7253 } 7254 7255 #[test] 7256 fn test_api_dbus_pause_resume() { 7257 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7258 let guest = Guest::new(Box::new(focal)); 7259 7260 _test_api_pause_resume(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7261 } 7262 } 7263 7264 mod common_sequential { 7265 use std::fs::remove_dir_all; 7266 7267 use crate::*; 7268 7269 #[test] 7270 #[cfg(not(feature = "mshv"))] 7271 fn test_memory_mergeable_on() { 7272 test_memory_mergeable(true) 7273 } 7274 7275 fn snapshot_and_check_events(api_socket: &str, snapshot_dir: &str, event_path: &str) { 7276 // Pause the VM 7277 assert!(remote_command(api_socket, "pause", None)); 7278 let latest_events: [&MetaEvent; 2] = [ 7279 &MetaEvent { 7280 event: "pausing".to_string(), 7281 device_id: None, 7282 }, 7283 &MetaEvent { 7284 event: "paused".to_string(), 7285 device_id: None, 7286 }, 7287 ]; 7288 // See: #5938 7289 thread::sleep(std::time::Duration::new(1, 0)); 7290 assert!(check_latest_events_exact(&latest_events, event_path)); 7291 7292 // Take a snapshot from the VM 7293 assert!(remote_command( 7294 api_socket, 7295 "snapshot", 7296 Some(format!("file://{snapshot_dir}").as_str()), 7297 )); 7298 7299 // Wait to make sure the snapshot is completed 7300 thread::sleep(std::time::Duration::new(10, 0)); 7301 7302 let latest_events = [ 7303 &MetaEvent { 7304 event: "snapshotting".to_string(), 7305 device_id: None, 7306 }, 7307 &MetaEvent { 7308 event: "snapshotted".to_string(), 7309 device_id: None, 7310 }, 7311 ]; 7312 // See: #5938 7313 thread::sleep(std::time::Duration::new(1, 0)); 7314 assert!(check_latest_events_exact(&latest_events, event_path)); 7315 } 7316 7317 // One thing to note about this test. The virtio-net device is heavily used 7318 // through each ssh command. There's no need to perform a dedicated test to 7319 // verify the migration went well for virtio-net. 7320 #[test] 7321 #[cfg(not(feature = "mshv"))] 7322 fn test_snapshot_restore_hotplug_virtiomem() { 7323 _test_snapshot_restore(true); 7324 } 7325 7326 #[test] 7327 fn test_snapshot_restore_basic() { 7328 _test_snapshot_restore(false); 7329 } 7330 7331 fn _test_snapshot_restore(use_hotplug: bool) { 7332 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7333 let guest = Guest::new(Box::new(focal)); 7334 let kernel_path = direct_kernel_boot_path(); 7335 7336 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7337 7338 let net_id = "net123"; 7339 let net_params = format!( 7340 "id={},tap=,mac={},ip={},mask=255.255.255.0", 7341 net_id, guest.network.guest_mac, guest.network.host_ip 7342 ); 7343 let mut mem_params = "size=2G"; 7344 7345 if use_hotplug { 7346 mem_params = "size=2G,hotplug_method=virtio-mem,hotplug_size=32G" 7347 } 7348 7349 let cloudinit_params = format!( 7350 "path={},iommu=on", 7351 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7352 ); 7353 7354 let socket = temp_vsock_path(&guest.tmp_dir); 7355 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7356 7357 let mut child = GuestCommand::new(&guest) 7358 .args(["--api-socket", &api_socket_source]) 7359 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7360 .args(["--cpus", "boot=4"]) 7361 .args(["--memory", mem_params]) 7362 .args(["--balloon", "size=0"]) 7363 .args(["--kernel", kernel_path.to_str().unwrap()]) 7364 .args([ 7365 "--disk", 7366 format!( 7367 "path={}", 7368 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7369 ) 7370 .as_str(), 7371 cloudinit_params.as_str(), 7372 ]) 7373 .args(["--net", net_params.as_str()]) 7374 .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) 7375 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7376 .capture_output() 7377 .spawn() 7378 .unwrap(); 7379 7380 let console_text = String::from("On a branch floating down river a cricket, singing."); 7381 // Create the snapshot directory 7382 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7383 7384 let r = std::panic::catch_unwind(|| { 7385 guest.wait_vm_boot(None).unwrap(); 7386 7387 // Check the number of vCPUs 7388 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 7389 // Check the guest RAM 7390 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 7391 if use_hotplug { 7392 // Increase guest RAM with virtio-mem 7393 resize_command( 7394 &api_socket_source, 7395 None, 7396 Some(6 << 30), 7397 None, 7398 Some(&event_path), 7399 ); 7400 thread::sleep(std::time::Duration::new(5, 0)); 7401 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7402 // Use balloon to remove RAM from the VM 7403 resize_command( 7404 &api_socket_source, 7405 None, 7406 None, 7407 Some(1 << 30), 7408 Some(&event_path), 7409 ); 7410 thread::sleep(std::time::Duration::new(5, 0)); 7411 let total_memory = guest.get_total_memory().unwrap_or_default(); 7412 assert!(total_memory > 4_800_000); 7413 assert!(total_memory < 5_760_000); 7414 } 7415 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 7416 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7417 7418 // x86_64: We check that removing and adding back the virtio-net device 7419 // does not break the snapshot/restore support for virtio-pci. 7420 // This is an important thing to test as the hotplug will 7421 // trigger a PCI BAR reprogramming, which is a good way of 7422 // checking if the stored resources are correctly restored. 7423 // Unplug the virtio-net device 7424 // AArch64: Device hotplug is currently not supported, skipping here. 7425 #[cfg(target_arch = "x86_64")] 7426 { 7427 assert!(remote_command( 7428 &api_socket_source, 7429 "remove-device", 7430 Some(net_id), 7431 )); 7432 thread::sleep(std::time::Duration::new(10, 0)); 7433 let latest_events = [&MetaEvent { 7434 event: "device-removed".to_string(), 7435 device_id: Some(net_id.to_string()), 7436 }]; 7437 // See: #5938 7438 thread::sleep(std::time::Duration::new(1, 0)); 7439 assert!(check_latest_events_exact(&latest_events, &event_path)); 7440 7441 // Plug the virtio-net device again 7442 assert!(remote_command( 7443 &api_socket_source, 7444 "add-net", 7445 Some(net_params.as_str()), 7446 )); 7447 thread::sleep(std::time::Duration::new(10, 0)); 7448 } 7449 7450 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7451 }); 7452 7453 // Shutdown the source VM and check console output 7454 kill_child(&mut child); 7455 let output = child.wait_with_output().unwrap(); 7456 handle_child_output(r, &output); 7457 7458 let r = std::panic::catch_unwind(|| { 7459 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7460 }); 7461 7462 handle_child_output(r, &output); 7463 7464 // Remove the vsock socket file. 7465 Command::new("rm") 7466 .arg("-f") 7467 .arg(socket.as_str()) 7468 .output() 7469 .unwrap(); 7470 7471 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7472 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7473 7474 // Restore the VM from the snapshot 7475 let mut child = GuestCommand::new(&guest) 7476 .args(["--api-socket", &api_socket_restored]) 7477 .args([ 7478 "--event-monitor", 7479 format!("path={event_path_restored}").as_str(), 7480 ]) 7481 .args([ 7482 "--restore", 7483 format!("source_url=file://{snapshot_dir}").as_str(), 7484 ]) 7485 .capture_output() 7486 .spawn() 7487 .unwrap(); 7488 7489 // Wait for the VM to be restored 7490 thread::sleep(std::time::Duration::new(20, 0)); 7491 let expected_events = [ 7492 &MetaEvent { 7493 event: "starting".to_string(), 7494 device_id: None, 7495 }, 7496 &MetaEvent { 7497 event: "activated".to_string(), 7498 device_id: Some("__console".to_string()), 7499 }, 7500 &MetaEvent { 7501 event: "activated".to_string(), 7502 device_id: Some("__rng".to_string()), 7503 }, 7504 &MetaEvent { 7505 event: "restoring".to_string(), 7506 device_id: None, 7507 }, 7508 ]; 7509 assert!(check_sequential_events( 7510 &expected_events, 7511 &event_path_restored 7512 )); 7513 let latest_events = [&MetaEvent { 7514 event: "restored".to_string(), 7515 device_id: None, 7516 }]; 7517 assert!(check_latest_events_exact( 7518 &latest_events, 7519 &event_path_restored 7520 )); 7521 7522 // Remove the snapshot dir 7523 let _ = remove_dir_all(snapshot_dir.as_str()); 7524 7525 let r = std::panic::catch_unwind(|| { 7526 // Resume the VM 7527 assert!(remote_command(&api_socket_restored, "resume", None)); 7528 // There is no way that we can ensure the 'write()' to the 7529 // event file is completed when the 'resume' request is 7530 // returned successfully, because the 'write()' was done 7531 // asynchronously from a different thread of Cloud 7532 // Hypervisor (e.g. the event-monitor thread). 7533 thread::sleep(std::time::Duration::new(1, 0)); 7534 let latest_events = [ 7535 &MetaEvent { 7536 event: "resuming".to_string(), 7537 device_id: None, 7538 }, 7539 &MetaEvent { 7540 event: "resumed".to_string(), 7541 device_id: None, 7542 }, 7543 ]; 7544 assert!(check_latest_events_exact( 7545 &latest_events, 7546 &event_path_restored 7547 )); 7548 7549 // Perform same checks to validate VM has been properly restored 7550 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 7551 let total_memory = guest.get_total_memory().unwrap_or_default(); 7552 if !use_hotplug { 7553 assert!(total_memory > 1_920_000); 7554 } else { 7555 assert!(total_memory > 4_800_000); 7556 assert!(total_memory < 5_760_000); 7557 // Deflate balloon to restore entire RAM to the VM 7558 resize_command(&api_socket_restored, None, None, Some(0), None); 7559 thread::sleep(std::time::Duration::new(5, 0)); 7560 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7561 // Decrease guest RAM with virtio-mem 7562 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 7563 thread::sleep(std::time::Duration::new(5, 0)); 7564 let total_memory = guest.get_total_memory().unwrap_or_default(); 7565 assert!(total_memory > 4_800_000); 7566 assert!(total_memory < 5_760_000); 7567 } 7568 7569 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7570 }); 7571 // Shutdown the target VM and check console output 7572 kill_child(&mut child); 7573 let output = child.wait_with_output().unwrap(); 7574 handle_child_output(r, &output); 7575 7576 let r = std::panic::catch_unwind(|| { 7577 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7578 }); 7579 7580 handle_child_output(r, &output); 7581 } 7582 7583 #[test] 7584 fn test_snapshot_restore_with_fd() { 7585 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7586 let guest = Guest::new(Box::new(focal)); 7587 let kernel_path = direct_kernel_boot_path(); 7588 7589 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7590 7591 let net_id = "net123"; 7592 let num_queue_pairs: usize = 2; 7593 // use a name that does not conflict with tap dev created from other tests 7594 let tap_name = "chtap999"; 7595 use std::str::FromStr; 7596 let taps = net_util::open_tap( 7597 Some(tap_name), 7598 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 7599 None, 7600 &mut None, 7601 None, 7602 num_queue_pairs, 7603 Some(libc::O_RDWR | libc::O_NONBLOCK), 7604 ) 7605 .unwrap(); 7606 let net_params = format!( 7607 "id={},fd=[{},{}],mac={},ip={},mask=255.255.255.0,num_queues={}", 7608 net_id, 7609 taps[0].as_raw_fd(), 7610 taps[1].as_raw_fd(), 7611 guest.network.guest_mac, 7612 guest.network.host_ip, 7613 num_queue_pairs * 2 7614 ); 7615 7616 let cloudinit_params = format!( 7617 "path={},iommu=on", 7618 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7619 ); 7620 7621 let n_cpu = 2; 7622 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7623 7624 let mut child = GuestCommand::new(&guest) 7625 .args(["--api-socket", &api_socket_source]) 7626 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7627 .args(["--cpus", format!("boot={}", n_cpu).as_str()]) 7628 .args(["--memory", "size=1G"]) 7629 .args(["--kernel", kernel_path.to_str().unwrap()]) 7630 .args([ 7631 "--disk", 7632 format!( 7633 "path={}", 7634 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7635 ) 7636 .as_str(), 7637 cloudinit_params.as_str(), 7638 ]) 7639 .args(["--net", net_params.as_str()]) 7640 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7641 .capture_output() 7642 .spawn() 7643 .unwrap(); 7644 7645 let console_text = String::from("On a branch floating down river a cricket, singing."); 7646 // Create the snapshot directory 7647 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7648 7649 let r = std::panic::catch_unwind(|| { 7650 guest.wait_vm_boot(None).unwrap(); 7651 7652 // close the fds after VM boots, as CH duplicates them before using 7653 for tap in taps.iter() { 7654 unsafe { libc::close(tap.as_raw_fd()) }; 7655 } 7656 7657 // Check the number of vCPUs 7658 assert_eq!(guest.get_cpu_count().unwrap_or_default(), n_cpu); 7659 // Check the guest RAM 7660 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 7661 7662 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 7663 guest.check_devices_common(None, Some(&console_text), None); 7664 7665 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7666 }); 7667 7668 // Shutdown the source VM and check console output 7669 kill_child(&mut child); 7670 let output = child.wait_with_output().unwrap(); 7671 handle_child_output(r, &output); 7672 7673 let r = std::panic::catch_unwind(|| { 7674 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7675 }); 7676 7677 handle_child_output(r, &output); 7678 7679 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7680 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7681 7682 // Restore the VM from the snapshot 7683 let mut child = GuestCommand::new(&guest) 7684 .args(["--api-socket", &api_socket_restored]) 7685 .args([ 7686 "--event-monitor", 7687 format!("path={event_path_restored}").as_str(), 7688 ]) 7689 .capture_output() 7690 .spawn() 7691 .unwrap(); 7692 thread::sleep(std::time::Duration::new(2, 0)); 7693 7694 let taps = net_util::open_tap( 7695 Some(tap_name), 7696 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 7697 None, 7698 &mut None, 7699 None, 7700 num_queue_pairs, 7701 Some(libc::O_RDWR | libc::O_NONBLOCK), 7702 ) 7703 .unwrap(); 7704 let restore_params = format!( 7705 "source_url=file://{},net_fds=[{}@[{},{}]]", 7706 snapshot_dir, 7707 net_id, 7708 taps[0].as_raw_fd(), 7709 taps[1].as_raw_fd() 7710 ); 7711 assert!(remote_command( 7712 &api_socket_restored, 7713 "restore", 7714 Some(restore_params.as_str()) 7715 )); 7716 7717 // Wait for the VM to be restored 7718 thread::sleep(std::time::Duration::new(20, 0)); 7719 7720 // close the fds as CH duplicates them before using 7721 for tap in taps.iter() { 7722 unsafe { libc::close(tap.as_raw_fd()) }; 7723 } 7724 7725 let expected_events = [ 7726 &MetaEvent { 7727 event: "starting".to_string(), 7728 device_id: None, 7729 }, 7730 &MetaEvent { 7731 event: "activated".to_string(), 7732 device_id: Some("__console".to_string()), 7733 }, 7734 &MetaEvent { 7735 event: "activated".to_string(), 7736 device_id: Some("__rng".to_string()), 7737 }, 7738 &MetaEvent { 7739 event: "restoring".to_string(), 7740 device_id: None, 7741 }, 7742 ]; 7743 assert!(check_sequential_events( 7744 &expected_events, 7745 &event_path_restored 7746 )); 7747 let latest_events = [&MetaEvent { 7748 event: "restored".to_string(), 7749 device_id: None, 7750 }]; 7751 assert!(check_latest_events_exact( 7752 &latest_events, 7753 &event_path_restored 7754 )); 7755 7756 // Remove the snapshot dir 7757 let _ = remove_dir_all(snapshot_dir.as_str()); 7758 7759 let r = std::panic::catch_unwind(|| { 7760 // Resume the VM 7761 assert!(remote_command(&api_socket_restored, "resume", None)); 7762 // There is no way that we can ensure the 'write()' to the 7763 // event file is completed when the 'resume' request is 7764 // returned successfully, because the 'write()' was done 7765 // asynchronously from a different thread of Cloud 7766 // Hypervisor (e.g. the event-monitor thread). 7767 thread::sleep(std::time::Duration::new(1, 0)); 7768 let latest_events = [ 7769 &MetaEvent { 7770 event: "resuming".to_string(), 7771 device_id: None, 7772 }, 7773 &MetaEvent { 7774 event: "resumed".to_string(), 7775 device_id: None, 7776 }, 7777 ]; 7778 assert!(check_latest_events_exact( 7779 &latest_events, 7780 &event_path_restored 7781 )); 7782 7783 // Perform same checks to validate VM has been properly restored 7784 assert_eq!(guest.get_cpu_count().unwrap_or_default(), n_cpu); 7785 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 7786 7787 guest.check_devices_common(None, Some(&console_text), None); 7788 }); 7789 // Shutdown the target VM and check console output 7790 kill_child(&mut child); 7791 let output = child.wait_with_output().unwrap(); 7792 handle_child_output(r, &output); 7793 7794 let r = std::panic::catch_unwind(|| { 7795 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7796 }); 7797 7798 handle_child_output(r, &output); 7799 } 7800 7801 #[test] 7802 #[cfg(target_arch = "x86_64")] 7803 fn test_snapshot_restore_pvpanic() { 7804 _test_snapshot_restore_devices(true); 7805 } 7806 7807 fn _test_snapshot_restore_devices(pvpanic: bool) { 7808 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7809 let guest = Guest::new(Box::new(focal)); 7810 let kernel_path = direct_kernel_boot_path(); 7811 7812 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7813 7814 let device_params = { 7815 let mut data = vec![]; 7816 if pvpanic { 7817 data.push("--pvpanic"); 7818 } 7819 data 7820 }; 7821 7822 let socket = temp_vsock_path(&guest.tmp_dir); 7823 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7824 7825 let mut child = GuestCommand::new(&guest) 7826 .args(["--api-socket", &api_socket_source]) 7827 .args(["--event-monitor", format!("path={}", event_path).as_str()]) 7828 .args(["--cpus", "boot=2"]) 7829 .args(["--memory", "size=1G"]) 7830 .args(["--kernel", kernel_path.to_str().unwrap()]) 7831 .default_disks() 7832 .default_net() 7833 .args(["--vsock", format!("cid=3,socket={}", socket).as_str()]) 7834 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7835 .args(device_params) 7836 .capture_output() 7837 .spawn() 7838 .unwrap(); 7839 7840 let console_text = String::from("On a branch floating down river a cricket, singing."); 7841 // Create the snapshot directory 7842 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7843 7844 let r = std::panic::catch_unwind(|| { 7845 guest.wait_vm_boot(None).unwrap(); 7846 7847 // Check the number of vCPUs 7848 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 7849 7850 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7851 }); 7852 7853 // Shutdown the source VM and check console output 7854 kill_child(&mut child); 7855 let output = child.wait_with_output().unwrap(); 7856 handle_child_output(r, &output); 7857 7858 // Remove the vsock socket file. 7859 Command::new("rm") 7860 .arg("-f") 7861 .arg(socket.as_str()) 7862 .output() 7863 .unwrap(); 7864 7865 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7866 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7867 7868 // Restore the VM from the snapshot 7869 let mut child = GuestCommand::new(&guest) 7870 .args(["--api-socket", &api_socket_restored]) 7871 .args([ 7872 "--event-monitor", 7873 format!("path={event_path_restored}").as_str(), 7874 ]) 7875 .args([ 7876 "--restore", 7877 format!("source_url=file://{snapshot_dir}").as_str(), 7878 ]) 7879 .capture_output() 7880 .spawn() 7881 .unwrap(); 7882 7883 // Wait for the VM to be restored 7884 thread::sleep(std::time::Duration::new(20, 0)); 7885 7886 let latest_events = [&MetaEvent { 7887 event: "restored".to_string(), 7888 device_id: None, 7889 }]; 7890 assert!(check_latest_events_exact( 7891 &latest_events, 7892 &event_path_restored 7893 )); 7894 7895 // Remove the snapshot dir 7896 let _ = remove_dir_all(snapshot_dir.as_str()); 7897 7898 let r = std::panic::catch_unwind(|| { 7899 // Resume the VM 7900 assert!(remote_command(&api_socket_restored, "resume", None)); 7901 // There is no way that we can ensure the 'write()' to the 7902 // event file is completed when the 'resume' request is 7903 // returned successfully, because the 'write()' was done 7904 // asynchronously from a different thread of Cloud 7905 // Hypervisor (e.g. the event-monitor thread). 7906 thread::sleep(std::time::Duration::new(1, 0)); 7907 let latest_events = [ 7908 &MetaEvent { 7909 event: "resuming".to_string(), 7910 device_id: None, 7911 }, 7912 &MetaEvent { 7913 event: "resumed".to_string(), 7914 device_id: None, 7915 }, 7916 ]; 7917 assert!(check_latest_events_exact( 7918 &latest_events, 7919 &event_path_restored 7920 )); 7921 7922 // Check the number of vCPUs 7923 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 7924 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7925 7926 if pvpanic { 7927 // Trigger guest a panic 7928 make_guest_panic(&guest); 7929 // Wait a while for guest 7930 thread::sleep(std::time::Duration::new(10, 0)); 7931 7932 let expected_sequential_events = [&MetaEvent { 7933 event: "panic".to_string(), 7934 device_id: None, 7935 }]; 7936 assert!(check_latest_events_exact( 7937 &expected_sequential_events, 7938 &event_path_restored 7939 )); 7940 } 7941 }); 7942 // Shutdown the target VM and check console output 7943 kill_child(&mut child); 7944 let output = child.wait_with_output().unwrap(); 7945 handle_child_output(r, &output); 7946 7947 let r = std::panic::catch_unwind(|| { 7948 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7949 }); 7950 7951 handle_child_output(r, &output); 7952 } 7953 } 7954 7955 mod windows { 7956 use once_cell::sync::Lazy; 7957 7958 use crate::*; 7959 7960 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 7961 7962 struct WindowsGuest { 7963 guest: Guest, 7964 auth: PasswordAuth, 7965 } 7966 7967 trait FsType { 7968 const FS_FAT: u8; 7969 const FS_NTFS: u8; 7970 } 7971 impl FsType for WindowsGuest { 7972 const FS_FAT: u8 = 0; 7973 const FS_NTFS: u8 = 1; 7974 } 7975 7976 impl WindowsGuest { 7977 fn new() -> Self { 7978 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 7979 let guest = Guest::new(Box::new(disk)); 7980 let auth = PasswordAuth { 7981 username: String::from("administrator"), 7982 password: String::from("Admin123"), 7983 }; 7984 7985 WindowsGuest { guest, auth } 7986 } 7987 7988 fn guest(&self) -> &Guest { 7989 &self.guest 7990 } 7991 7992 fn ssh_cmd(&self, cmd: &str) -> String { 7993 ssh_command_ip_with_auth( 7994 cmd, 7995 &self.auth, 7996 &self.guest.network.guest_ip, 7997 DEFAULT_SSH_RETRIES, 7998 DEFAULT_SSH_TIMEOUT, 7999 ) 8000 .unwrap() 8001 } 8002 8003 fn cpu_count(&self) -> u8 { 8004 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 8005 .trim() 8006 .parse::<u8>() 8007 .unwrap_or(0) 8008 } 8009 8010 fn ram_size(&self) -> usize { 8011 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 8012 .trim() 8013 .parse::<usize>() 8014 .unwrap_or(0) 8015 } 8016 8017 fn netdev_count(&self) -> u8 { 8018 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 8019 .trim() 8020 .parse::<u8>() 8021 .unwrap_or(0) 8022 } 8023 8024 fn disk_count(&self) -> u8 { 8025 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 8026 .trim() 8027 .parse::<u8>() 8028 .unwrap_or(0) 8029 } 8030 8031 fn reboot(&self) { 8032 let _ = self.ssh_cmd("shutdown /r /t 0"); 8033 } 8034 8035 fn shutdown(&self) { 8036 let _ = self.ssh_cmd("shutdown /s /t 0"); 8037 } 8038 8039 fn run_dnsmasq(&self) -> std::process::Child { 8040 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 8041 let dhcp_host = format!( 8042 "--dhcp-host={},{}", 8043 self.guest.network.guest_mac, self.guest.network.guest_ip 8044 ); 8045 let dhcp_range = format!( 8046 "--dhcp-range=eth,{},{}", 8047 self.guest.network.guest_ip, self.guest.network.guest_ip 8048 ); 8049 8050 Command::new("dnsmasq") 8051 .arg("--no-daemon") 8052 .arg("--log-queries") 8053 .arg(listen_address.as_str()) 8054 .arg("--except-interface=lo") 8055 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 8056 .arg("--conf-file=/dev/null") 8057 .arg(dhcp_host.as_str()) 8058 .arg(dhcp_range.as_str()) 8059 .spawn() 8060 .unwrap() 8061 } 8062 8063 // TODO Cleanup image file explicitly after test, if there's some space issues. 8064 fn disk_new(&self, fs: u8, sz: usize) -> String { 8065 let mut guard = NEXT_DISK_ID.lock().unwrap(); 8066 let id = *guard; 8067 *guard = id + 1; 8068 8069 let img = PathBuf::from(format!("/tmp/test-hotplug-{id}.raw")); 8070 let _ = fs::remove_file(&img); 8071 8072 // Create an image file 8073 let out = Command::new("qemu-img") 8074 .args([ 8075 "create", 8076 "-f", 8077 "raw", 8078 img.to_str().unwrap(), 8079 format!("{sz}m").as_str(), 8080 ]) 8081 .output() 8082 .expect("qemu-img command failed") 8083 .stdout; 8084 println!("{out:?}"); 8085 8086 // Associate image to a loop device 8087 let out = Command::new("losetup") 8088 .args(["--show", "-f", img.to_str().unwrap()]) 8089 .output() 8090 .expect("failed to create loop device") 8091 .stdout; 8092 let _tmp = String::from_utf8_lossy(&out); 8093 let loop_dev = _tmp.trim(); 8094 println!("{out:?}"); 8095 8096 // Create a partition table 8097 // echo 'type=7' | sudo sfdisk "${LOOP}" 8098 let mut child = Command::new("sfdisk") 8099 .args([loop_dev]) 8100 .stdin(Stdio::piped()) 8101 .spawn() 8102 .unwrap(); 8103 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 8104 stdin 8105 .write_all("type=7".as_bytes()) 8106 .expect("failed to write stdin"); 8107 let out = child.wait_with_output().expect("sfdisk failed").stdout; 8108 println!("{out:?}"); 8109 8110 // Disengage the loop device 8111 let out = Command::new("losetup") 8112 .args(["-d", loop_dev]) 8113 .output() 8114 .expect("loop device not found") 8115 .stdout; 8116 println!("{out:?}"); 8117 8118 // Re-associate loop device pointing to the partition only 8119 let out = Command::new("losetup") 8120 .args([ 8121 "--show", 8122 "--offset", 8123 (512 * 2048).to_string().as_str(), 8124 "-f", 8125 img.to_str().unwrap(), 8126 ]) 8127 .output() 8128 .expect("failed to create loop device") 8129 .stdout; 8130 let _tmp = String::from_utf8_lossy(&out); 8131 let loop_dev = _tmp.trim(); 8132 println!("{out:?}"); 8133 8134 // Create filesystem. 8135 let fs_cmd = match fs { 8136 WindowsGuest::FS_FAT => "mkfs.msdos", 8137 WindowsGuest::FS_NTFS => "mkfs.ntfs", 8138 _ => panic!("Unknown filesystem type '{fs}'"), 8139 }; 8140 let out = Command::new(fs_cmd) 8141 .args([&loop_dev]) 8142 .output() 8143 .unwrap_or_else(|_| panic!("{fs_cmd} failed")) 8144 .stdout; 8145 println!("{out:?}"); 8146 8147 // Disengage the loop device 8148 let out = Command::new("losetup") 8149 .args(["-d", loop_dev]) 8150 .output() 8151 .unwrap_or_else(|_| panic!("loop device '{loop_dev}' not found")) 8152 .stdout; 8153 println!("{out:?}"); 8154 8155 img.to_str().unwrap().to_string() 8156 } 8157 8158 fn disks_set_rw(&self) { 8159 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 8160 } 8161 8162 fn disks_online(&self) { 8163 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 8164 } 8165 8166 fn disk_file_put(&self, fname: &str, data: &str) { 8167 let _ = self.ssh_cmd(&format!( 8168 "powershell -Command \"'{data}' | Set-Content -Path {fname}\"" 8169 )); 8170 } 8171 8172 fn disk_file_read(&self, fname: &str) -> String { 8173 self.ssh_cmd(&format!( 8174 "powershell -Command \"Get-Content -Path {fname}\"" 8175 )) 8176 } 8177 8178 fn wait_for_boot(&self) -> bool { 8179 let cmd = "dir /b c:\\ | find \"Windows\""; 8180 let tmo_max = 180; 8181 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 8182 // interval must be small. 8183 let tmo_int = 2; 8184 let out = ssh_command_ip_with_auth( 8185 cmd, 8186 &self.auth, 8187 &self.guest.network.guest_ip, 8188 { 8189 let mut ret = 1; 8190 let mut tmo_acc = 0; 8191 loop { 8192 tmo_acc += tmo_int * ret; 8193 if tmo_acc >= tmo_max { 8194 break; 8195 } 8196 ret += 1; 8197 } 8198 ret 8199 }, 8200 tmo_int, 8201 ) 8202 .unwrap(); 8203 8204 if "Windows" == out.trim() { 8205 return true; 8206 } 8207 8208 false 8209 } 8210 } 8211 8212 fn vcpu_threads_count(pid: u32) -> u8 { 8213 // ps -T -p 12345 | grep vcpu | wc -l 8214 let out = Command::new("ps") 8215 .args(["-T", "-p", format!("{pid}").as_str()]) 8216 .output() 8217 .expect("ps command failed") 8218 .stdout; 8219 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 8220 } 8221 8222 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 8223 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 8224 let out = Command::new("ps") 8225 .args(["-T", "-p", format!("{pid}").as_str()]) 8226 .output() 8227 .expect("ps command failed") 8228 .stdout; 8229 let mut n = 0; 8230 String::from_utf8_lossy(&out) 8231 .split_whitespace() 8232 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 8233 n 8234 } 8235 8236 fn disk_ctrl_threads_count(pid: u32) -> u8 { 8237 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 8238 let out = Command::new("ps") 8239 .args(["-T", "-p", format!("{pid}").as_str()]) 8240 .output() 8241 .expect("ps command failed") 8242 .stdout; 8243 let mut n = 0; 8244 String::from_utf8_lossy(&out) 8245 .split_whitespace() 8246 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 8247 n 8248 } 8249 8250 #[test] 8251 fn test_windows_guest() { 8252 let windows_guest = WindowsGuest::new(); 8253 8254 let mut child = GuestCommand::new(windows_guest.guest()) 8255 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8256 .args(["--memory", "size=4G"]) 8257 .args(["--kernel", edk2_path().to_str().unwrap()]) 8258 .args(["--serial", "tty"]) 8259 .args(["--console", "off"]) 8260 .default_disks() 8261 .default_net() 8262 .capture_output() 8263 .spawn() 8264 .unwrap(); 8265 8266 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8267 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8268 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8269 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8270 8271 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8272 8273 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8274 8275 let r = std::panic::catch_unwind(|| { 8276 // Wait to make sure Windows boots up 8277 assert!(windows_guest.wait_for_boot()); 8278 8279 windows_guest.shutdown(); 8280 }); 8281 8282 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8283 let _ = child.kill(); 8284 let output = child.wait_with_output().unwrap(); 8285 8286 let _ = child_dnsmasq.kill(); 8287 let _ = child_dnsmasq.wait(); 8288 8289 handle_child_output(r, &output); 8290 } 8291 8292 #[test] 8293 fn test_windows_guest_multiple_queues() { 8294 let windows_guest = WindowsGuest::new(); 8295 8296 let mut ovmf_path = dirs::home_dir().unwrap(); 8297 ovmf_path.push("workloads"); 8298 ovmf_path.push(OVMF_NAME); 8299 8300 let mut child = GuestCommand::new(windows_guest.guest()) 8301 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 8302 .args(["--memory", "size=4G"]) 8303 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8304 .args(["--serial", "tty"]) 8305 .args(["--console", "off"]) 8306 .args([ 8307 "--disk", 8308 format!( 8309 "path={},num_queues=4", 8310 windows_guest 8311 .guest() 8312 .disk_config 8313 .disk(DiskType::OperatingSystem) 8314 .unwrap() 8315 ) 8316 .as_str(), 8317 ]) 8318 .args([ 8319 "--net", 8320 format!( 8321 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 8322 windows_guest.guest().network.guest_mac, 8323 windows_guest.guest().network.host_ip 8324 ) 8325 .as_str(), 8326 ]) 8327 .capture_output() 8328 .spawn() 8329 .unwrap(); 8330 8331 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8332 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8333 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8334 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8335 8336 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8337 8338 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8339 8340 let r = std::panic::catch_unwind(|| { 8341 // Wait to make sure Windows boots up 8342 assert!(windows_guest.wait_for_boot()); 8343 8344 windows_guest.shutdown(); 8345 }); 8346 8347 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8348 let _ = child.kill(); 8349 let output = child.wait_with_output().unwrap(); 8350 8351 let _ = child_dnsmasq.kill(); 8352 let _ = child_dnsmasq.wait(); 8353 8354 handle_child_output(r, &output); 8355 } 8356 8357 #[test] 8358 #[cfg(not(feature = "mshv"))] 8359 #[ignore = "See #4327"] 8360 fn test_windows_guest_snapshot_restore() { 8361 let windows_guest = WindowsGuest::new(); 8362 8363 let mut ovmf_path = dirs::home_dir().unwrap(); 8364 ovmf_path.push("workloads"); 8365 ovmf_path.push(OVMF_NAME); 8366 8367 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8368 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 8369 8370 let mut child = GuestCommand::new(windows_guest.guest()) 8371 .args(["--api-socket", &api_socket_source]) 8372 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8373 .args(["--memory", "size=4G"]) 8374 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8375 .args(["--serial", "tty"]) 8376 .args(["--console", "off"]) 8377 .default_disks() 8378 .default_net() 8379 .capture_output() 8380 .spawn() 8381 .unwrap(); 8382 8383 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8384 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8385 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8386 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8387 8388 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8389 8390 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8391 8392 // Wait to make sure Windows boots up 8393 assert!(windows_guest.wait_for_boot()); 8394 8395 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 8396 8397 // Pause the VM 8398 assert!(remote_command(&api_socket_source, "pause", None)); 8399 8400 // Take a snapshot from the VM 8401 assert!(remote_command( 8402 &api_socket_source, 8403 "snapshot", 8404 Some(format!("file://{snapshot_dir}").as_str()), 8405 )); 8406 8407 // Wait to make sure the snapshot is completed 8408 thread::sleep(std::time::Duration::new(30, 0)); 8409 8410 let _ = child.kill(); 8411 child.wait().unwrap(); 8412 8413 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 8414 8415 // Restore the VM from the snapshot 8416 let mut child = GuestCommand::new(windows_guest.guest()) 8417 .args(["--api-socket", &api_socket_restored]) 8418 .args([ 8419 "--restore", 8420 format!("source_url=file://{snapshot_dir}").as_str(), 8421 ]) 8422 .capture_output() 8423 .spawn() 8424 .unwrap(); 8425 8426 // Wait for the VM to be restored 8427 thread::sleep(std::time::Duration::new(20, 0)); 8428 8429 let r = std::panic::catch_unwind(|| { 8430 // Resume the VM 8431 assert!(remote_command(&api_socket_restored, "resume", None)); 8432 8433 windows_guest.shutdown(); 8434 }); 8435 8436 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8437 let _ = child.kill(); 8438 let output = child.wait_with_output().unwrap(); 8439 8440 let _ = child_dnsmasq.kill(); 8441 let _ = child_dnsmasq.wait(); 8442 8443 handle_child_output(r, &output); 8444 } 8445 8446 #[test] 8447 #[cfg(not(feature = "mshv"))] 8448 #[cfg(not(target_arch = "aarch64"))] 8449 fn test_windows_guest_cpu_hotplug() { 8450 let windows_guest = WindowsGuest::new(); 8451 8452 let mut ovmf_path = dirs::home_dir().unwrap(); 8453 ovmf_path.push("workloads"); 8454 ovmf_path.push(OVMF_NAME); 8455 8456 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8457 let api_socket = temp_api_path(&tmp_dir); 8458 8459 let mut child = GuestCommand::new(windows_guest.guest()) 8460 .args(["--api-socket", &api_socket]) 8461 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 8462 .args(["--memory", "size=4G"]) 8463 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8464 .args(["--serial", "tty"]) 8465 .args(["--console", "off"]) 8466 .default_disks() 8467 .default_net() 8468 .capture_output() 8469 .spawn() 8470 .unwrap(); 8471 8472 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8473 8474 let r = std::panic::catch_unwind(|| { 8475 // Wait to make sure Windows boots up 8476 assert!(windows_guest.wait_for_boot()); 8477 8478 let vcpu_num = 2; 8479 // Check the initial number of CPUs the guest sees 8480 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8481 // Check the initial number of vcpu threads in the CH process 8482 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8483 8484 let vcpu_num = 6; 8485 // Hotplug some CPUs 8486 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8487 // Wait to make sure CPUs are added 8488 thread::sleep(std::time::Duration::new(10, 0)); 8489 // Check the guest sees the correct number 8490 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8491 // Check the CH process has the correct number of vcpu threads 8492 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8493 8494 let vcpu_num = 4; 8495 // Remove some CPUs. Note that Windows doesn't support hot-remove. 8496 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8497 // Wait to make sure CPUs are removed 8498 thread::sleep(std::time::Duration::new(10, 0)); 8499 // Reboot to let Windows catch up 8500 windows_guest.reboot(); 8501 // Wait to make sure Windows completely rebooted 8502 thread::sleep(std::time::Duration::new(60, 0)); 8503 // Check the guest sees the correct number 8504 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8505 // Check the CH process has the correct number of vcpu threads 8506 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8507 8508 windows_guest.shutdown(); 8509 }); 8510 8511 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8512 let _ = child.kill(); 8513 let output = child.wait_with_output().unwrap(); 8514 8515 let _ = child_dnsmasq.kill(); 8516 let _ = child_dnsmasq.wait(); 8517 8518 handle_child_output(r, &output); 8519 } 8520 8521 #[test] 8522 #[cfg(not(feature = "mshv"))] 8523 #[cfg(not(target_arch = "aarch64"))] 8524 fn test_windows_guest_ram_hotplug() { 8525 let windows_guest = WindowsGuest::new(); 8526 8527 let mut ovmf_path = dirs::home_dir().unwrap(); 8528 ovmf_path.push("workloads"); 8529 ovmf_path.push(OVMF_NAME); 8530 8531 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8532 let api_socket = temp_api_path(&tmp_dir); 8533 8534 let mut child = GuestCommand::new(windows_guest.guest()) 8535 .args(["--api-socket", &api_socket]) 8536 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8537 .args(["--memory", "size=2G,hotplug_size=5G"]) 8538 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8539 .args(["--serial", "tty"]) 8540 .args(["--console", "off"]) 8541 .default_disks() 8542 .default_net() 8543 .capture_output() 8544 .spawn() 8545 .unwrap(); 8546 8547 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8548 8549 let r = std::panic::catch_unwind(|| { 8550 // Wait to make sure Windows boots up 8551 assert!(windows_guest.wait_for_boot()); 8552 8553 let ram_size = 2 * 1024 * 1024 * 1024; 8554 // Check the initial number of RAM the guest sees 8555 let current_ram_size = windows_guest.ram_size(); 8556 // This size seems to be reserved by the system and thus the 8557 // reported amount differs by this constant value. 8558 let reserved_ram_size = ram_size - current_ram_size; 8559 // Verify that there's not more than 4mb constant diff wasted 8560 // by the reserved ram. 8561 assert!(reserved_ram_size < 4 * 1024 * 1024); 8562 8563 let ram_size = 4 * 1024 * 1024 * 1024; 8564 // Hotplug some RAM 8565 resize_command(&api_socket, None, Some(ram_size), None, None); 8566 // Wait to make sure RAM has been added 8567 thread::sleep(std::time::Duration::new(10, 0)); 8568 // Check the guest sees the correct number 8569 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8570 8571 let ram_size = 3 * 1024 * 1024 * 1024; 8572 // Unplug some RAM. Note that hot-remove most likely won't work. 8573 resize_command(&api_socket, None, Some(ram_size), None, None); 8574 // Wait to make sure RAM has been added 8575 thread::sleep(std::time::Duration::new(10, 0)); 8576 // Reboot to let Windows catch up 8577 windows_guest.reboot(); 8578 // Wait to make sure guest completely rebooted 8579 thread::sleep(std::time::Duration::new(60, 0)); 8580 // Check the guest sees the correct number 8581 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8582 8583 windows_guest.shutdown(); 8584 }); 8585 8586 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8587 let _ = child.kill(); 8588 let output = child.wait_with_output().unwrap(); 8589 8590 let _ = child_dnsmasq.kill(); 8591 let _ = child_dnsmasq.wait(); 8592 8593 handle_child_output(r, &output); 8594 } 8595 8596 #[test] 8597 #[cfg(not(feature = "mshv"))] 8598 fn test_windows_guest_netdev_hotplug() { 8599 let windows_guest = WindowsGuest::new(); 8600 8601 let mut ovmf_path = dirs::home_dir().unwrap(); 8602 ovmf_path.push("workloads"); 8603 ovmf_path.push(OVMF_NAME); 8604 8605 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8606 let api_socket = temp_api_path(&tmp_dir); 8607 8608 let mut child = GuestCommand::new(windows_guest.guest()) 8609 .args(["--api-socket", &api_socket]) 8610 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8611 .args(["--memory", "size=4G"]) 8612 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8613 .args(["--serial", "tty"]) 8614 .args(["--console", "off"]) 8615 .default_disks() 8616 .default_net() 8617 .capture_output() 8618 .spawn() 8619 .unwrap(); 8620 8621 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8622 8623 let r = std::panic::catch_unwind(|| { 8624 // Wait to make sure Windows boots up 8625 assert!(windows_guest.wait_for_boot()); 8626 8627 // Initially present network device 8628 let netdev_num = 1; 8629 assert_eq!(windows_guest.netdev_count(), netdev_num); 8630 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8631 8632 // Hotplug network device 8633 let (cmd_success, cmd_output) = remote_command_w_output( 8634 &api_socket, 8635 "add-net", 8636 Some(windows_guest.guest().default_net_string().as_str()), 8637 ); 8638 assert!(cmd_success); 8639 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 8640 thread::sleep(std::time::Duration::new(5, 0)); 8641 // Verify the device is on the system 8642 let netdev_num = 2; 8643 assert_eq!(windows_guest.netdev_count(), netdev_num); 8644 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8645 8646 // Remove network device 8647 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 8648 assert!(cmd_success); 8649 thread::sleep(std::time::Duration::new(5, 0)); 8650 // Verify the device has been removed 8651 let netdev_num = 1; 8652 assert_eq!(windows_guest.netdev_count(), netdev_num); 8653 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8654 8655 windows_guest.shutdown(); 8656 }); 8657 8658 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8659 let _ = child.kill(); 8660 let output = child.wait_with_output().unwrap(); 8661 8662 let _ = child_dnsmasq.kill(); 8663 let _ = child_dnsmasq.wait(); 8664 8665 handle_child_output(r, &output); 8666 } 8667 8668 #[test] 8669 #[ignore = "See #6037"] 8670 #[cfg(not(feature = "mshv"))] 8671 #[cfg(not(target_arch = "aarch64"))] 8672 fn test_windows_guest_disk_hotplug() { 8673 let windows_guest = WindowsGuest::new(); 8674 8675 let mut ovmf_path = dirs::home_dir().unwrap(); 8676 ovmf_path.push("workloads"); 8677 ovmf_path.push(OVMF_NAME); 8678 8679 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8680 let api_socket = temp_api_path(&tmp_dir); 8681 8682 let mut child = GuestCommand::new(windows_guest.guest()) 8683 .args(["--api-socket", &api_socket]) 8684 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8685 .args(["--memory", "size=4G"]) 8686 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8687 .args(["--serial", "tty"]) 8688 .args(["--console", "off"]) 8689 .default_disks() 8690 .default_net() 8691 .capture_output() 8692 .spawn() 8693 .unwrap(); 8694 8695 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8696 8697 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 8698 8699 let r = std::panic::catch_unwind(|| { 8700 // Wait to make sure Windows boots up 8701 assert!(windows_guest.wait_for_boot()); 8702 8703 // Initially present disk device 8704 let disk_num = 1; 8705 assert_eq!(windows_guest.disk_count(), disk_num); 8706 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8707 8708 // Hotplug disk device 8709 let (cmd_success, cmd_output) = remote_command_w_output( 8710 &api_socket, 8711 "add-disk", 8712 Some(format!("path={disk},readonly=off").as_str()), 8713 ); 8714 assert!(cmd_success); 8715 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 8716 thread::sleep(std::time::Duration::new(5, 0)); 8717 // Online disk device 8718 windows_guest.disks_set_rw(); 8719 windows_guest.disks_online(); 8720 // Verify the device is on the system 8721 let disk_num = 2; 8722 assert_eq!(windows_guest.disk_count(), disk_num); 8723 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8724 8725 let data = "hello"; 8726 let fname = "d:\\world"; 8727 windows_guest.disk_file_put(fname, data); 8728 8729 // Unmount disk device 8730 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 8731 assert!(cmd_success); 8732 thread::sleep(std::time::Duration::new(5, 0)); 8733 // Verify the device has been removed 8734 let disk_num = 1; 8735 assert_eq!(windows_guest.disk_count(), disk_num); 8736 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8737 8738 // Remount and check the file exists with the expected contents 8739 let (cmd_success, _cmd_output) = remote_command_w_output( 8740 &api_socket, 8741 "add-disk", 8742 Some(format!("path={disk},readonly=off").as_str()), 8743 ); 8744 assert!(cmd_success); 8745 thread::sleep(std::time::Duration::new(5, 0)); 8746 let out = windows_guest.disk_file_read(fname); 8747 assert_eq!(data, out.trim()); 8748 8749 // Intentionally no unmount, it'll happen at shutdown. 8750 8751 windows_guest.shutdown(); 8752 }); 8753 8754 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8755 let _ = child.kill(); 8756 let output = child.wait_with_output().unwrap(); 8757 8758 let _ = child_dnsmasq.kill(); 8759 let _ = child_dnsmasq.wait(); 8760 8761 handle_child_output(r, &output); 8762 } 8763 8764 #[test] 8765 #[ignore = "See #6037"] 8766 #[cfg(not(feature = "mshv"))] 8767 #[cfg(not(target_arch = "aarch64"))] 8768 fn test_windows_guest_disk_hotplug_multi() { 8769 let windows_guest = WindowsGuest::new(); 8770 8771 let mut ovmf_path = dirs::home_dir().unwrap(); 8772 ovmf_path.push("workloads"); 8773 ovmf_path.push(OVMF_NAME); 8774 8775 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8776 let api_socket = temp_api_path(&tmp_dir); 8777 8778 let mut child = GuestCommand::new(windows_guest.guest()) 8779 .args(["--api-socket", &api_socket]) 8780 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8781 .args(["--memory", "size=2G"]) 8782 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8783 .args(["--serial", "tty"]) 8784 .args(["--console", "off"]) 8785 .default_disks() 8786 .default_net() 8787 .capture_output() 8788 .spawn() 8789 .unwrap(); 8790 8791 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8792 8793 // Predefined data to used at various test stages 8794 let disk_test_data: [[String; 4]; 2] = [ 8795 [ 8796 "_disk2".to_string(), 8797 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 8798 "d:\\world".to_string(), 8799 "hello".to_string(), 8800 ], 8801 [ 8802 "_disk3".to_string(), 8803 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 8804 "e:\\hello".to_string(), 8805 "world".to_string(), 8806 ], 8807 ]; 8808 8809 let r = std::panic::catch_unwind(|| { 8810 // Wait to make sure Windows boots up 8811 assert!(windows_guest.wait_for_boot()); 8812 8813 // Initially present disk device 8814 let disk_num = 1; 8815 assert_eq!(windows_guest.disk_count(), disk_num); 8816 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8817 8818 for it in &disk_test_data { 8819 let disk_id = it[0].as_str(); 8820 let disk = it[1].as_str(); 8821 // Hotplug disk device 8822 let (cmd_success, cmd_output) = remote_command_w_output( 8823 &api_socket, 8824 "add-disk", 8825 Some(format!("path={disk},readonly=off").as_str()), 8826 ); 8827 assert!(cmd_success); 8828 assert!(String::from_utf8_lossy(&cmd_output) 8829 .contains(format!("\"id\":\"{disk_id}\"").as_str())); 8830 thread::sleep(std::time::Duration::new(5, 0)); 8831 // Online disk devices 8832 windows_guest.disks_set_rw(); 8833 windows_guest.disks_online(); 8834 } 8835 // Verify the devices are on the system 8836 let disk_num = (disk_test_data.len() + 1) as u8; 8837 assert_eq!(windows_guest.disk_count(), disk_num); 8838 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8839 8840 // Put test data 8841 for it in &disk_test_data { 8842 let fname = it[2].as_str(); 8843 let data = it[3].as_str(); 8844 windows_guest.disk_file_put(fname, data); 8845 } 8846 8847 // Unmount disk devices 8848 for it in &disk_test_data { 8849 let disk_id = it[0].as_str(); 8850 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 8851 assert!(cmd_success); 8852 thread::sleep(std::time::Duration::new(5, 0)); 8853 } 8854 8855 // Verify the devices have been removed 8856 let disk_num = 1; 8857 assert_eq!(windows_guest.disk_count(), disk_num); 8858 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8859 8860 // Remount 8861 for it in &disk_test_data { 8862 let disk = it[1].as_str(); 8863 let (cmd_success, _cmd_output) = remote_command_w_output( 8864 &api_socket, 8865 "add-disk", 8866 Some(format!("path={disk},readonly=off").as_str()), 8867 ); 8868 assert!(cmd_success); 8869 thread::sleep(std::time::Duration::new(5, 0)); 8870 } 8871 8872 // Check the files exists with the expected contents 8873 for it in &disk_test_data { 8874 let fname = it[2].as_str(); 8875 let data = it[3].as_str(); 8876 let out = windows_guest.disk_file_read(fname); 8877 assert_eq!(data, out.trim()); 8878 } 8879 8880 // Intentionally no unmount, it'll happen at shutdown. 8881 8882 windows_guest.shutdown(); 8883 }); 8884 8885 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8886 let _ = child.kill(); 8887 let output = child.wait_with_output().unwrap(); 8888 8889 let _ = child_dnsmasq.kill(); 8890 let _ = child_dnsmasq.wait(); 8891 8892 handle_child_output(r, &output); 8893 } 8894 8895 #[test] 8896 #[cfg(not(feature = "mshv"))] 8897 #[cfg(not(target_arch = "aarch64"))] 8898 fn test_windows_guest_netdev_multi() { 8899 let windows_guest = WindowsGuest::new(); 8900 8901 let mut ovmf_path = dirs::home_dir().unwrap(); 8902 ovmf_path.push("workloads"); 8903 ovmf_path.push(OVMF_NAME); 8904 8905 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8906 let api_socket = temp_api_path(&tmp_dir); 8907 8908 let mut child = GuestCommand::new(windows_guest.guest()) 8909 .args(["--api-socket", &api_socket]) 8910 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8911 .args(["--memory", "size=4G"]) 8912 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8913 .args(["--serial", "tty"]) 8914 .args(["--console", "off"]) 8915 .default_disks() 8916 // The multi net dev config is borrowed from test_multiple_network_interfaces 8917 .args([ 8918 "--net", 8919 windows_guest.guest().default_net_string().as_str(), 8920 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 8921 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 8922 ]) 8923 .capture_output() 8924 .spawn() 8925 .unwrap(); 8926 8927 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8928 8929 let r = std::panic::catch_unwind(|| { 8930 // Wait to make sure Windows boots up 8931 assert!(windows_guest.wait_for_boot()); 8932 8933 let netdev_num = 3; 8934 assert_eq!(windows_guest.netdev_count(), netdev_num); 8935 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8936 8937 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 8938 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 8939 8940 windows_guest.shutdown(); 8941 }); 8942 8943 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8944 let _ = child.kill(); 8945 let output = child.wait_with_output().unwrap(); 8946 8947 let _ = child_dnsmasq.kill(); 8948 let _ = child_dnsmasq.wait(); 8949 8950 handle_child_output(r, &output); 8951 } 8952 } 8953 8954 #[cfg(target_arch = "x86_64")] 8955 mod sgx { 8956 use crate::*; 8957 8958 #[test] 8959 fn test_sgx() { 8960 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 8961 let jammy = UbuntuDiskConfig::new(jammy_image); 8962 let guest = Guest::new(Box::new(jammy)); 8963 8964 let mut child = GuestCommand::new(&guest) 8965 .args(["--cpus", "boot=1"]) 8966 .args(["--memory", "size=512M"]) 8967 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8968 .default_disks() 8969 .default_net() 8970 .args(["--sgx-epc", "id=epc0,size=64M"]) 8971 .capture_output() 8972 .spawn() 8973 .unwrap(); 8974 8975 let r = std::panic::catch_unwind(|| { 8976 guest.wait_vm_boot(None).unwrap(); 8977 8978 // Check if SGX is correctly detected in the guest. 8979 guest.check_sgx_support().unwrap(); 8980 8981 // Validate the SGX EPC section is 64MiB. 8982 assert_eq!( 8983 guest 8984 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 8985 .unwrap() 8986 .trim(), 8987 "0x0000000004000000" 8988 ); 8989 }); 8990 8991 let _ = child.kill(); 8992 let output = child.wait_with_output().unwrap(); 8993 8994 handle_child_output(r, &output); 8995 } 8996 } 8997 8998 #[cfg(target_arch = "x86_64")] 8999 mod vfio { 9000 use crate::*; 9001 const NVIDIA_VFIO_DEVICE: &str = "/sys/bus/pci/devices/0002:00:01.0"; 9002 9003 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 9004 let jammy = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string()); 9005 let guest = Guest::new(Box::new(jammy)); 9006 let api_socket = temp_api_path(&guest.tmp_dir); 9007 9008 let mut child = GuestCommand::new(&guest) 9009 .args(["--cpus", "boot=4"]) 9010 .args([ 9011 "--memory", 9012 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(), 9013 ]) 9014 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9015 .args(["--device", format!("path={NVIDIA_VFIO_DEVICE}").as_str()]) 9016 .args(["--api-socket", &api_socket]) 9017 .default_disks() 9018 .default_net() 9019 .capture_output() 9020 .spawn() 9021 .unwrap(); 9022 9023 let r = std::panic::catch_unwind(|| { 9024 guest.wait_vm_boot(None).unwrap(); 9025 9026 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9027 9028 guest.enable_memory_hotplug(); 9029 9030 // Add RAM to the VM 9031 let desired_ram = 6 << 30; 9032 resize_command(&api_socket, None, Some(desired_ram), None, None); 9033 thread::sleep(std::time::Duration::new(30, 0)); 9034 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9035 9036 // Check the VFIO device works when RAM is increased to 6GiB 9037 guest.check_nvidia_gpu(); 9038 }); 9039 9040 let _ = child.kill(); 9041 let output = child.wait_with_output().unwrap(); 9042 9043 handle_child_output(r, &output); 9044 } 9045 9046 #[test] 9047 fn test_nvidia_card_memory_hotplug_acpi() { 9048 test_nvidia_card_memory_hotplug("acpi") 9049 } 9050 9051 #[test] 9052 fn test_nvidia_card_memory_hotplug_virtio_mem() { 9053 test_nvidia_card_memory_hotplug("virtio-mem") 9054 } 9055 9056 #[test] 9057 fn test_nvidia_card_pci_hotplug() { 9058 let jammy = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string()); 9059 let guest = Guest::new(Box::new(jammy)); 9060 let api_socket = temp_api_path(&guest.tmp_dir); 9061 9062 let mut child = GuestCommand::new(&guest) 9063 .args(["--cpus", "boot=4"]) 9064 .args(["--memory", "size=4G"]) 9065 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9066 .args(["--api-socket", &api_socket]) 9067 .default_disks() 9068 .default_net() 9069 .capture_output() 9070 .spawn() 9071 .unwrap(); 9072 9073 let r = std::panic::catch_unwind(|| { 9074 guest.wait_vm_boot(None).unwrap(); 9075 9076 // Hotplug the card to the VM 9077 let (cmd_success, cmd_output) = remote_command_w_output( 9078 &api_socket, 9079 "add-device", 9080 Some(format!("id=vfio0,path={NVIDIA_VFIO_DEVICE}").as_str()), 9081 ); 9082 assert!(cmd_success); 9083 assert!(String::from_utf8_lossy(&cmd_output) 9084 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 9085 9086 thread::sleep(std::time::Duration::new(10, 0)); 9087 9088 // Check the VFIO device works after hotplug 9089 guest.check_nvidia_gpu(); 9090 }); 9091 9092 let _ = child.kill(); 9093 let output = child.wait_with_output().unwrap(); 9094 9095 handle_child_output(r, &output); 9096 } 9097 9098 #[test] 9099 fn test_nvidia_card_reboot() { 9100 let jammy = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string()); 9101 let guest = Guest::new(Box::new(jammy)); 9102 let api_socket = temp_api_path(&guest.tmp_dir); 9103 9104 let mut child = GuestCommand::new(&guest) 9105 .args(["--cpus", "boot=4"]) 9106 .args(["--memory", "size=4G"]) 9107 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9108 .args(["--device", format!("path={NVIDIA_VFIO_DEVICE}").as_str()]) 9109 .args(["--api-socket", &api_socket]) 9110 .default_disks() 9111 .default_net() 9112 .capture_output() 9113 .spawn() 9114 .unwrap(); 9115 9116 let r = std::panic::catch_unwind(|| { 9117 guest.wait_vm_boot(None).unwrap(); 9118 9119 // Check the VFIO device works after boot 9120 guest.check_nvidia_gpu(); 9121 9122 guest.reboot_linux(0, None); 9123 9124 // Check the VFIO device works after reboot 9125 guest.check_nvidia_gpu(); 9126 }); 9127 9128 let _ = child.kill(); 9129 let output = child.wait_with_output().unwrap(); 9130 9131 handle_child_output(r, &output); 9132 } 9133 } 9134 9135 mod live_migration { 9136 use crate::*; 9137 9138 fn start_live_migration( 9139 migration_socket: &str, 9140 src_api_socket: &str, 9141 dest_api_socket: &str, 9142 local: bool, 9143 ) -> bool { 9144 // Start to receive migration from the destination VM 9145 let mut receive_migration = Command::new(clh_command("ch-remote")) 9146 .args([ 9147 &format!("--api-socket={dest_api_socket}"), 9148 "receive-migration", 9149 &format! {"unix:{migration_socket}"}, 9150 ]) 9151 .stderr(Stdio::piped()) 9152 .stdout(Stdio::piped()) 9153 .spawn() 9154 .unwrap(); 9155 // Give it '1s' to make sure the 'migration_socket' file is properly created 9156 thread::sleep(std::time::Duration::new(1, 0)); 9157 // Start to send migration from the source VM 9158 9159 let mut args = [ 9160 format!("--api-socket={}", &src_api_socket), 9161 "send-migration".to_string(), 9162 format! {"unix:{migration_socket}"}, 9163 ] 9164 .to_vec(); 9165 9166 if local { 9167 args.insert(2, "--local".to_string()); 9168 } 9169 9170 let mut send_migration = Command::new(clh_command("ch-remote")) 9171 .args(&args) 9172 .stderr(Stdio::piped()) 9173 .stdout(Stdio::piped()) 9174 .spawn() 9175 .unwrap(); 9176 9177 // The 'send-migration' command should be executed successfully within the given timeout 9178 let send_success = if let Some(status) = send_migration 9179 .wait_timeout(std::time::Duration::from_secs(30)) 9180 .unwrap() 9181 { 9182 status.success() 9183 } else { 9184 false 9185 }; 9186 9187 if !send_success { 9188 let _ = send_migration.kill(); 9189 let output = send_migration.wait_with_output().unwrap(); 9190 eprintln!( 9191 "\n\n==== Start 'send_migration' output ==== \ 9192 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 9193 \n\n==== End 'send_migration' output ====\n\n", 9194 String::from_utf8_lossy(&output.stdout), 9195 String::from_utf8_lossy(&output.stderr) 9196 ); 9197 } 9198 9199 // The 'receive-migration' command should be executed successfully within the given timeout 9200 let receive_success = if let Some(status) = receive_migration 9201 .wait_timeout(std::time::Duration::from_secs(30)) 9202 .unwrap() 9203 { 9204 status.success() 9205 } else { 9206 false 9207 }; 9208 9209 if !receive_success { 9210 let _ = receive_migration.kill(); 9211 let output = receive_migration.wait_with_output().unwrap(); 9212 eprintln!( 9213 "\n\n==== Start 'receive_migration' output ==== \ 9214 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 9215 \n\n==== End 'receive_migration' output ====\n\n", 9216 String::from_utf8_lossy(&output.stdout), 9217 String::from_utf8_lossy(&output.stderr) 9218 ); 9219 } 9220 9221 send_success && receive_success 9222 } 9223 9224 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 9225 let mut src_vm = src_vm; 9226 let mut dest_vm = dest_vm; 9227 9228 let _ = src_vm.kill(); 9229 let src_output = src_vm.wait_with_output().unwrap(); 9230 eprintln!( 9231 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 9232 String::from_utf8_lossy(&src_output.stdout) 9233 ); 9234 eprintln!( 9235 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 9236 String::from_utf8_lossy(&src_output.stderr) 9237 ); 9238 let _ = dest_vm.kill(); 9239 let dest_output = dest_vm.wait_with_output().unwrap(); 9240 eprintln!( 9241 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 9242 String::from_utf8_lossy(&dest_output.stdout) 9243 ); 9244 eprintln!( 9245 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 9246 String::from_utf8_lossy(&dest_output.stderr) 9247 ); 9248 9249 if let Some(ovs_vm) = ovs_vm { 9250 let mut ovs_vm = ovs_vm; 9251 let _ = ovs_vm.kill(); 9252 let ovs_output = ovs_vm.wait_with_output().unwrap(); 9253 eprintln!( 9254 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 9255 String::from_utf8_lossy(&ovs_output.stdout) 9256 ); 9257 eprintln!( 9258 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 9259 String::from_utf8_lossy(&ovs_output.stderr) 9260 ); 9261 9262 cleanup_ovs_dpdk(); 9263 } 9264 9265 panic!("Test failed: {message}") 9266 } 9267 9268 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 9269 // same host. It ensures the following behaviors: 9270 // 1. The source VM is up and functional (including various virtio-devices are working properly); 9271 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 9272 // 3. The source VM terminated gracefully after live migration; 9273 // 4. The destination VM is functional (including various virtio-devices are working properly) after 9274 // live migration; 9275 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 9276 fn _test_live_migration(upgrade_test: bool, local: bool) { 9277 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9278 let guest = Guest::new(Box::new(focal)); 9279 let kernel_path = direct_kernel_boot_path(); 9280 let console_text = String::from("On a branch floating down river a cricket, singing."); 9281 let net_id = "net123"; 9282 let net_params = format!( 9283 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9284 net_id, guest.network.guest_mac, guest.network.host_ip 9285 ); 9286 9287 let memory_param: &[&str] = if local { 9288 &["--memory", "size=4G,shared=on"] 9289 } else { 9290 &["--memory", "size=4G"] 9291 }; 9292 9293 let boot_vcpus = 2; 9294 let max_vcpus = 4; 9295 9296 let pmem_temp_file = TempFile::new().unwrap(); 9297 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9298 std::process::Command::new("mkfs.ext4") 9299 .arg(pmem_temp_file.as_path()) 9300 .output() 9301 .expect("Expect creating disk image to succeed"); 9302 let pmem_path = String::from("/dev/pmem0"); 9303 9304 // Start the source VM 9305 let src_vm_path = if !upgrade_test { 9306 clh_command("cloud-hypervisor") 9307 } else { 9308 cloud_hypervisor_release_path() 9309 }; 9310 let src_api_socket = temp_api_path(&guest.tmp_dir); 9311 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9312 src_vm_cmd 9313 .args([ 9314 "--cpus", 9315 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9316 ]) 9317 .args(memory_param) 9318 .args(["--kernel", kernel_path.to_str().unwrap()]) 9319 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9320 .default_disks() 9321 .args(["--net", net_params.as_str()]) 9322 .args(["--api-socket", &src_api_socket]) 9323 .args([ 9324 "--pmem", 9325 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9326 ]); 9327 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9328 9329 // Start the destination VM 9330 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9331 dest_api_socket.push_str(".dest"); 9332 let mut dest_child = GuestCommand::new(&guest) 9333 .args(["--api-socket", &dest_api_socket]) 9334 .capture_output() 9335 .spawn() 9336 .unwrap(); 9337 9338 let r = std::panic::catch_unwind(|| { 9339 guest.wait_vm_boot(None).unwrap(); 9340 9341 // Make sure the source VM is functional 9342 // Check the number of vCPUs 9343 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9344 9345 // Check the guest RAM 9346 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9347 9348 // Check the guest virtio-devices, e.g. block, rng, console, and net 9349 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9350 9351 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9352 // to make sure that removing and adding back the virtio-net device does 9353 // not break the live-migration support for virtio-pci. 9354 #[cfg(target_arch = "x86_64")] 9355 { 9356 assert!(remote_command( 9357 &src_api_socket, 9358 "remove-device", 9359 Some(net_id), 9360 )); 9361 thread::sleep(std::time::Duration::new(10, 0)); 9362 9363 // Plug the virtio-net device again 9364 assert!(remote_command( 9365 &src_api_socket, 9366 "add-net", 9367 Some(net_params.as_str()), 9368 )); 9369 thread::sleep(std::time::Duration::new(10, 0)); 9370 } 9371 9372 // Start the live-migration 9373 let migration_socket = String::from( 9374 guest 9375 .tmp_dir 9376 .as_path() 9377 .join("live-migration.sock") 9378 .to_str() 9379 .unwrap(), 9380 ); 9381 9382 assert!( 9383 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9384 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9385 ); 9386 }); 9387 9388 // Check and report any errors occurred during the live-migration 9389 if r.is_err() { 9390 print_and_panic( 9391 src_child, 9392 dest_child, 9393 None, 9394 "Error occurred during live-migration", 9395 ); 9396 } 9397 9398 // Check the source vm has been terminated successful (give it '3s' to settle) 9399 thread::sleep(std::time::Duration::new(3, 0)); 9400 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9401 print_and_panic( 9402 src_child, 9403 dest_child, 9404 None, 9405 "source VM was not terminated successfully.", 9406 ); 9407 }; 9408 9409 // Post live-migration check to make sure the destination VM is functional 9410 let r = std::panic::catch_unwind(|| { 9411 // Perform same checks to validate VM has been properly migrated 9412 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9413 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9414 9415 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9416 }); 9417 9418 // Clean-up the destination VM and make sure it terminated correctly 9419 let _ = dest_child.kill(); 9420 let dest_output = dest_child.wait_with_output().unwrap(); 9421 handle_child_output(r, &dest_output); 9422 9423 // Check the destination VM has the expected 'console_text' from its output 9424 let r = std::panic::catch_unwind(|| { 9425 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9426 }); 9427 handle_child_output(r, &dest_output); 9428 } 9429 9430 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 9431 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9432 let guest = Guest::new(Box::new(focal)); 9433 let kernel_path = direct_kernel_boot_path(); 9434 let console_text = String::from("On a branch floating down river a cricket, singing."); 9435 let net_id = "net123"; 9436 let net_params = format!( 9437 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9438 net_id, guest.network.guest_mac, guest.network.host_ip 9439 ); 9440 9441 let memory_param: &[&str] = if local { 9442 &[ 9443 "--memory", 9444 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 9445 "--balloon", 9446 "size=0", 9447 ] 9448 } else { 9449 &[ 9450 "--memory", 9451 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 9452 "--balloon", 9453 "size=0", 9454 ] 9455 }; 9456 9457 let boot_vcpus = 2; 9458 let max_vcpus = 4; 9459 9460 let pmem_temp_file = TempFile::new().unwrap(); 9461 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9462 std::process::Command::new("mkfs.ext4") 9463 .arg(pmem_temp_file.as_path()) 9464 .output() 9465 .expect("Expect creating disk image to succeed"); 9466 let pmem_path = String::from("/dev/pmem0"); 9467 9468 // Start the source VM 9469 let src_vm_path = if !upgrade_test { 9470 clh_command("cloud-hypervisor") 9471 } else { 9472 cloud_hypervisor_release_path() 9473 }; 9474 let src_api_socket = temp_api_path(&guest.tmp_dir); 9475 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9476 src_vm_cmd 9477 .args([ 9478 "--cpus", 9479 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9480 ]) 9481 .args(memory_param) 9482 .args(["--kernel", kernel_path.to_str().unwrap()]) 9483 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9484 .default_disks() 9485 .args(["--net", net_params.as_str()]) 9486 .args(["--api-socket", &src_api_socket]) 9487 .args([ 9488 "--pmem", 9489 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9490 ]); 9491 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9492 9493 // Start the destination VM 9494 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9495 dest_api_socket.push_str(".dest"); 9496 let mut dest_child = GuestCommand::new(&guest) 9497 .args(["--api-socket", &dest_api_socket]) 9498 .capture_output() 9499 .spawn() 9500 .unwrap(); 9501 9502 let r = std::panic::catch_unwind(|| { 9503 guest.wait_vm_boot(None).unwrap(); 9504 9505 // Make sure the source VM is functional 9506 // Check the number of vCPUs 9507 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9508 9509 // Check the guest RAM 9510 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9511 // Increase the guest RAM 9512 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 9513 thread::sleep(std::time::Duration::new(5, 0)); 9514 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9515 // Use balloon to remove RAM from the VM 9516 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 9517 thread::sleep(std::time::Duration::new(5, 0)); 9518 let total_memory = guest.get_total_memory().unwrap_or_default(); 9519 assert!(total_memory > 4_800_000); 9520 assert!(total_memory < 5_760_000); 9521 9522 // Check the guest virtio-devices, e.g. block, rng, console, and net 9523 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9524 9525 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9526 // to make sure that removing and adding back the virtio-net device does 9527 // not break the live-migration support for virtio-pci. 9528 #[cfg(target_arch = "x86_64")] 9529 { 9530 assert!(remote_command( 9531 &src_api_socket, 9532 "remove-device", 9533 Some(net_id), 9534 )); 9535 thread::sleep(std::time::Duration::new(10, 0)); 9536 9537 // Plug the virtio-net device again 9538 assert!(remote_command( 9539 &src_api_socket, 9540 "add-net", 9541 Some(net_params.as_str()), 9542 )); 9543 thread::sleep(std::time::Duration::new(10, 0)); 9544 } 9545 9546 // Start the live-migration 9547 let migration_socket = String::from( 9548 guest 9549 .tmp_dir 9550 .as_path() 9551 .join("live-migration.sock") 9552 .to_str() 9553 .unwrap(), 9554 ); 9555 9556 assert!( 9557 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9558 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9559 ); 9560 }); 9561 9562 // Check and report any errors occurred during the live-migration 9563 if r.is_err() { 9564 print_and_panic( 9565 src_child, 9566 dest_child, 9567 None, 9568 "Error occurred during live-migration", 9569 ); 9570 } 9571 9572 // Check the source vm has been terminated successful (give it '3s' to settle) 9573 thread::sleep(std::time::Duration::new(3, 0)); 9574 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9575 print_and_panic( 9576 src_child, 9577 dest_child, 9578 None, 9579 "source VM was not terminated successfully.", 9580 ); 9581 }; 9582 9583 // Post live-migration check to make sure the destination VM is functional 9584 let r = std::panic::catch_unwind(|| { 9585 // Perform same checks to validate VM has been properly migrated 9586 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9587 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9588 9589 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9590 9591 // Perform checks on guest RAM using balloon 9592 let total_memory = guest.get_total_memory().unwrap_or_default(); 9593 assert!(total_memory > 4_800_000); 9594 assert!(total_memory < 5_760_000); 9595 // Deflate balloon to restore entire RAM to the VM 9596 resize_command(&dest_api_socket, None, None, Some(0), None); 9597 thread::sleep(std::time::Duration::new(5, 0)); 9598 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9599 // Decrease guest RAM with virtio-mem 9600 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 9601 thread::sleep(std::time::Duration::new(5, 0)); 9602 let total_memory = guest.get_total_memory().unwrap_or_default(); 9603 assert!(total_memory > 4_800_000); 9604 assert!(total_memory < 5_760_000); 9605 }); 9606 9607 // Clean-up the destination VM and make sure it terminated correctly 9608 let _ = dest_child.kill(); 9609 let dest_output = dest_child.wait_with_output().unwrap(); 9610 handle_child_output(r, &dest_output); 9611 9612 // Check the destination VM has the expected 'console_text' from its output 9613 let r = std::panic::catch_unwind(|| { 9614 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9615 }); 9616 handle_child_output(r, &dest_output); 9617 } 9618 9619 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 9620 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9621 let guest = Guest::new(Box::new(focal)); 9622 let kernel_path = direct_kernel_boot_path(); 9623 let console_text = String::from("On a branch floating down river a cricket, singing."); 9624 let net_id = "net123"; 9625 let net_params = format!( 9626 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9627 net_id, guest.network.guest_mac, guest.network.host_ip 9628 ); 9629 9630 let memory_param: &[&str] = if local { 9631 &[ 9632 "--memory", 9633 "size=0,hotplug_method=virtio-mem,shared=on", 9634 "--memory-zone", 9635 "id=mem0,size=1G,hotplug_size=4G,shared=on", 9636 "id=mem1,size=1G,hotplug_size=4G,shared=on", 9637 "id=mem2,size=2G,hotplug_size=4G,shared=on", 9638 "--numa", 9639 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9640 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9641 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9642 ] 9643 } else { 9644 &[ 9645 "--memory", 9646 "size=0,hotplug_method=virtio-mem", 9647 "--memory-zone", 9648 "id=mem0,size=1G,hotplug_size=4G", 9649 "id=mem1,size=1G,hotplug_size=4G", 9650 "id=mem2,size=2G,hotplug_size=4G", 9651 "--numa", 9652 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9653 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9654 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9655 ] 9656 }; 9657 9658 let boot_vcpus = 6; 9659 let max_vcpus = 12; 9660 9661 let pmem_temp_file = TempFile::new().unwrap(); 9662 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9663 std::process::Command::new("mkfs.ext4") 9664 .arg(pmem_temp_file.as_path()) 9665 .output() 9666 .expect("Expect creating disk image to succeed"); 9667 let pmem_path = String::from("/dev/pmem0"); 9668 9669 // Start the source VM 9670 let src_vm_path = if !upgrade_test { 9671 clh_command("cloud-hypervisor") 9672 } else { 9673 cloud_hypervisor_release_path() 9674 }; 9675 let src_api_socket = temp_api_path(&guest.tmp_dir); 9676 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9677 src_vm_cmd 9678 .args([ 9679 "--cpus", 9680 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9681 ]) 9682 .args(memory_param) 9683 .args(["--kernel", kernel_path.to_str().unwrap()]) 9684 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9685 .default_disks() 9686 .args(["--net", net_params.as_str()]) 9687 .args(["--api-socket", &src_api_socket]) 9688 .args([ 9689 "--pmem", 9690 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9691 ]); 9692 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9693 9694 // Start the destination VM 9695 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9696 dest_api_socket.push_str(".dest"); 9697 let mut dest_child = GuestCommand::new(&guest) 9698 .args(["--api-socket", &dest_api_socket]) 9699 .capture_output() 9700 .spawn() 9701 .unwrap(); 9702 9703 let r = std::panic::catch_unwind(|| { 9704 guest.wait_vm_boot(None).unwrap(); 9705 9706 // Make sure the source VM is functional 9707 // Check the number of vCPUs 9708 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9709 9710 // Check the guest RAM 9711 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 9712 9713 // Check the guest virtio-devices, e.g. block, rng, console, and net 9714 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9715 9716 // Check the NUMA parameters are applied correctly and resize 9717 // each zone to test the case where we migrate a VM with the 9718 // virtio-mem regions being used. 9719 { 9720 guest.check_numa_common( 9721 Some(&[960_000, 960_000, 1_920_000]), 9722 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9723 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9724 ); 9725 9726 // AArch64 currently does not support hotplug, and therefore we only 9727 // test hotplug-related function on x86_64 here. 9728 #[cfg(target_arch = "x86_64")] 9729 { 9730 guest.enable_memory_hotplug(); 9731 9732 // Resize every memory zone and check each associated NUMA node 9733 // has been assigned the right amount of memory. 9734 resize_zone_command(&src_api_socket, "mem0", "2G"); 9735 resize_zone_command(&src_api_socket, "mem1", "2G"); 9736 resize_zone_command(&src_api_socket, "mem2", "3G"); 9737 thread::sleep(std::time::Duration::new(5, 0)); 9738 9739 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 9740 } 9741 } 9742 9743 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9744 // to make sure that removing and adding back the virtio-net device does 9745 // not break the live-migration support for virtio-pci. 9746 #[cfg(target_arch = "x86_64")] 9747 { 9748 assert!(remote_command( 9749 &src_api_socket, 9750 "remove-device", 9751 Some(net_id), 9752 )); 9753 thread::sleep(std::time::Duration::new(10, 0)); 9754 9755 // Plug the virtio-net device again 9756 assert!(remote_command( 9757 &src_api_socket, 9758 "add-net", 9759 Some(net_params.as_str()), 9760 )); 9761 thread::sleep(std::time::Duration::new(10, 0)); 9762 } 9763 9764 // Start the live-migration 9765 let migration_socket = String::from( 9766 guest 9767 .tmp_dir 9768 .as_path() 9769 .join("live-migration.sock") 9770 .to_str() 9771 .unwrap(), 9772 ); 9773 9774 assert!( 9775 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9776 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9777 ); 9778 }); 9779 9780 // Check and report any errors occurred during the live-migration 9781 if r.is_err() { 9782 print_and_panic( 9783 src_child, 9784 dest_child, 9785 None, 9786 "Error occurred during live-migration", 9787 ); 9788 } 9789 9790 // Check the source vm has been terminated successful (give it '3s' to settle) 9791 thread::sleep(std::time::Duration::new(3, 0)); 9792 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9793 print_and_panic( 9794 src_child, 9795 dest_child, 9796 None, 9797 "source VM was not terminated successfully.", 9798 ); 9799 }; 9800 9801 // Post live-migration check to make sure the destination VM is functional 9802 let r = std::panic::catch_unwind(|| { 9803 // Perform same checks to validate VM has been properly migrated 9804 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9805 #[cfg(target_arch = "x86_64")] 9806 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 9807 #[cfg(target_arch = "aarch64")] 9808 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9809 9810 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9811 9812 // Perform NUMA related checks 9813 { 9814 #[cfg(target_arch = "aarch64")] 9815 { 9816 guest.check_numa_common( 9817 Some(&[960_000, 960_000, 1_920_000]), 9818 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9819 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9820 ); 9821 } 9822 9823 // AArch64 currently does not support hotplug, and therefore we only 9824 // test hotplug-related function on x86_64 here. 9825 #[cfg(target_arch = "x86_64")] 9826 { 9827 guest.check_numa_common( 9828 Some(&[1_920_000, 1_920_000, 2_880_000]), 9829 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9830 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9831 ); 9832 9833 guest.enable_memory_hotplug(); 9834 9835 // Resize every memory zone and check each associated NUMA node 9836 // has been assigned the right amount of memory. 9837 resize_zone_command(&dest_api_socket, "mem0", "4G"); 9838 resize_zone_command(&dest_api_socket, "mem1", "4G"); 9839 resize_zone_command(&dest_api_socket, "mem2", "4G"); 9840 // Resize to the maximum amount of CPUs and check each NUMA 9841 // node has been assigned the right CPUs set. 9842 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 9843 thread::sleep(std::time::Duration::new(5, 0)); 9844 9845 guest.check_numa_common( 9846 Some(&[3_840_000, 3_840_000, 3_840_000]), 9847 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 9848 None, 9849 ); 9850 } 9851 } 9852 }); 9853 9854 // Clean-up the destination VM and make sure it terminated correctly 9855 let _ = dest_child.kill(); 9856 let dest_output = dest_child.wait_with_output().unwrap(); 9857 handle_child_output(r, &dest_output); 9858 9859 // Check the destination VM has the expected 'console_text' from its output 9860 let r = std::panic::catch_unwind(|| { 9861 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9862 }); 9863 handle_child_output(r, &dest_output); 9864 } 9865 9866 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 9867 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9868 let guest = Guest::new(Box::new(focal)); 9869 let kernel_path = direct_kernel_boot_path(); 9870 let console_text = String::from("On a branch floating down river a cricket, singing."); 9871 let net_id = "net123"; 9872 let net_params = format!( 9873 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9874 net_id, guest.network.guest_mac, guest.network.host_ip 9875 ); 9876 9877 let memory_param: &[&str] = if local { 9878 &["--memory", "size=4G,shared=on"] 9879 } else { 9880 &["--memory", "size=4G"] 9881 }; 9882 9883 let boot_vcpus = 2; 9884 let max_vcpus = 4; 9885 9886 let pmem_temp_file = TempFile::new().unwrap(); 9887 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9888 std::process::Command::new("mkfs.ext4") 9889 .arg(pmem_temp_file.as_path()) 9890 .output() 9891 .expect("Expect creating disk image to succeed"); 9892 let pmem_path = String::from("/dev/pmem0"); 9893 9894 // Start the source VM 9895 let src_vm_path = if !upgrade_test { 9896 clh_command("cloud-hypervisor") 9897 } else { 9898 cloud_hypervisor_release_path() 9899 }; 9900 let src_api_socket = temp_api_path(&guest.tmp_dir); 9901 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9902 src_vm_cmd 9903 .args([ 9904 "--cpus", 9905 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9906 ]) 9907 .args(memory_param) 9908 .args(["--kernel", kernel_path.to_str().unwrap()]) 9909 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9910 .default_disks() 9911 .args(["--net", net_params.as_str()]) 9912 .args(["--api-socket", &src_api_socket]) 9913 .args([ 9914 "--pmem", 9915 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9916 ]) 9917 .args(["--watchdog"]); 9918 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9919 9920 // Start the destination VM 9921 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9922 dest_api_socket.push_str(".dest"); 9923 let mut dest_child = GuestCommand::new(&guest) 9924 .args(["--api-socket", &dest_api_socket]) 9925 .capture_output() 9926 .spawn() 9927 .unwrap(); 9928 9929 let r = std::panic::catch_unwind(|| { 9930 guest.wait_vm_boot(None).unwrap(); 9931 9932 // Make sure the source VM is functional 9933 // Check the number of vCPUs 9934 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9935 // Check the guest RAM 9936 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9937 // Check the guest virtio-devices, e.g. block, rng, console, and net 9938 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9939 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9940 // to make sure that removing and adding back the virtio-net device does 9941 // not break the live-migration support for virtio-pci. 9942 #[cfg(target_arch = "x86_64")] 9943 { 9944 assert!(remote_command( 9945 &src_api_socket, 9946 "remove-device", 9947 Some(net_id), 9948 )); 9949 thread::sleep(std::time::Duration::new(10, 0)); 9950 9951 // Plug the virtio-net device again 9952 assert!(remote_command( 9953 &src_api_socket, 9954 "add-net", 9955 Some(net_params.as_str()), 9956 )); 9957 thread::sleep(std::time::Duration::new(10, 0)); 9958 } 9959 9960 // Enable watchdog and ensure its functional 9961 let expected_reboot_count = 1; 9962 // Enable the watchdog with a 15s timeout 9963 enable_guest_watchdog(&guest, 15); 9964 9965 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9966 assert_eq!( 9967 guest 9968 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 9969 .unwrap() 9970 .trim() 9971 .parse::<u32>() 9972 .unwrap_or_default(), 9973 1 9974 ); 9975 // Allow some normal time to elapse to check we don't get spurious reboots 9976 thread::sleep(std::time::Duration::new(40, 0)); 9977 // Check no reboot 9978 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9979 9980 // Start the live-migration 9981 let migration_socket = String::from( 9982 guest 9983 .tmp_dir 9984 .as_path() 9985 .join("live-migration.sock") 9986 .to_str() 9987 .unwrap(), 9988 ); 9989 9990 assert!( 9991 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9992 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9993 ); 9994 }); 9995 9996 // Check and report any errors occurred during the live-migration 9997 if r.is_err() { 9998 print_and_panic( 9999 src_child, 10000 dest_child, 10001 None, 10002 "Error occurred during live-migration", 10003 ); 10004 } 10005 10006 // Check the source vm has been terminated successful (give it '3s' to settle) 10007 thread::sleep(std::time::Duration::new(3, 0)); 10008 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 10009 print_and_panic( 10010 src_child, 10011 dest_child, 10012 None, 10013 "source VM was not terminated successfully.", 10014 ); 10015 }; 10016 10017 // Post live-migration check to make sure the destination VM is functional 10018 let r = std::panic::catch_unwind(|| { 10019 // Perform same checks to validate VM has been properly migrated 10020 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10021 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10022 10023 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 10024 10025 // Perform checks on watchdog 10026 let mut expected_reboot_count = 1; 10027 10028 // Allow some normal time to elapse to check we don't get spurious reboots 10029 thread::sleep(std::time::Duration::new(40, 0)); 10030 // Check no reboot 10031 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10032 10033 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 10034 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 10035 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 10036 guest.wait_vm_boot(Some(50)).unwrap(); 10037 // Check a reboot is triggered by the watchdog 10038 expected_reboot_count += 1; 10039 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10040 10041 #[cfg(target_arch = "x86_64")] 10042 { 10043 // Now pause the VM and remain offline for 30s 10044 assert!(remote_command(&dest_api_socket, "pause", None)); 10045 thread::sleep(std::time::Duration::new(30, 0)); 10046 assert!(remote_command(&dest_api_socket, "resume", None)); 10047 10048 // Check no reboot 10049 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10050 } 10051 }); 10052 10053 // Clean-up the destination VM and make sure it terminated correctly 10054 let _ = dest_child.kill(); 10055 let dest_output = dest_child.wait_with_output().unwrap(); 10056 handle_child_output(r, &dest_output); 10057 10058 // Check the destination VM has the expected 'console_text' from its output 10059 let r = std::panic::catch_unwind(|| { 10060 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 10061 }); 10062 handle_child_output(r, &dest_output); 10063 } 10064 10065 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 10066 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10067 let ovs_guest = Guest::new(Box::new(ovs_focal)); 10068 10069 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10070 let migration_guest = Guest::new(Box::new(migration_focal)); 10071 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 10072 10073 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 10074 let (mut ovs_child, mut src_child) = 10075 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 10076 10077 // Start the destination VM 10078 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 10079 dest_api_socket.push_str(".dest"); 10080 let mut dest_child = GuestCommand::new(&migration_guest) 10081 .args(["--api-socket", &dest_api_socket]) 10082 .capture_output() 10083 .spawn() 10084 .unwrap(); 10085 10086 let r = std::panic::catch_unwind(|| { 10087 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 10088 thread::sleep(std::time::Duration::new(1, 0)); 10089 10090 // Start the live-migration 10091 let migration_socket = String::from( 10092 migration_guest 10093 .tmp_dir 10094 .as_path() 10095 .join("live-migration.sock") 10096 .to_str() 10097 .unwrap(), 10098 ); 10099 10100 assert!( 10101 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 10102 "Unsuccessful command: 'send-migration' or 'receive-migration'." 10103 ); 10104 }); 10105 10106 // Check and report any errors occurred during the live-migration 10107 if r.is_err() { 10108 print_and_panic( 10109 src_child, 10110 dest_child, 10111 Some(ovs_child), 10112 "Error occurred during live-migration", 10113 ); 10114 } 10115 10116 // Check the source vm has been terminated successful (give it '3s' to settle) 10117 thread::sleep(std::time::Duration::new(3, 0)); 10118 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 10119 print_and_panic( 10120 src_child, 10121 dest_child, 10122 Some(ovs_child), 10123 "source VM was not terminated successfully.", 10124 ); 10125 }; 10126 10127 // Post live-migration check to make sure the destination VM is functional 10128 let r = std::panic::catch_unwind(|| { 10129 // Perform same checks to validate VM has been properly migrated 10130 // Spawn a new netcat listener in the OVS VM 10131 let guest_ip = ovs_guest.network.guest_ip.clone(); 10132 thread::spawn(move || { 10133 ssh_command_ip( 10134 "nc -l 12345", 10135 &guest_ip, 10136 DEFAULT_SSH_RETRIES, 10137 DEFAULT_SSH_TIMEOUT, 10138 ) 10139 .unwrap(); 10140 }); 10141 10142 // Wait for the server to be listening 10143 thread::sleep(std::time::Duration::new(5, 0)); 10144 10145 // And check the connection is still functional after live-migration 10146 migration_guest 10147 .ssh_command("nc -vz 172.100.0.1 12345") 10148 .unwrap(); 10149 }); 10150 10151 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 10152 let _ = dest_child.kill(); 10153 let _ = ovs_child.kill(); 10154 let dest_output = dest_child.wait_with_output().unwrap(); 10155 let ovs_output = ovs_child.wait_with_output().unwrap(); 10156 10157 cleanup_ovs_dpdk(); 10158 10159 handle_child_output(r, &dest_output); 10160 handle_child_output(Ok(()), &ovs_output); 10161 } 10162 10163 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 10164 // same host with Landlock enabled on both VMs. The test validates the following: 10165 // 1. The source VM is up and functional 10166 // 2. Ensure Landlock is enabled on source VM by hotplugging a disk. As the path for this 10167 // disk is not known to the source VM this step will fail. 10168 // 3. The 'send-migration' and 'receive-migration' command finished successfully; 10169 // 4. The source VM terminated gracefully after live migration; 10170 // 5. The destination VM is functional after live migration; 10171 // 6. Ensure Landlock is enabled on destination VM by hotplugging a disk. As the path for 10172 // this disk is not known to the destination VM this step will fail. 10173 fn _test_live_migration_with_landlock() { 10174 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10175 let guest = Guest::new(Box::new(focal)); 10176 let kernel_path = direct_kernel_boot_path(); 10177 let net_id = "net123"; 10178 let net_params = format!( 10179 "id={},tap=,mac={},ip={},mask=255.255.255.0", 10180 net_id, guest.network.guest_mac, guest.network.host_ip 10181 ); 10182 10183 let boot_vcpus = 2; 10184 let max_vcpus = 4; 10185 10186 let mut blk_file_path = dirs::home_dir().unwrap(); 10187 blk_file_path.push("workloads"); 10188 blk_file_path.push("blk.img"); 10189 10190 let src_api_socket = temp_api_path(&guest.tmp_dir); 10191 let mut src_child = GuestCommand::new(&guest) 10192 .args([ 10193 "--cpus", 10194 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 10195 ]) 10196 .args(["--memory", "size=4G,shared=on"]) 10197 .args(["--kernel", kernel_path.to_str().unwrap()]) 10198 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10199 .default_disks() 10200 .args(["--api-socket", &src_api_socket]) 10201 .args(["--landlock"]) 10202 .args(["--net", net_params.as_str()]) 10203 .args([ 10204 "--landlock-rules", 10205 format!("path={:?},access=rw", guest.tmp_dir.as_path()).as_str(), 10206 ]) 10207 .capture_output() 10208 .spawn() 10209 .unwrap(); 10210 10211 // Start the destination VM 10212 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 10213 dest_api_socket.push_str(".dest"); 10214 let mut dest_child = GuestCommand::new(&guest) 10215 .args(["--api-socket", &dest_api_socket]) 10216 .capture_output() 10217 .spawn() 10218 .unwrap(); 10219 10220 let r = std::panic::catch_unwind(|| { 10221 guest.wait_vm_boot(None).unwrap(); 10222 10223 // Make sure the source VM is functaionl 10224 // Check the number of vCPUs 10225 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10226 10227 // Check the guest RAM 10228 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10229 10230 // Check Landlock is enabled by hot-plugging a disk. 10231 assert!(!remote_command( 10232 &src_api_socket, 10233 "add-disk", 10234 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 10235 )); 10236 10237 // Start the live-migration 10238 let migration_socket = String::from( 10239 guest 10240 .tmp_dir 10241 .as_path() 10242 .join("live-migration.sock") 10243 .to_str() 10244 .unwrap(), 10245 ); 10246 10247 assert!( 10248 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, true), 10249 "Unsuccessful command: 'send-migration' or 'receive-migration'." 10250 ); 10251 }); 10252 10253 // Check and report any errors occurred during the live-migration 10254 if r.is_err() { 10255 print_and_panic( 10256 src_child, 10257 dest_child, 10258 None, 10259 "Error occurred during live-migration", 10260 ); 10261 } 10262 10263 // Check the source vm has been terminated successful (give it '3s' to settle) 10264 thread::sleep(std::time::Duration::new(3, 0)); 10265 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 10266 print_and_panic( 10267 src_child, 10268 dest_child, 10269 None, 10270 "source VM was not terminated successfully.", 10271 ); 10272 }; 10273 10274 // Post live-migration check to make sure the destination VM is funcational 10275 let r = std::panic::catch_unwind(|| { 10276 // Perform same checks to validate VM has been properly migrated 10277 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10278 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10279 }); 10280 10281 // Check Landlock is enabled on destination VM by hot-plugging a disk. 10282 assert!(!remote_command( 10283 &dest_api_socket, 10284 "add-disk", 10285 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 10286 )); 10287 10288 // Clean-up the destination VM and make sure it terminated correctly 10289 let _ = dest_child.kill(); 10290 let dest_output = dest_child.wait_with_output().unwrap(); 10291 handle_child_output(r, &dest_output); 10292 } 10293 10294 mod live_migration_parallel { 10295 use super::*; 10296 #[test] 10297 fn test_live_migration_basic() { 10298 _test_live_migration(false, false) 10299 } 10300 10301 #[test] 10302 fn test_live_migration_local() { 10303 _test_live_migration(false, true) 10304 } 10305 10306 #[test] 10307 fn test_live_migration_watchdog() { 10308 _test_live_migration_watchdog(false, false) 10309 } 10310 10311 #[test] 10312 fn test_live_migration_watchdog_local() { 10313 _test_live_migration_watchdog(false, true) 10314 } 10315 10316 #[test] 10317 fn test_live_upgrade_basic() { 10318 _test_live_migration(true, false) 10319 } 10320 10321 #[test] 10322 fn test_live_upgrade_local() { 10323 _test_live_migration(true, true) 10324 } 10325 10326 #[test] 10327 fn test_live_upgrade_watchdog() { 10328 _test_live_migration_watchdog(true, false) 10329 } 10330 10331 #[test] 10332 fn test_live_upgrade_watchdog_local() { 10333 _test_live_migration_watchdog(true, true) 10334 } 10335 #[test] 10336 #[cfg(target_arch = "x86_64")] 10337 fn test_live_migration_with_landlock() { 10338 _test_live_migration_with_landlock() 10339 } 10340 } 10341 10342 mod live_migration_sequential { 10343 use super::*; 10344 10345 // NUMA & balloon live migration tests are large so run sequentially 10346 10347 #[test] 10348 fn test_live_migration_balloon() { 10349 _test_live_migration_balloon(false, false) 10350 } 10351 10352 #[test] 10353 fn test_live_migration_balloon_local() { 10354 _test_live_migration_balloon(false, true) 10355 } 10356 10357 #[test] 10358 fn test_live_upgrade_balloon() { 10359 _test_live_migration_balloon(true, false) 10360 } 10361 10362 #[test] 10363 fn test_live_upgrade_balloon_local() { 10364 _test_live_migration_balloon(true, true) 10365 } 10366 10367 #[test] 10368 #[cfg(not(feature = "mshv"))] 10369 fn test_live_migration_numa() { 10370 _test_live_migration_numa(false, false) 10371 } 10372 10373 #[test] 10374 #[cfg(not(feature = "mshv"))] 10375 fn test_live_migration_numa_local() { 10376 _test_live_migration_numa(false, true) 10377 } 10378 10379 #[test] 10380 #[cfg(not(feature = "mshv"))] 10381 fn test_live_upgrade_numa() { 10382 _test_live_migration_numa(true, false) 10383 } 10384 10385 #[test] 10386 #[cfg(not(feature = "mshv"))] 10387 fn test_live_upgrade_numa_local() { 10388 _test_live_migration_numa(true, true) 10389 } 10390 10391 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 10392 #[test] 10393 #[ignore = "See #5532"] 10394 #[cfg(target_arch = "x86_64")] 10395 #[cfg(not(feature = "mshv"))] 10396 fn test_live_migration_ovs_dpdk() { 10397 _test_live_migration_ovs_dpdk(false, false); 10398 } 10399 10400 #[test] 10401 #[cfg(target_arch = "x86_64")] 10402 #[cfg(not(feature = "mshv"))] 10403 fn test_live_migration_ovs_dpdk_local() { 10404 _test_live_migration_ovs_dpdk(false, true); 10405 } 10406 10407 #[test] 10408 #[ignore = "See #5532"] 10409 #[cfg(target_arch = "x86_64")] 10410 #[cfg(not(feature = "mshv"))] 10411 fn test_live_upgrade_ovs_dpdk() { 10412 _test_live_migration_ovs_dpdk(true, false); 10413 } 10414 10415 #[test] 10416 #[ignore = "See #5532"] 10417 #[cfg(target_arch = "x86_64")] 10418 #[cfg(not(feature = "mshv"))] 10419 fn test_live_upgrade_ovs_dpdk_local() { 10420 _test_live_migration_ovs_dpdk(true, true); 10421 } 10422 } 10423 } 10424 10425 #[cfg(target_arch = "aarch64")] 10426 mod aarch64_acpi { 10427 use crate::*; 10428 10429 #[test] 10430 fn test_simple_launch_acpi() { 10431 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10432 10433 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 10434 let guest = Guest::new(disk_config); 10435 10436 let mut child = GuestCommand::new(&guest) 10437 .args(["--cpus", "boot=1"]) 10438 .args(["--memory", "size=512M"]) 10439 .args(["--kernel", edk2_path().to_str().unwrap()]) 10440 .default_disks() 10441 .default_net() 10442 .args(["--serial", "tty", "--console", "off"]) 10443 .capture_output() 10444 .spawn() 10445 .unwrap(); 10446 10447 let r = std::panic::catch_unwind(|| { 10448 guest.wait_vm_boot(Some(120)).unwrap(); 10449 10450 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 10451 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 10452 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 10453 }); 10454 10455 let _ = child.kill(); 10456 let output = child.wait_with_output().unwrap(); 10457 10458 handle_child_output(r, &output); 10459 }); 10460 } 10461 10462 #[test] 10463 fn test_guest_numa_nodes_acpi() { 10464 _test_guest_numa_nodes(true); 10465 } 10466 10467 #[test] 10468 fn test_cpu_topology_421_acpi() { 10469 test_cpu_topology(4, 2, 1, true); 10470 } 10471 10472 #[test] 10473 fn test_cpu_topology_142_acpi() { 10474 test_cpu_topology(1, 4, 2, true); 10475 } 10476 10477 #[test] 10478 fn test_cpu_topology_262_acpi() { 10479 test_cpu_topology(2, 6, 2, true); 10480 } 10481 10482 #[test] 10483 fn test_power_button_acpi() { 10484 _test_power_button(true); 10485 } 10486 10487 #[test] 10488 fn test_virtio_iommu() { 10489 _test_virtio_iommu(true) 10490 } 10491 } 10492 10493 mod rate_limiter { 10494 use super::*; 10495 10496 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 10497 // compared to given 'limit' rate. 10498 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 10499 let upper_limit = limit * (1_f64 + difference); 10500 let lower_limit = limit * (1_f64 - difference); 10501 10502 if measured > lower_limit && measured < upper_limit { 10503 return true; 10504 } 10505 10506 eprintln!( 10507 "\n\n==== Start 'check_rate_limit' failed ==== \ 10508 \n\nmeasured={measured}, , lower_limit={lower_limit}, upper_limit={upper_limit} \ 10509 \n\n==== End 'check_rate_limit' failed ====\n\n" 10510 ); 10511 10512 false 10513 } 10514 10515 fn _test_rate_limiter_net(rx: bool) { 10516 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10517 let guest = Guest::new(Box::new(focal)); 10518 10519 let test_timeout = 10; 10520 let num_queues = 2; 10521 let queue_size = 256; 10522 let bw_size = 10485760_u64; // bytes 10523 let bw_refill_time = 100; // ms 10524 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 10525 10526 let net_params = format!( 10527 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 10528 guest.network.guest_mac, 10529 guest.network.host_ip, 10530 num_queues, 10531 queue_size, 10532 bw_size, 10533 bw_refill_time, 10534 ); 10535 10536 let mut child = GuestCommand::new(&guest) 10537 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 10538 .args(["--memory", "size=4G"]) 10539 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10540 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10541 .default_disks() 10542 .args(["--net", net_params.as_str()]) 10543 .capture_output() 10544 .spawn() 10545 .unwrap(); 10546 10547 let r = std::panic::catch_unwind(|| { 10548 guest.wait_vm_boot(None).unwrap(); 10549 let measured_bps = 10550 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, true) 10551 .unwrap(); 10552 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 10553 }); 10554 10555 let _ = child.kill(); 10556 let output = child.wait_with_output().unwrap(); 10557 handle_child_output(r, &output); 10558 } 10559 10560 #[test] 10561 fn test_rate_limiter_net_rx() { 10562 _test_rate_limiter_net(true); 10563 } 10564 10565 #[test] 10566 fn test_rate_limiter_net_tx() { 10567 _test_rate_limiter_net(false); 10568 } 10569 10570 fn _test_rate_limiter_block(bandwidth: bool, num_queues: u32) { 10571 let test_timeout = 10; 10572 let fio_ops = FioOps::RandRW; 10573 10574 let bw_size = if bandwidth { 10575 10485760_u64 // bytes 10576 } else { 10577 100_u64 // I/O 10578 }; 10579 let bw_refill_time = 100; // ms 10580 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10581 10582 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10583 let guest = Guest::new(Box::new(focal)); 10584 let api_socket = temp_api_path(&guest.tmp_dir); 10585 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10586 let blk_rate_limiter_test_img = 10587 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 10588 10589 // Create the test block image 10590 assert!(exec_host_command_output(&format!( 10591 "dd if=/dev/zero of={blk_rate_limiter_test_img} bs=1M count=1024" 10592 )) 10593 .status 10594 .success()); 10595 10596 let test_blk_params = if bandwidth { 10597 format!( 10598 "path={blk_rate_limiter_test_img},num_queues={num_queues},bw_size={bw_size},bw_refill_time={bw_refill_time}" 10599 ) 10600 } else { 10601 format!( 10602 "path={blk_rate_limiter_test_img},num_queues={num_queues},ops_size={bw_size},ops_refill_time={bw_refill_time}" 10603 ) 10604 }; 10605 10606 let mut child = GuestCommand::new(&guest) 10607 .args(["--cpus", &format!("boot={num_queues}")]) 10608 .args(["--memory", "size=4G"]) 10609 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10610 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10611 .args([ 10612 "--disk", 10613 format!( 10614 "path={}", 10615 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10616 ) 10617 .as_str(), 10618 format!( 10619 "path={}", 10620 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10621 ) 10622 .as_str(), 10623 test_blk_params.as_str(), 10624 ]) 10625 .default_net() 10626 .args(["--api-socket", &api_socket]) 10627 .capture_output() 10628 .spawn() 10629 .unwrap(); 10630 10631 let r = std::panic::catch_unwind(|| { 10632 guest.wait_vm_boot(None).unwrap(); 10633 10634 let fio_command = format!( 10635 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 10636 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10637 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10638 ); 10639 let output = guest.ssh_command(&fio_command).unwrap(); 10640 10641 // Parse fio output 10642 let measured_rate = if bandwidth { 10643 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 10644 } else { 10645 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 10646 }; 10647 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10648 }); 10649 10650 let _ = child.kill(); 10651 let output = child.wait_with_output().unwrap(); 10652 handle_child_output(r, &output); 10653 } 10654 10655 fn _test_rate_limiter_group_block(bandwidth: bool, num_queues: u32, num_disks: u32) { 10656 let test_timeout = 10; 10657 let fio_ops = FioOps::RandRW; 10658 10659 let bw_size = if bandwidth { 10660 10485760_u64 // bytes 10661 } else { 10662 100_u64 // I/O 10663 }; 10664 let bw_refill_time = 100; // ms 10665 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10666 10667 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10668 let guest = Guest::new(Box::new(focal)); 10669 let api_socket = temp_api_path(&guest.tmp_dir); 10670 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10671 10672 let rate_limit_group_arg = if bandwidth { 10673 format!("id=group0,bw_size={bw_size},bw_refill_time={bw_refill_time}") 10674 } else { 10675 format!("id=group0,ops_size={bw_size},ops_refill_time={bw_refill_time}") 10676 }; 10677 10678 let mut disk_args = vec![ 10679 "--disk".to_string(), 10680 format!( 10681 "path={}", 10682 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10683 ), 10684 format!( 10685 "path={}", 10686 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10687 ), 10688 ]; 10689 10690 for i in 0..num_disks { 10691 let test_img_path = String::from( 10692 test_img_dir 10693 .as_path() 10694 .join(format!("blk{}.img", i)) 10695 .to_str() 10696 .unwrap(), 10697 ); 10698 10699 assert!(exec_host_command_output(&format!( 10700 "dd if=/dev/zero of={test_img_path} bs=1M count=1024" 10701 )) 10702 .status 10703 .success()); 10704 10705 disk_args.push(format!( 10706 "path={test_img_path},num_queues={num_queues},rate_limit_group=group0" 10707 )); 10708 } 10709 10710 let mut child = GuestCommand::new(&guest) 10711 .args(["--cpus", &format!("boot={}", num_queues * num_disks)]) 10712 .args(["--memory", "size=4G"]) 10713 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10714 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10715 .args(["--rate-limit-group", &rate_limit_group_arg]) 10716 .args(disk_args) 10717 .default_net() 10718 .args(["--api-socket", &api_socket]) 10719 .capture_output() 10720 .spawn() 10721 .unwrap(); 10722 10723 let r = std::panic::catch_unwind(|| { 10724 guest.wait_vm_boot(None).unwrap(); 10725 10726 let mut fio_command = format!( 10727 "sudo fio --name=global --output-format=json \ 10728 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10729 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10730 ); 10731 10732 // Generate additional argument for each disk: 10733 // --name=job0 --filename=/dev/vdc \ 10734 // --name=job1 --filename=/dev/vdd \ 10735 // --name=job2 --filename=/dev/vde \ 10736 // ... 10737 for i in 0..num_disks { 10738 let c: char = 'c'; 10739 let arg = format!( 10740 " --name=job{i} --filename=/dev/vd{}", 10741 char::from_u32((c as u32) + i).unwrap() 10742 ); 10743 fio_command += &arg; 10744 } 10745 let output = guest.ssh_command(&fio_command).unwrap(); 10746 10747 // Parse fio output 10748 let measured_rate = if bandwidth { 10749 parse_fio_output(&output, &fio_ops, num_queues * num_disks).unwrap() 10750 } else { 10751 parse_fio_output_iops(&output, &fio_ops, num_queues * num_disks).unwrap() 10752 }; 10753 assert!(check_rate_limit(measured_rate, limit_rate, 0.2)); 10754 }); 10755 10756 let _ = child.kill(); 10757 let output = child.wait_with_output().unwrap(); 10758 handle_child_output(r, &output); 10759 } 10760 10761 #[test] 10762 fn test_rate_limiter_block_bandwidth() { 10763 _test_rate_limiter_block(true, 1); 10764 _test_rate_limiter_block(true, 2) 10765 } 10766 10767 #[test] 10768 fn test_rate_limiter_group_block_bandwidth() { 10769 _test_rate_limiter_group_block(true, 1, 1); 10770 _test_rate_limiter_group_block(true, 2, 1); 10771 _test_rate_limiter_group_block(true, 1, 2); 10772 _test_rate_limiter_group_block(true, 2, 2); 10773 } 10774 10775 #[test] 10776 fn test_rate_limiter_block_iops() { 10777 _test_rate_limiter_block(false, 1); 10778 _test_rate_limiter_block(false, 2); 10779 } 10780 10781 #[test] 10782 fn test_rate_limiter_group_block_iops() { 10783 _test_rate_limiter_group_block(false, 1, 1); 10784 _test_rate_limiter_group_block(false, 2, 1); 10785 _test_rate_limiter_group_block(false, 1, 2); 10786 _test_rate_limiter_group_block(false, 2, 2); 10787 } 10788 } 10789