1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 use std::sync::Mutex; 28 use std::thread; 29 use test_infra::*; 30 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 31 use wait_timeout::ChildExt; 32 33 // Constant taken from the VMM crate. 34 const MAX_NUM_PCI_SEGMENTS: u16 = 96; 35 36 #[cfg(target_arch = "x86_64")] 37 mod x86_64 { 38 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 39 pub const JAMMY_NVIDIA_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-nvidia.raw"; 40 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 41 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 42 "focal-server-cloudimg-amd64-custom-20210609-0-backing.qcow2"; 43 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 44 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 45 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20230119-0.raw"; 46 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2022-amd64-2.raw"; 47 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 48 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 49 } 50 51 #[cfg(target_arch = "x86_64")] 52 use x86_64::*; 53 54 #[cfg(target_arch = "aarch64")] 55 mod aarch64 { 56 pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 57 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 58 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 59 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 60 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 61 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 62 "focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2"; 63 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 64 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 65 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 66 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 67 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 68 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 69 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 70 } 71 72 #[cfg(target_arch = "aarch64")] 73 use aarch64::*; 74 75 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 76 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 77 78 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 79 80 // This enum exists to make it more convenient to 81 // implement test for both D-Bus and REST APIs. 82 enum TargetApi { 83 // API socket 84 HttpApi(String), 85 // well known service name, object path 86 DBusApi(String, String), 87 } 88 89 impl TargetApi { 90 fn new_http_api(tmp_dir: &TempDir) -> Self { 91 Self::HttpApi(temp_api_path(tmp_dir)) 92 } 93 94 fn new_dbus_api(tmp_dir: &TempDir) -> Self { 95 // `tmp_dir` is in the form of "/tmp/chXXXXXX" 96 // and we take the `chXXXXXX` part as a unique identifier for the guest 97 let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); 98 99 Self::DBusApi( 100 format!("org.cloudhypervisor.{id}"), 101 format!("/org/cloudhypervisor/{id}"), 102 ) 103 } 104 105 fn guest_args(&self) -> Vec<String> { 106 match self { 107 TargetApi::HttpApi(api_socket) => { 108 vec![format!("--api-socket={}", api_socket.as_str())] 109 } 110 TargetApi::DBusApi(service_name, object_path) => { 111 vec![ 112 format!("--dbus-service-name={}", service_name.as_str()), 113 format!("--dbus-object-path={}", object_path.as_str()), 114 ] 115 } 116 } 117 } 118 119 fn remote_args(&self) -> Vec<String> { 120 // `guest_args` and `remote_args` are consistent with each other 121 self.guest_args() 122 } 123 124 fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { 125 let mut cmd = Command::new(clh_command("ch-remote")); 126 cmd.args(self.remote_args()); 127 cmd.arg(command); 128 129 if let Some(arg) = arg { 130 cmd.arg(arg); 131 } 132 133 let output = cmd.output().unwrap(); 134 if output.status.success() { 135 true 136 } else { 137 eprintln!("Error running ch-remote command: {:?}", &cmd); 138 let stderr = String::from_utf8_lossy(&output.stderr); 139 eprintln!("stderr: {stderr}"); 140 false 141 } 142 } 143 } 144 145 // Start cloud-hypervisor with no VM parameters, only the API server running. 146 // From the API: Create a VM, boot it and check that it looks as expected. 147 fn _test_api_create_boot(target_api: TargetApi, guest: Guest) { 148 let mut child = GuestCommand::new(&guest) 149 .args(target_api.guest_args()) 150 .capture_output() 151 .spawn() 152 .unwrap(); 153 154 thread::sleep(std::time::Duration::new(1, 0)); 155 156 // Verify API server is running 157 assert!(target_api.remote_command("ping", None)); 158 159 // Create the VM first 160 let cpu_count: u8 = 4; 161 let request_body = guest.api_create_body( 162 cpu_count, 163 direct_kernel_boot_path().to_str().unwrap(), 164 DIRECT_KERNEL_BOOT_CMDLINE, 165 ); 166 167 let temp_config_path = guest.tmp_dir.as_path().join("config"); 168 std::fs::write(&temp_config_path, request_body).unwrap(); 169 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 170 171 assert!(target_api.remote_command("create", Some(create_config),)); 172 173 // Then boot it 174 assert!(target_api.remote_command("boot", None)); 175 thread::sleep(std::time::Duration::new(20, 0)); 176 177 let r = std::panic::catch_unwind(|| { 178 // Check that the VM booted as expected 179 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 180 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 181 }); 182 183 let _ = child.kill(); 184 let output = child.wait_with_output().unwrap(); 185 186 handle_child_output(r, &output); 187 } 188 189 // Start cloud-hypervisor with no VM parameters, only the API server running. 190 // From the API: Create a VM, boot it and check it can be shutdown and then 191 // booted again 192 fn _test_api_shutdown(target_api: TargetApi, guest: Guest) { 193 let mut child = GuestCommand::new(&guest) 194 .args(target_api.guest_args()) 195 .capture_output() 196 .spawn() 197 .unwrap(); 198 199 thread::sleep(std::time::Duration::new(1, 0)); 200 201 // Verify API server is running 202 assert!(target_api.remote_command("ping", None)); 203 204 // Create the VM first 205 let cpu_count: u8 = 4; 206 let request_body = guest.api_create_body( 207 cpu_count, 208 direct_kernel_boot_path().to_str().unwrap(), 209 DIRECT_KERNEL_BOOT_CMDLINE, 210 ); 211 212 let temp_config_path = guest.tmp_dir.as_path().join("config"); 213 std::fs::write(&temp_config_path, request_body).unwrap(); 214 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 215 216 let r = std::panic::catch_unwind(|| { 217 assert!(target_api.remote_command("create", Some(create_config))); 218 219 // Then boot it 220 assert!(target_api.remote_command("boot", None)); 221 222 guest.wait_vm_boot(None).unwrap(); 223 224 // Check that the VM booted as expected 225 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 226 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 227 228 // Sync and shutdown without powering off to prevent filesystem 229 // corruption. 230 guest.ssh_command("sync").unwrap(); 231 guest.ssh_command("sudo shutdown -H now").unwrap(); 232 233 // Wait for the guest to be fully shutdown 234 thread::sleep(std::time::Duration::new(20, 0)); 235 236 // Then shut it down 237 assert!(target_api.remote_command("shutdown", None)); 238 239 // Then boot it again 240 assert!(target_api.remote_command("boot", None)); 241 242 guest.wait_vm_boot(None).unwrap(); 243 244 // Check that the VM booted as expected 245 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 246 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 247 }); 248 249 let _ = child.kill(); 250 let output = child.wait_with_output().unwrap(); 251 252 handle_child_output(r, &output); 253 } 254 255 // Start cloud-hypervisor with no VM parameters, only the API server running. 256 // From the API: Create a VM, boot it and check it can be deleted and then recreated 257 // booted again. 258 fn _test_api_delete(target_api: TargetApi, guest: Guest) { 259 let mut child = GuestCommand::new(&guest) 260 .args(target_api.guest_args()) 261 .capture_output() 262 .spawn() 263 .unwrap(); 264 265 thread::sleep(std::time::Duration::new(1, 0)); 266 267 // Verify API server is running 268 assert!(target_api.remote_command("ping", None)); 269 270 // Create the VM first 271 let cpu_count: u8 = 4; 272 let request_body = guest.api_create_body( 273 cpu_count, 274 direct_kernel_boot_path().to_str().unwrap(), 275 DIRECT_KERNEL_BOOT_CMDLINE, 276 ); 277 let temp_config_path = guest.tmp_dir.as_path().join("config"); 278 std::fs::write(&temp_config_path, request_body).unwrap(); 279 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 280 281 let r = std::panic::catch_unwind(|| { 282 assert!(target_api.remote_command("create", Some(create_config))); 283 284 // Then boot it 285 assert!(target_api.remote_command("boot", None)); 286 287 guest.wait_vm_boot(None).unwrap(); 288 289 // Check that the VM booted as expected 290 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 291 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 292 293 // Sync and shutdown without powering off to prevent filesystem 294 // corruption. 295 guest.ssh_command("sync").unwrap(); 296 guest.ssh_command("sudo shutdown -H now").unwrap(); 297 298 // Wait for the guest to be fully shutdown 299 thread::sleep(std::time::Duration::new(20, 0)); 300 301 // Then delete it 302 assert!(target_api.remote_command("delete", None)); 303 304 assert!(target_api.remote_command("create", Some(create_config))); 305 306 // Then boot it again 307 assert!(target_api.remote_command("boot", None)); 308 309 guest.wait_vm_boot(None).unwrap(); 310 311 // Check that the VM booted as expected 312 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 313 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 314 }); 315 316 let _ = child.kill(); 317 let output = child.wait_with_output().unwrap(); 318 319 handle_child_output(r, &output); 320 } 321 322 // Start cloud-hypervisor with no VM parameters, only the API server running. 323 // From the API: Create a VM, boot it and check that it looks as expected. 324 // Then we pause the VM, check that it's no longer available. 325 // Finally we resume the VM and check that it's available. 326 fn _test_api_pause_resume(target_api: TargetApi, guest: Guest) { 327 let mut child = GuestCommand::new(&guest) 328 .args(target_api.guest_args()) 329 .capture_output() 330 .spawn() 331 .unwrap(); 332 333 thread::sleep(std::time::Duration::new(1, 0)); 334 335 // Verify API server is running 336 assert!(target_api.remote_command("ping", None)); 337 338 // Create the VM first 339 let cpu_count: u8 = 4; 340 let request_body = guest.api_create_body( 341 cpu_count, 342 direct_kernel_boot_path().to_str().unwrap(), 343 DIRECT_KERNEL_BOOT_CMDLINE, 344 ); 345 346 let temp_config_path = guest.tmp_dir.as_path().join("config"); 347 std::fs::write(&temp_config_path, request_body).unwrap(); 348 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 349 350 assert!(target_api.remote_command("create", Some(create_config))); 351 352 // Then boot it 353 assert!(target_api.remote_command("boot", None)); 354 thread::sleep(std::time::Duration::new(20, 0)); 355 356 let r = std::panic::catch_unwind(|| { 357 // Check that the VM booted as expected 358 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 359 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 360 361 // We now pause the VM 362 assert!(target_api.remote_command("pause", None)); 363 364 // Check pausing again fails 365 assert!(!target_api.remote_command("pause", None)); 366 367 thread::sleep(std::time::Duration::new(2, 0)); 368 369 // SSH into the VM should fail 370 assert!(ssh_command_ip( 371 "grep -c processor /proc/cpuinfo", 372 &guest.network.guest_ip, 373 2, 374 5 375 ) 376 .is_err()); 377 378 // Resume the VM 379 assert!(target_api.remote_command("resume", None)); 380 381 // Check resuming again fails 382 assert!(!target_api.remote_command("resume", None)); 383 384 thread::sleep(std::time::Duration::new(2, 0)); 385 386 // Now we should be able to SSH back in and get the right number of CPUs 387 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 388 }); 389 390 let _ = child.kill(); 391 let output = child.wait_with_output().unwrap(); 392 393 handle_child_output(r, &output); 394 } 395 396 fn _test_pty_interaction(pty_path: PathBuf) { 397 let mut cf = std::fs::OpenOptions::new() 398 .write(true) 399 .read(true) 400 .open(pty_path) 401 .unwrap(); 402 403 // Some dumb sleeps but we don't want to write 404 // before the console is up and we don't want 405 // to try and write the next line before the 406 // login process is ready. 407 thread::sleep(std::time::Duration::new(5, 0)); 408 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 409 thread::sleep(std::time::Duration::new(2, 0)); 410 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 411 thread::sleep(std::time::Duration::new(2, 0)); 412 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 413 thread::sleep(std::time::Duration::new(2, 0)); 414 415 // read pty and ensure they have a login shell 416 // some fairly hacky workarounds to avoid looping 417 // forever in case the channel is blocked getting output 418 let ptyc = pty_read(cf); 419 let mut empty = 0; 420 let mut prev = String::new(); 421 loop { 422 thread::sleep(std::time::Duration::new(2, 0)); 423 match ptyc.try_recv() { 424 Ok(line) => { 425 empty = 0; 426 prev = prev + &line; 427 if prev.contains("test_pty_console") { 428 break; 429 } 430 } 431 Err(mpsc::TryRecvError::Empty) => { 432 empty += 1; 433 assert!(empty <= 5, "No login on pty"); 434 } 435 _ => { 436 panic!("No login on pty") 437 } 438 } 439 } 440 } 441 442 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 443 let mut workload_path = dirs::home_dir().unwrap(); 444 workload_path.push("workloads"); 445 446 let mut virtiofsd_path = workload_path; 447 virtiofsd_path.push("virtiofsd"); 448 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 449 450 let virtiofsd_socket_path = 451 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 452 453 // Start the daemon 454 let child = Command::new(virtiofsd_path.as_str()) 455 .args(["--shared-dir", shared_dir]) 456 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 457 .args(["--cache", "never"]) 458 .spawn() 459 .unwrap(); 460 461 thread::sleep(std::time::Duration::new(10, 0)); 462 463 (child, virtiofsd_socket_path) 464 } 465 466 fn prepare_vubd( 467 tmp_dir: &TempDir, 468 blk_img: &str, 469 num_queues: usize, 470 rdonly: bool, 471 direct: bool, 472 ) -> (std::process::Child, String) { 473 let mut workload_path = dirs::home_dir().unwrap(); 474 workload_path.push("workloads"); 475 476 let mut blk_file_path = workload_path; 477 blk_file_path.push(blk_img); 478 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 479 480 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 481 482 // Start the daemon 483 let child = Command::new(clh_command("vhost_user_block")) 484 .args([ 485 "--block-backend", 486 format!( 487 "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" 488 ) 489 .as_str(), 490 ]) 491 .spawn() 492 .unwrap(); 493 494 thread::sleep(std::time::Duration::new(10, 0)); 495 496 (child, vubd_socket_path) 497 } 498 499 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 500 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 501 } 502 503 fn temp_api_path(tmp_dir: &TempDir) -> String { 504 String::from( 505 tmp_dir 506 .as_path() 507 .join("cloud-hypervisor.sock") 508 .to_str() 509 .unwrap(), 510 ) 511 } 512 513 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 514 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 515 } 516 517 // Creates the directory and returns the path. 518 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 519 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 520 std::fs::create_dir(&snapshot_dir).unwrap(); 521 snapshot_dir 522 } 523 524 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 525 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 526 vmcore_file 527 } 528 529 // Creates the path for direct kernel boot and return the path. 530 // For x86_64, this function returns the vmlinux kernel path. 531 // For AArch64, this function returns the PE kernel path. 532 fn direct_kernel_boot_path() -> PathBuf { 533 let mut workload_path = dirs::home_dir().unwrap(); 534 workload_path.push("workloads"); 535 536 let mut kernel_path = workload_path; 537 #[cfg(target_arch = "x86_64")] 538 kernel_path.push("vmlinux"); 539 #[cfg(target_arch = "aarch64")] 540 kernel_path.push("Image"); 541 542 kernel_path 543 } 544 545 fn edk2_path() -> PathBuf { 546 let mut workload_path = dirs::home_dir().unwrap(); 547 workload_path.push("workloads"); 548 let mut edk2_path = workload_path; 549 edk2_path.push(OVMF_NAME); 550 551 edk2_path 552 } 553 554 fn cloud_hypervisor_release_path() -> String { 555 let mut workload_path = dirs::home_dir().unwrap(); 556 workload_path.push("workloads"); 557 558 let mut ch_release_path = workload_path; 559 #[cfg(target_arch = "x86_64")] 560 ch_release_path.push("cloud-hypervisor-static"); 561 #[cfg(target_arch = "aarch64")] 562 ch_release_path.push("cloud-hypervisor-static-aarch64"); 563 564 ch_release_path.into_os_string().into_string().unwrap() 565 } 566 567 fn prepare_vhost_user_net_daemon( 568 tmp_dir: &TempDir, 569 ip: &str, 570 tap: Option<&str>, 571 mtu: Option<u16>, 572 num_queues: usize, 573 client_mode: bool, 574 ) -> (std::process::Command, String) { 575 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 576 577 // Start the daemon 578 let mut net_params = format!( 579 "ip={ip},mask=255.255.255.0,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" 580 ); 581 582 if let Some(tap) = tap { 583 net_params.push_str(format!(",tap={tap}").as_str()); 584 } 585 586 if let Some(mtu) = mtu { 587 net_params.push_str(format!(",mtu={mtu}").as_str()); 588 } 589 590 let mut command = Command::new(clh_command("vhost_user_net")); 591 command.args(["--net-backend", net_params.as_str()]); 592 593 (command, vunet_socket_path) 594 } 595 596 fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { 597 let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); 598 let swtpm_socket_path = String::from( 599 tmp_dir 600 .as_path() 601 .join("swtpm") 602 .join("swtpm.sock") 603 .to_str() 604 .unwrap(), 605 ); 606 std::fs::create_dir(&swtpm_tpm_dir).unwrap(); 607 608 let mut swtpm_command = Command::new("swtpm"); 609 let swtpm_args = [ 610 "socket", 611 "--tpmstate", 612 &format!("dir={swtpm_tpm_dir}"), 613 "--ctrl", 614 &format!("type=unixio,path={swtpm_socket_path}"), 615 "--flags", 616 "startup-clear", 617 "--tpm2", 618 ]; 619 swtpm_command.args(swtpm_args); 620 621 (swtpm_command, swtpm_socket_path) 622 } 623 624 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 625 let mut cmd = Command::new(clh_command("ch-remote")); 626 cmd.args([&format!("--api-socket={api_socket}"), command]); 627 628 if let Some(arg) = arg { 629 cmd.arg(arg); 630 } 631 let output = cmd.output().unwrap(); 632 if output.status.success() { 633 true 634 } else { 635 eprintln!("Error running ch-remote command: {:?}", &cmd); 636 let stderr = String::from_utf8_lossy(&output.stderr); 637 eprintln!("stderr: {stderr}"); 638 false 639 } 640 } 641 642 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 643 let mut cmd = Command::new(clh_command("ch-remote")); 644 cmd.args([&format!("--api-socket={api_socket}"), command]); 645 646 if let Some(arg) = arg { 647 cmd.arg(arg); 648 } 649 650 let output = cmd.output().expect("Failed to launch ch-remote"); 651 652 (output.status.success(), output.stdout) 653 } 654 655 fn resize_command( 656 api_socket: &str, 657 desired_vcpus: Option<u8>, 658 desired_ram: Option<usize>, 659 desired_balloon: Option<usize>, 660 event_file: Option<&str>, 661 ) -> bool { 662 let mut cmd = Command::new(clh_command("ch-remote")); 663 cmd.args([&format!("--api-socket={api_socket}"), "resize"]); 664 665 if let Some(desired_vcpus) = desired_vcpus { 666 cmd.arg(format!("--cpus={desired_vcpus}")); 667 } 668 669 if let Some(desired_ram) = desired_ram { 670 cmd.arg(format!("--memory={desired_ram}")); 671 } 672 673 if let Some(desired_balloon) = desired_balloon { 674 cmd.arg(format!("--balloon={desired_balloon}")); 675 } 676 677 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 678 679 if let Some(event_path) = event_file { 680 let latest_events = [ 681 &MetaEvent { 682 event: "resizing".to_string(), 683 device_id: None, 684 }, 685 &MetaEvent { 686 event: "resized".to_string(), 687 device_id: None, 688 }, 689 ]; 690 // See: #5938 691 thread::sleep(std::time::Duration::new(1, 0)); 692 assert!(check_latest_events_exact(&latest_events, event_path)); 693 } 694 695 ret 696 } 697 698 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 699 let mut cmd = Command::new(clh_command("ch-remote")); 700 cmd.args([ 701 &format!("--api-socket={api_socket}"), 702 "resize-zone", 703 &format!("--id={id}"), 704 &format!("--size={desired_size}"), 705 ]); 706 707 cmd.status().expect("Failed to launch ch-remote").success() 708 } 709 710 // setup OVS-DPDK bridge and ports 711 fn setup_ovs_dpdk() { 712 // setup OVS-DPDK 713 assert!(exec_host_command_status("service openvswitch-switch start").success()); 714 assert!(exec_host_command_status("ovs-vsctl init").success()); 715 assert!( 716 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 717 .success() 718 ); 719 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 720 721 // Create OVS-DPDK bridge and ports 722 assert!(exec_host_command_status( 723 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 724 ) 725 .success()); 726 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 727 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 728 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 729 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 730 } 731 fn cleanup_ovs_dpdk() { 732 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 733 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 734 } 735 // Setup two guests and ensure they are connected through ovs-dpdk 736 fn setup_ovs_dpdk_guests( 737 guest1: &Guest, 738 guest2: &Guest, 739 api_socket: &str, 740 release_binary: bool, 741 ) -> (Child, Child) { 742 setup_ovs_dpdk(); 743 744 let clh_path = if !release_binary { 745 clh_command("cloud-hypervisor") 746 } else { 747 cloud_hypervisor_release_path() 748 }; 749 750 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 751 .args(["--cpus", "boot=2"]) 752 .args(["--memory", "size=0,shared=on"]) 753 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 754 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 755 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 756 .default_disks() 757 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 758 .capture_output() 759 .spawn() 760 .unwrap(); 761 762 #[cfg(target_arch = "x86_64")] 763 let guest_net_iface = "ens5"; 764 #[cfg(target_arch = "aarch64")] 765 let guest_net_iface = "enp0s5"; 766 767 let r = std::panic::catch_unwind(|| { 768 guest1.wait_vm_boot(None).unwrap(); 769 770 guest1 771 .ssh_command(&format!( 772 "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" 773 )) 774 .unwrap(); 775 guest1 776 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 777 .unwrap(); 778 779 let guest_ip = guest1.network.guest_ip.clone(); 780 thread::spawn(move || { 781 ssh_command_ip( 782 "nc -l 12345", 783 &guest_ip, 784 DEFAULT_SSH_RETRIES, 785 DEFAULT_SSH_TIMEOUT, 786 ) 787 .unwrap(); 788 }); 789 }); 790 if r.is_err() { 791 cleanup_ovs_dpdk(); 792 793 let _ = child1.kill(); 794 let output = child1.wait_with_output().unwrap(); 795 handle_child_output(r, &output); 796 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 797 } 798 799 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 800 .args(["--api-socket", api_socket]) 801 .args(["--cpus", "boot=2"]) 802 .args(["--memory", "size=0,shared=on"]) 803 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 804 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 805 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 806 .default_disks() 807 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 808 .capture_output() 809 .spawn() 810 .unwrap(); 811 812 let r = std::panic::catch_unwind(|| { 813 guest2.wait_vm_boot(None).unwrap(); 814 815 guest2 816 .ssh_command(&format!( 817 "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" 818 )) 819 .unwrap(); 820 guest2 821 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 822 .unwrap(); 823 824 // Check the connection works properly between the two VMs 825 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 826 }); 827 if r.is_err() { 828 cleanup_ovs_dpdk(); 829 830 let _ = child1.kill(); 831 let _ = child2.kill(); 832 let output = child2.wait_with_output().unwrap(); 833 handle_child_output(r, &output); 834 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 835 } 836 837 (child1, child2) 838 } 839 840 enum FwType { 841 Ovmf, 842 RustHypervisorFirmware, 843 } 844 845 fn fw_path(_fw_type: FwType) -> String { 846 let mut workload_path = dirs::home_dir().unwrap(); 847 workload_path.push("workloads"); 848 849 let mut fw_path = workload_path; 850 #[cfg(target_arch = "aarch64")] 851 fw_path.push("CLOUDHV_EFI.fd"); 852 #[cfg(target_arch = "x86_64")] 853 { 854 match _fw_type { 855 FwType::Ovmf => fw_path.push(OVMF_NAME), 856 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 857 } 858 } 859 860 fw_path.to_str().unwrap().to_string() 861 } 862 863 #[derive(Debug)] 864 struct MetaEvent { 865 event: String, 866 device_id: Option<String>, 867 } 868 869 impl MetaEvent { 870 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 871 let mut matched = false; 872 if v["event"].as_str().unwrap() == self.event { 873 if let Some(device_id) = &self.device_id { 874 if v["properties"]["id"].as_str().unwrap() == device_id { 875 matched = true 876 } 877 } else { 878 matched = true; 879 } 880 } 881 matched 882 } 883 } 884 885 // Parse the event_monitor file based on the format that each event 886 // is followed by a double newline 887 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 888 let content = fs::read(event_file).unwrap(); 889 let mut ret = Vec::new(); 890 for entry in String::from_utf8_lossy(&content) 891 .trim() 892 .split("\n\n") 893 .collect::<Vec<&str>>() 894 { 895 ret.push(serde_json::from_str(entry).unwrap()); 896 } 897 898 ret 899 } 900 901 // Return true if all events from the input 'expected_events' are matched sequentially 902 // with events from the 'event_file' 903 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 904 let json_events = parse_event_file(event_file); 905 let len = expected_events.len(); 906 let mut idx = 0; 907 for e in &json_events { 908 if idx == len { 909 break; 910 } 911 if expected_events[idx].match_with_json_event(e) { 912 idx += 1; 913 } 914 } 915 916 let ret = idx == len; 917 918 if !ret { 919 eprintln!( 920 "\n\n==== Start 'check_sequential_events' failed ==== \ 921 \n\nexpected_events={:?}\nactual_events={:?} \ 922 \n\n==== End 'check_sequential_events' failed ====", 923 expected_events, json_events, 924 ); 925 } 926 927 ret 928 } 929 930 // Return true if all events from the input 'expected_events' are matched exactly 931 // with events from the 'event_file' 932 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 933 let json_events = parse_event_file(event_file); 934 assert!(expected_events.len() <= json_events.len()); 935 let json_events = &json_events[..expected_events.len()]; 936 937 for (idx, e) in json_events.iter().enumerate() { 938 if !expected_events[idx].match_with_json_event(e) { 939 eprintln!( 940 "\n\n==== Start 'check_sequential_events_exact' failed ==== \ 941 \n\nexpected_events={:?}\nactual_events={:?} \ 942 \n\n==== End 'check_sequential_events_exact' failed ====", 943 expected_events, json_events, 944 ); 945 946 return false; 947 } 948 } 949 950 true 951 } 952 953 // Return true if events from the input 'latest_events' are matched exactly 954 // with the most recent events from the 'event_file' 955 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 956 let json_events = parse_event_file(event_file); 957 assert!(latest_events.len() <= json_events.len()); 958 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 959 960 for (idx, e) in json_events.iter().enumerate() { 961 if !latest_events[idx].match_with_json_event(e) { 962 eprintln!( 963 "\n\n==== Start 'check_latest_events_exact' failed ==== \ 964 \n\nexpected_events={:?}\nactual_events={:?} \ 965 \n\n==== End 'check_latest_events_exact' failed ====", 966 latest_events, json_events, 967 ); 968 969 return false; 970 } 971 } 972 973 true 974 } 975 976 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 977 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 978 let guest = Guest::new(Box::new(focal)); 979 let total_vcpus = threads_per_core * cores_per_package * packages; 980 let direct_kernel_boot_path = direct_kernel_boot_path(); 981 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 982 let fw_path = fw_path(FwType::RustHypervisorFirmware); 983 if use_fw { 984 kernel_path = fw_path.as_str(); 985 } 986 987 let mut child = GuestCommand::new(&guest) 988 .args([ 989 "--cpus", 990 &format!( 991 "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" 992 ), 993 ]) 994 .args(["--memory", "size=512M"]) 995 .args(["--kernel", kernel_path]) 996 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 997 .default_disks() 998 .default_net() 999 .capture_output() 1000 .spawn() 1001 .unwrap(); 1002 1003 let r = std::panic::catch_unwind(|| { 1004 guest.wait_vm_boot(None).unwrap(); 1005 assert_eq!( 1006 guest.get_cpu_count().unwrap_or_default(), 1007 u32::from(total_vcpus) 1008 ); 1009 assert_eq!( 1010 guest 1011 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1012 .unwrap() 1013 .trim() 1014 .parse::<u8>() 1015 .unwrap_or(0), 1016 threads_per_core 1017 ); 1018 1019 assert_eq!( 1020 guest 1021 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1022 .unwrap() 1023 .trim() 1024 .parse::<u8>() 1025 .unwrap_or(0), 1026 cores_per_package 1027 ); 1028 1029 assert_eq!( 1030 guest 1031 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1032 .unwrap() 1033 .trim() 1034 .parse::<u8>() 1035 .unwrap_or(0), 1036 packages 1037 ); 1038 1039 #[cfg(target_arch = "x86_64")] 1040 { 1041 let mut cpu_id = 0; 1042 for package_id in 0..packages { 1043 for core_id in 0..cores_per_package { 1044 for _ in 0..threads_per_core { 1045 assert_eq!( 1046 guest 1047 .ssh_command(&format!("cat /sys/devices/system/cpu/cpu{cpu_id}/topology/physical_package_id")) 1048 .unwrap() 1049 .trim() 1050 .parse::<u8>() 1051 .unwrap_or(0), 1052 package_id 1053 ); 1054 1055 assert_eq!( 1056 guest 1057 .ssh_command(&format!( 1058 "cat /sys/devices/system/cpu/cpu{cpu_id}/topology/core_id" 1059 )) 1060 .unwrap() 1061 .trim() 1062 .parse::<u8>() 1063 .unwrap_or(0), 1064 core_id 1065 ); 1066 1067 cpu_id += 1; 1068 } 1069 } 1070 } 1071 } 1072 }); 1073 1074 let _ = child.kill(); 1075 let output = child.wait_with_output().unwrap(); 1076 1077 handle_child_output(r, &output); 1078 } 1079 1080 #[allow(unused_variables)] 1081 fn _test_guest_numa_nodes(acpi: bool) { 1082 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1083 let guest = Guest::new(Box::new(focal)); 1084 let api_socket = temp_api_path(&guest.tmp_dir); 1085 #[cfg(target_arch = "x86_64")] 1086 let kernel_path = direct_kernel_boot_path(); 1087 #[cfg(target_arch = "aarch64")] 1088 let kernel_path = if acpi { 1089 edk2_path() 1090 } else { 1091 direct_kernel_boot_path() 1092 }; 1093 1094 let mut child = GuestCommand::new(&guest) 1095 .args(["--cpus", "boot=6,max=12"]) 1096 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 1097 .args([ 1098 "--memory-zone", 1099 "id=mem0,size=1G,hotplug_size=3G", 1100 "id=mem1,size=2G,hotplug_size=3G", 1101 "id=mem2,size=3G,hotplug_size=3G", 1102 ]) 1103 .args([ 1104 "--numa", 1105 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 1106 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 1107 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 1108 ]) 1109 .args(["--kernel", kernel_path.to_str().unwrap()]) 1110 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1111 .args(["--api-socket", &api_socket]) 1112 .capture_output() 1113 .default_disks() 1114 .default_net() 1115 .spawn() 1116 .unwrap(); 1117 1118 let r = std::panic::catch_unwind(|| { 1119 guest.wait_vm_boot(None).unwrap(); 1120 1121 guest.check_numa_common( 1122 Some(&[960_000, 1_920_000, 2_880_000]), 1123 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 1124 Some(&["10 15 20", "20 10 25", "25 30 10"]), 1125 ); 1126 1127 // AArch64 currently does not support hotplug, and therefore we only 1128 // test hotplug-related function on x86_64 here. 1129 #[cfg(target_arch = "x86_64")] 1130 { 1131 guest.enable_memory_hotplug(); 1132 1133 // Resize every memory zone and check each associated NUMA node 1134 // has been assigned the right amount of memory. 1135 resize_zone_command(&api_socket, "mem0", "4G"); 1136 resize_zone_command(&api_socket, "mem1", "4G"); 1137 resize_zone_command(&api_socket, "mem2", "4G"); 1138 // Resize to the maximum amount of CPUs and check each NUMA 1139 // node has been assigned the right CPUs set. 1140 resize_command(&api_socket, Some(12), None, None, None); 1141 thread::sleep(std::time::Duration::new(5, 0)); 1142 1143 guest.check_numa_common( 1144 Some(&[3_840_000, 3_840_000, 3_840_000]), 1145 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 1146 None, 1147 ); 1148 } 1149 }); 1150 1151 let _ = child.kill(); 1152 let output = child.wait_with_output().unwrap(); 1153 1154 handle_child_output(r, &output); 1155 } 1156 1157 #[allow(unused_variables)] 1158 fn _test_power_button(acpi: bool) { 1159 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1160 let guest = Guest::new(Box::new(focal)); 1161 let mut cmd = GuestCommand::new(&guest); 1162 let api_socket = temp_api_path(&guest.tmp_dir); 1163 1164 #[cfg(target_arch = "x86_64")] 1165 let kernel_path = direct_kernel_boot_path(); 1166 #[cfg(target_arch = "aarch64")] 1167 let kernel_path = if acpi { 1168 edk2_path() 1169 } else { 1170 direct_kernel_boot_path() 1171 }; 1172 1173 cmd.args(["--cpus", "boot=1"]) 1174 .args(["--memory", "size=512M"]) 1175 .args(["--kernel", kernel_path.to_str().unwrap()]) 1176 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1177 .capture_output() 1178 .default_disks() 1179 .default_net() 1180 .args(["--api-socket", &api_socket]); 1181 1182 let child = cmd.spawn().unwrap(); 1183 1184 let r = std::panic::catch_unwind(|| { 1185 guest.wait_vm_boot(None).unwrap(); 1186 assert!(remote_command(&api_socket, "power-button", None)); 1187 }); 1188 1189 let output = child.wait_with_output().unwrap(); 1190 assert!(output.status.success()); 1191 handle_child_output(r, &output); 1192 } 1193 1194 type PrepareNetDaemon = dyn Fn( 1195 &TempDir, 1196 &str, 1197 Option<&str>, 1198 Option<u16>, 1199 usize, 1200 bool, 1201 ) -> (std::process::Command, String); 1202 1203 fn test_vhost_user_net( 1204 tap: Option<&str>, 1205 num_queues: usize, 1206 prepare_daemon: &PrepareNetDaemon, 1207 generate_host_mac: bool, 1208 client_mode_daemon: bool, 1209 ) { 1210 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1211 let guest = Guest::new(Box::new(focal)); 1212 let api_socket = temp_api_path(&guest.tmp_dir); 1213 1214 let kernel_path = direct_kernel_boot_path(); 1215 1216 let host_mac = if generate_host_mac { 1217 Some(MacAddr::local_random()) 1218 } else { 1219 None 1220 }; 1221 1222 let mtu = Some(3000); 1223 1224 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 1225 &guest.tmp_dir, 1226 &guest.network.host_ip, 1227 tap, 1228 mtu, 1229 num_queues, 1230 client_mode_daemon, 1231 ); 1232 1233 let net_params = format!( 1234 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 1235 guest.network.guest_mac, 1236 vunet_socket_path, 1237 num_queues, 1238 if let Some(host_mac) = host_mac { 1239 format!(",host_mac={host_mac}") 1240 } else { 1241 "".to_owned() 1242 }, 1243 if client_mode_daemon { 1244 "server" 1245 } else { 1246 "client" 1247 }, 1248 ); 1249 1250 let mut ch_command = GuestCommand::new(&guest); 1251 ch_command 1252 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 1253 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1254 .args(["--kernel", kernel_path.to_str().unwrap()]) 1255 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1256 .default_disks() 1257 .args(["--net", net_params.as_str()]) 1258 .args(["--api-socket", &api_socket]) 1259 .capture_output(); 1260 1261 let mut daemon_child: std::process::Child; 1262 let mut child: std::process::Child; 1263 1264 if client_mode_daemon { 1265 child = ch_command.spawn().unwrap(); 1266 // Make sure the VMM is waiting for the backend to connect 1267 thread::sleep(std::time::Duration::new(10, 0)); 1268 daemon_child = daemon_command.spawn().unwrap(); 1269 } else { 1270 daemon_child = daemon_command.spawn().unwrap(); 1271 // Make sure the backend is waiting for the VMM to connect 1272 thread::sleep(std::time::Duration::new(10, 0)); 1273 child = ch_command.spawn().unwrap(); 1274 } 1275 1276 let r = std::panic::catch_unwind(|| { 1277 guest.wait_vm_boot(None).unwrap(); 1278 1279 if let Some(tap_name) = tap { 1280 let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); 1281 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1282 } 1283 1284 if let Some(host_mac) = tap { 1285 let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); 1286 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1287 } 1288 1289 #[cfg(target_arch = "aarch64")] 1290 let iface = "enp0s4"; 1291 #[cfg(target_arch = "x86_64")] 1292 let iface = "ens4"; 1293 1294 assert_eq!( 1295 guest 1296 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 1297 .unwrap() 1298 .trim(), 1299 "3000" 1300 ); 1301 1302 // 1 network interface + default localhost ==> 2 interfaces 1303 // It's important to note that this test is fully exercising the 1304 // vhost-user-net implementation and the associated backend since 1305 // it does not define any --net network interface. That means all 1306 // the ssh communication in that test happens through the network 1307 // interface backed by vhost-user-net. 1308 assert_eq!( 1309 guest 1310 .ssh_command("ip -o link | wc -l") 1311 .unwrap() 1312 .trim() 1313 .parse::<u32>() 1314 .unwrap_or_default(), 1315 2 1316 ); 1317 1318 // The following pci devices will appear on guest with PCI-MSI 1319 // interrupt vectors assigned. 1320 // 1 virtio-console with 3 vectors: config, Rx, Tx 1321 // 1 virtio-blk with 2 vectors: config, Request 1322 // 1 virtio-blk with 2 vectors: config, Request 1323 // 1 virtio-rng with 2 vectors: config, Request 1324 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1325 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1326 // Based on the above, the total vectors should 14. 1327 #[cfg(target_arch = "x86_64")] 1328 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1329 #[cfg(target_arch = "aarch64")] 1330 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1331 assert_eq!( 1332 guest 1333 .ssh_command(grep_cmd) 1334 .unwrap() 1335 .trim() 1336 .parse::<u32>() 1337 .unwrap_or_default(), 1338 10 + (num_queues as u32) 1339 ); 1340 1341 // ACPI feature is needed. 1342 #[cfg(target_arch = "x86_64")] 1343 { 1344 guest.enable_memory_hotplug(); 1345 1346 // Add RAM to the VM 1347 let desired_ram = 1024 << 20; 1348 resize_command(&api_socket, None, Some(desired_ram), None, None); 1349 1350 thread::sleep(std::time::Duration::new(10, 0)); 1351 1352 // Here by simply checking the size (through ssh), we validate 1353 // the connection is still working, which means vhost-user-net 1354 // keeps working after the resize. 1355 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1356 } 1357 }); 1358 1359 let _ = child.kill(); 1360 let output = child.wait_with_output().unwrap(); 1361 1362 thread::sleep(std::time::Duration::new(5, 0)); 1363 let _ = daemon_child.kill(); 1364 let _ = daemon_child.wait(); 1365 1366 handle_child_output(r, &output); 1367 } 1368 1369 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1370 1371 fn test_vhost_user_blk( 1372 num_queues: usize, 1373 readonly: bool, 1374 direct: bool, 1375 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1376 ) { 1377 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1378 let guest = Guest::new(Box::new(focal)); 1379 let api_socket = temp_api_path(&guest.tmp_dir); 1380 1381 let kernel_path = direct_kernel_boot_path(); 1382 1383 let (blk_params, daemon_child) = { 1384 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1385 // Start the daemon 1386 let (daemon_child, vubd_socket_path) = 1387 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1388 1389 ( 1390 format!( 1391 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1392 ), 1393 Some(daemon_child), 1394 ) 1395 }; 1396 1397 let mut child = GuestCommand::new(&guest) 1398 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1399 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1400 .args(["--kernel", kernel_path.to_str().unwrap()]) 1401 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1402 .args([ 1403 "--disk", 1404 format!( 1405 "path={}", 1406 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1407 ) 1408 .as_str(), 1409 format!( 1410 "path={}", 1411 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1412 ) 1413 .as_str(), 1414 blk_params.as_str(), 1415 ]) 1416 .default_net() 1417 .args(["--api-socket", &api_socket]) 1418 .capture_output() 1419 .spawn() 1420 .unwrap(); 1421 1422 let r = std::panic::catch_unwind(|| { 1423 guest.wait_vm_boot(None).unwrap(); 1424 1425 // Check both if /dev/vdc exists and if the block size is 16M. 1426 assert_eq!( 1427 guest 1428 .ssh_command("lsblk | grep vdc | grep -c 16M") 1429 .unwrap() 1430 .trim() 1431 .parse::<u32>() 1432 .unwrap_or_default(), 1433 1 1434 ); 1435 1436 // Check if this block is RO or RW. 1437 assert_eq!( 1438 guest 1439 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1440 .unwrap() 1441 .trim() 1442 .parse::<u32>() 1443 .unwrap_or_default(), 1444 readonly as u32 1445 ); 1446 1447 // Check if the number of queues in /sys/block/vdc/mq matches the 1448 // expected num_queues. 1449 assert_eq!( 1450 guest 1451 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1452 .unwrap() 1453 .trim() 1454 .parse::<u32>() 1455 .unwrap_or_default(), 1456 num_queues as u32 1457 ); 1458 1459 // Mount the device 1460 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1461 guest.ssh_command("mkdir mount_image").unwrap(); 1462 guest 1463 .ssh_command( 1464 format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), 1465 ) 1466 .unwrap(); 1467 1468 // Check the content of the block device. The file "foo" should 1469 // contain "bar". 1470 assert_eq!( 1471 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1472 "bar" 1473 ); 1474 1475 // ACPI feature is needed. 1476 #[cfg(target_arch = "x86_64")] 1477 { 1478 guest.enable_memory_hotplug(); 1479 1480 // Add RAM to the VM 1481 let desired_ram = 1024 << 20; 1482 resize_command(&api_socket, None, Some(desired_ram), None, None); 1483 1484 thread::sleep(std::time::Duration::new(10, 0)); 1485 1486 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1487 1488 // Check again the content of the block device after the resize 1489 // has been performed. 1490 assert_eq!( 1491 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1492 "bar" 1493 ); 1494 } 1495 1496 // Unmount the device 1497 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1498 guest.ssh_command("rm -r mount_image").unwrap(); 1499 }); 1500 1501 let _ = child.kill(); 1502 let output = child.wait_with_output().unwrap(); 1503 1504 if let Some(mut daemon_child) = daemon_child { 1505 thread::sleep(std::time::Duration::new(5, 0)); 1506 let _ = daemon_child.kill(); 1507 let _ = daemon_child.wait(); 1508 } 1509 1510 handle_child_output(r, &output); 1511 } 1512 1513 fn test_boot_from_vhost_user_blk( 1514 num_queues: usize, 1515 readonly: bool, 1516 direct: bool, 1517 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1518 ) { 1519 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1520 let guest = Guest::new(Box::new(focal)); 1521 1522 let kernel_path = direct_kernel_boot_path(); 1523 1524 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1525 1526 let (blk_boot_params, daemon_child) = { 1527 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1528 // Start the daemon 1529 let (daemon_child, vubd_socket_path) = prepare_daemon( 1530 &guest.tmp_dir, 1531 disk_path.as_str(), 1532 num_queues, 1533 readonly, 1534 direct, 1535 ); 1536 1537 ( 1538 format!( 1539 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1540 ), 1541 Some(daemon_child), 1542 ) 1543 }; 1544 1545 let mut child = GuestCommand::new(&guest) 1546 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1547 .args(["--memory", "size=512M,shared=on"]) 1548 .args(["--kernel", kernel_path.to_str().unwrap()]) 1549 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1550 .args([ 1551 "--disk", 1552 blk_boot_params.as_str(), 1553 format!( 1554 "path={}", 1555 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1556 ) 1557 .as_str(), 1558 ]) 1559 .default_net() 1560 .capture_output() 1561 .spawn() 1562 .unwrap(); 1563 1564 let r = std::panic::catch_unwind(|| { 1565 guest.wait_vm_boot(None).unwrap(); 1566 1567 // Just check the VM booted correctly. 1568 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1569 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1570 }); 1571 let _ = child.kill(); 1572 let output = child.wait_with_output().unwrap(); 1573 1574 if let Some(mut daemon_child) = daemon_child { 1575 thread::sleep(std::time::Duration::new(5, 0)); 1576 let _ = daemon_child.kill(); 1577 let _ = daemon_child.wait(); 1578 } 1579 1580 handle_child_output(r, &output); 1581 } 1582 1583 fn _test_virtio_fs( 1584 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1585 hotplug: bool, 1586 pci_segment: Option<u16>, 1587 ) { 1588 #[cfg(target_arch = "aarch64")] 1589 let focal_image = if hotplug { 1590 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1591 } else { 1592 FOCAL_IMAGE_NAME.to_string() 1593 }; 1594 #[cfg(target_arch = "x86_64")] 1595 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1596 let focal = UbuntuDiskConfig::new(focal_image); 1597 let guest = Guest::new(Box::new(focal)); 1598 let api_socket = temp_api_path(&guest.tmp_dir); 1599 1600 let mut workload_path = dirs::home_dir().unwrap(); 1601 workload_path.push("workloads"); 1602 1603 let mut shared_dir = workload_path; 1604 shared_dir.push("shared_dir"); 1605 1606 #[cfg(target_arch = "x86_64")] 1607 let kernel_path = direct_kernel_boot_path(); 1608 #[cfg(target_arch = "aarch64")] 1609 let kernel_path = if hotplug { 1610 edk2_path() 1611 } else { 1612 direct_kernel_boot_path() 1613 }; 1614 1615 let (mut daemon_child, virtiofsd_socket_path) = 1616 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1617 1618 let mut guest_command = GuestCommand::new(&guest); 1619 guest_command 1620 .args(["--cpus", "boot=1"]) 1621 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1622 .args(["--kernel", kernel_path.to_str().unwrap()]) 1623 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1624 .default_disks() 1625 .default_net() 1626 .args(["--api-socket", &api_socket]); 1627 if pci_segment.is_some() { 1628 guest_command.args([ 1629 "--platform", 1630 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 1631 ]); 1632 } 1633 1634 let fs_params = format!( 1635 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1636 virtiofsd_socket_path, 1637 if let Some(pci_segment) = pci_segment { 1638 format!(",pci_segment={pci_segment}") 1639 } else { 1640 "".to_owned() 1641 } 1642 ); 1643 1644 if !hotplug { 1645 guest_command.args(["--fs", fs_params.as_str()]); 1646 } 1647 1648 let mut child = guest_command.capture_output().spawn().unwrap(); 1649 1650 let r = std::panic::catch_unwind(|| { 1651 guest.wait_vm_boot(None).unwrap(); 1652 1653 if hotplug { 1654 // Add fs to the VM 1655 let (cmd_success, cmd_output) = 1656 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1657 assert!(cmd_success); 1658 1659 if let Some(pci_segment) = pci_segment { 1660 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1661 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1662 ))); 1663 } else { 1664 assert!(String::from_utf8_lossy(&cmd_output) 1665 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1666 } 1667 1668 thread::sleep(std::time::Duration::new(10, 0)); 1669 } 1670 1671 // Mount shared directory through virtio_fs filesystem 1672 guest 1673 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1674 .unwrap(); 1675 1676 // Check file1 exists and its content is "foo" 1677 assert_eq!( 1678 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1679 "foo" 1680 ); 1681 // Check file2 does not exist 1682 guest 1683 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1684 .unwrap(); 1685 1686 // Check file3 exists and its content is "bar" 1687 assert_eq!( 1688 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1689 "bar" 1690 ); 1691 1692 // ACPI feature is needed. 1693 #[cfg(target_arch = "x86_64")] 1694 { 1695 guest.enable_memory_hotplug(); 1696 1697 // Add RAM to the VM 1698 let desired_ram = 1024 << 20; 1699 resize_command(&api_socket, None, Some(desired_ram), None, None); 1700 1701 thread::sleep(std::time::Duration::new(30, 0)); 1702 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1703 1704 // After the resize, check again that file1 exists and its 1705 // content is "foo". 1706 assert_eq!( 1707 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1708 "foo" 1709 ); 1710 } 1711 1712 if hotplug { 1713 // Remove from VM 1714 guest.ssh_command("sudo umount mount_dir").unwrap(); 1715 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1716 } 1717 }); 1718 1719 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1720 thread::sleep(std::time::Duration::new(10, 0)); 1721 let (daemon_child, virtiofsd_socket_path) = 1722 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1723 1724 let r = std::panic::catch_unwind(|| { 1725 thread::sleep(std::time::Duration::new(10, 0)); 1726 let fs_params = format!( 1727 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1728 virtiofsd_socket_path, 1729 if let Some(pci_segment) = pci_segment { 1730 format!(",pci_segment={pci_segment}") 1731 } else { 1732 "".to_owned() 1733 } 1734 ); 1735 1736 // Add back and check it works 1737 let (cmd_success, cmd_output) = 1738 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1739 assert!(cmd_success); 1740 if let Some(pci_segment) = pci_segment { 1741 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1742 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1743 ))); 1744 } else { 1745 assert!(String::from_utf8_lossy(&cmd_output) 1746 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1747 } 1748 1749 thread::sleep(std::time::Duration::new(10, 0)); 1750 // Mount shared directory through virtio_fs filesystem 1751 guest 1752 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1753 .unwrap(); 1754 1755 // Check file1 exists and its content is "foo" 1756 assert_eq!( 1757 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1758 "foo" 1759 ); 1760 }); 1761 1762 (r, Some(daemon_child)) 1763 } else { 1764 (r, None) 1765 }; 1766 1767 let _ = child.kill(); 1768 let output = child.wait_with_output().unwrap(); 1769 1770 let _ = daemon_child.kill(); 1771 let _ = daemon_child.wait(); 1772 1773 if let Some(mut daemon_child) = hotplug_daemon_child { 1774 let _ = daemon_child.kill(); 1775 let _ = daemon_child.wait(); 1776 } 1777 1778 handle_child_output(r, &output); 1779 } 1780 1781 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1782 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1783 let guest = Guest::new(Box::new(focal)); 1784 1785 let kernel_path = direct_kernel_boot_path(); 1786 1787 let pmem_temp_file = TempFile::new().unwrap(); 1788 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1789 1790 std::process::Command::new("mkfs.ext4") 1791 .arg(pmem_temp_file.as_path()) 1792 .output() 1793 .expect("Expect creating disk image to succeed"); 1794 1795 let mut child = GuestCommand::new(&guest) 1796 .args(["--cpus", "boot=1"]) 1797 .args(["--memory", "size=512M"]) 1798 .args(["--kernel", kernel_path.to_str().unwrap()]) 1799 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1800 .default_disks() 1801 .default_net() 1802 .args([ 1803 "--pmem", 1804 format!( 1805 "file={}{}{}", 1806 pmem_temp_file.as_path().to_str().unwrap(), 1807 if specify_size { ",size=128M" } else { "" }, 1808 if discard_writes { 1809 ",discard_writes=on" 1810 } else { 1811 "" 1812 } 1813 ) 1814 .as_str(), 1815 ]) 1816 .capture_output() 1817 .spawn() 1818 .unwrap(); 1819 1820 let r = std::panic::catch_unwind(|| { 1821 guest.wait_vm_boot(None).unwrap(); 1822 1823 // Check for the presence of /dev/pmem0 1824 assert_eq!( 1825 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1826 "/dev/pmem0" 1827 ); 1828 1829 // Check changes persist after reboot 1830 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1831 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1832 guest 1833 .ssh_command("echo test123 | sudo tee /mnt/test") 1834 .unwrap(); 1835 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1836 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1837 1838 guest.reboot_linux(0, None); 1839 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1840 assert_eq!( 1841 guest 1842 .ssh_command("sudo cat /mnt/test || true") 1843 .unwrap() 1844 .trim(), 1845 if discard_writes { "" } else { "test123" } 1846 ); 1847 }); 1848 1849 let _ = child.kill(); 1850 let output = child.wait_with_output().unwrap(); 1851 1852 handle_child_output(r, &output); 1853 } 1854 1855 fn get_fd_count(pid: u32) -> usize { 1856 fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() 1857 } 1858 1859 fn _test_virtio_vsock(hotplug: bool) { 1860 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1861 let guest = Guest::new(Box::new(focal)); 1862 1863 #[cfg(target_arch = "x86_64")] 1864 let kernel_path = direct_kernel_boot_path(); 1865 #[cfg(target_arch = "aarch64")] 1866 let kernel_path = if hotplug { 1867 edk2_path() 1868 } else { 1869 direct_kernel_boot_path() 1870 }; 1871 1872 let socket = temp_vsock_path(&guest.tmp_dir); 1873 let api_socket = temp_api_path(&guest.tmp_dir); 1874 1875 let mut cmd = GuestCommand::new(&guest); 1876 cmd.args(["--api-socket", &api_socket]); 1877 cmd.args(["--cpus", "boot=1"]); 1878 cmd.args(["--memory", "size=512M"]); 1879 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1880 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1881 cmd.default_disks(); 1882 cmd.default_net(); 1883 1884 if !hotplug { 1885 cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); 1886 } 1887 1888 let mut child = cmd.capture_output().spawn().unwrap(); 1889 1890 let r = std::panic::catch_unwind(|| { 1891 guest.wait_vm_boot(None).unwrap(); 1892 1893 if hotplug { 1894 let (cmd_success, cmd_output) = remote_command_w_output( 1895 &api_socket, 1896 "add-vsock", 1897 Some(format!("cid=3,socket={socket},id=test0").as_str()), 1898 ); 1899 assert!(cmd_success); 1900 assert!(String::from_utf8_lossy(&cmd_output) 1901 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1902 thread::sleep(std::time::Duration::new(10, 0)); 1903 // Check adding a second one fails 1904 assert!(!remote_command( 1905 &api_socket, 1906 "add-vsock", 1907 Some("cid=1234,socket=/tmp/fail") 1908 )); 1909 } 1910 1911 // Validate vsock works as expected. 1912 guest.check_vsock(socket.as_str()); 1913 guest.reboot_linux(0, None); 1914 // Validate vsock still works after a reboot. 1915 guest.check_vsock(socket.as_str()); 1916 1917 if hotplug { 1918 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1919 } 1920 }); 1921 1922 let _ = child.kill(); 1923 let output = child.wait_with_output().unwrap(); 1924 1925 handle_child_output(r, &output); 1926 } 1927 1928 fn get_ksm_pages_shared() -> u32 { 1929 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1930 .unwrap() 1931 .trim() 1932 .parse::<u32>() 1933 .unwrap() 1934 } 1935 1936 fn test_memory_mergeable(mergeable: bool) { 1937 let memory_param = if mergeable { 1938 "mergeable=on" 1939 } else { 1940 "mergeable=off" 1941 }; 1942 1943 // We are assuming the rest of the system in our CI is not using mergeable memory 1944 let ksm_ps_init = get_ksm_pages_shared(); 1945 assert!(ksm_ps_init == 0); 1946 1947 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1948 let guest1 = Guest::new(Box::new(focal1)); 1949 let mut child1 = GuestCommand::new(&guest1) 1950 .args(["--cpus", "boot=1"]) 1951 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1952 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1953 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1954 .default_disks() 1955 .args(["--net", guest1.default_net_string().as_str()]) 1956 .args(["--serial", "tty", "--console", "off"]) 1957 .capture_output() 1958 .spawn() 1959 .unwrap(); 1960 1961 let r = std::panic::catch_unwind(|| { 1962 guest1.wait_vm_boot(None).unwrap(); 1963 }); 1964 if r.is_err() { 1965 let _ = child1.kill(); 1966 let output = child1.wait_with_output().unwrap(); 1967 handle_child_output(r, &output); 1968 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1969 } 1970 1971 let ksm_ps_guest1 = get_ksm_pages_shared(); 1972 1973 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1974 let guest2 = Guest::new(Box::new(focal2)); 1975 let mut child2 = GuestCommand::new(&guest2) 1976 .args(["--cpus", "boot=1"]) 1977 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1978 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1979 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1980 .default_disks() 1981 .args(["--net", guest2.default_net_string().as_str()]) 1982 .args(["--serial", "tty", "--console", "off"]) 1983 .capture_output() 1984 .spawn() 1985 .unwrap(); 1986 1987 let r = std::panic::catch_unwind(|| { 1988 guest2.wait_vm_boot(None).unwrap(); 1989 let ksm_ps_guest2 = get_ksm_pages_shared(); 1990 1991 if mergeable { 1992 println!( 1993 "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" 1994 ); 1995 // We are expecting the number of shared pages to increase as the number of VM increases 1996 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1997 } else { 1998 assert!(ksm_ps_guest1 == 0); 1999 assert!(ksm_ps_guest2 == 0); 2000 } 2001 }); 2002 2003 let _ = child1.kill(); 2004 let _ = child2.kill(); 2005 2006 let output = child1.wait_with_output().unwrap(); 2007 child2.wait().unwrap(); 2008 2009 handle_child_output(r, &output); 2010 } 2011 2012 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 2013 let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); 2014 let reader = io::BufReader::new(smaps); 2015 2016 let mut skip_map: bool = false; 2017 let mut region_name: String = "".to_string(); 2018 let mut region_maps = HashMap::new(); 2019 for line in reader.lines() { 2020 let l = line.unwrap(); 2021 2022 if l.contains('-') { 2023 let values: Vec<&str> = l.split_whitespace().collect(); 2024 region_name = values.last().unwrap().trim().to_string(); 2025 if region_name == "0" { 2026 region_name = "anonymous".to_string() 2027 } 2028 } 2029 2030 // Each section begins with something that looks like: 2031 // Size: 2184 kB 2032 if l.starts_with("Size:") { 2033 let values: Vec<&str> = l.split_whitespace().collect(); 2034 let map_size = values[1].parse::<u32>().unwrap(); 2035 // We skip the assigned guest RAM map, its RSS is only 2036 // dependent on the guest actual memory usage. 2037 // Everything else can be added to the VMM overhead. 2038 skip_map = map_size >= guest_memory_size; 2039 continue; 2040 } 2041 2042 // If this is a map we're taking into account, then we only 2043 // count the RSS. The sum of all counted RSS is the VMM overhead. 2044 if !skip_map && l.starts_with("Rss:") { 2045 let values: Vec<&str> = l.split_whitespace().collect(); 2046 let value = values[1].trim().parse::<u32>().unwrap(); 2047 *region_maps.entry(region_name.clone()).or_insert(0) += value; 2048 } 2049 } 2050 2051 region_maps 2052 } 2053 2054 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 2055 let mut total = 0; 2056 2057 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 2058 eprintln!("{region_name}: {value}"); 2059 total += value; 2060 } 2061 2062 total 2063 } 2064 2065 fn process_rss_kib(pid: u32) -> usize { 2066 let command = format!("ps -q {pid} -o rss="); 2067 let rss = exec_host_command_output(&command); 2068 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 2069 } 2070 2071 // 10MB is our maximum accepted overhead. 2072 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 2073 2074 #[derive(PartialEq, Eq, PartialOrd)] 2075 struct Counters { 2076 rx_bytes: u64, 2077 rx_frames: u64, 2078 tx_bytes: u64, 2079 tx_frames: u64, 2080 read_bytes: u64, 2081 write_bytes: u64, 2082 read_ops: u64, 2083 write_ops: u64, 2084 } 2085 2086 fn get_counters(api_socket: &str) -> Counters { 2087 // Get counters 2088 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 2089 assert!(cmd_success); 2090 2091 let counters: HashMap<&str, HashMap<&str, u64>> = 2092 serde_json::from_slice(&cmd_output).unwrap_or_default(); 2093 2094 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 2095 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 2096 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 2097 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 2098 2099 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 2100 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 2101 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 2102 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 2103 2104 Counters { 2105 rx_bytes, 2106 rx_frames, 2107 tx_bytes, 2108 tx_frames, 2109 read_bytes, 2110 write_bytes, 2111 read_ops, 2112 write_ops, 2113 } 2114 } 2115 2116 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 2117 let (tx, rx) = mpsc::channel::<String>(); 2118 thread::spawn(move || loop { 2119 thread::sleep(std::time::Duration::new(1, 0)); 2120 let mut buf = [0; 512]; 2121 match pty.read(&mut buf) { 2122 Ok(_bytes) => { 2123 let output = std::str::from_utf8(&buf).unwrap().to_string(); 2124 match tx.send(output) { 2125 Ok(_) => (), 2126 Err(_) => break, 2127 } 2128 } 2129 Err(_) => break, 2130 } 2131 }); 2132 rx 2133 } 2134 2135 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 2136 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2137 assert!(cmd_success); 2138 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2139 assert_eq!("Pty", info["config"][pty_type]["mode"]); 2140 PathBuf::from( 2141 info["config"][pty_type]["file"] 2142 .as_str() 2143 .expect("Missing pty path"), 2144 ) 2145 } 2146 2147 // VFIO test network setup. 2148 // We reserve a different IP class for it: 172.18.0.0/24. 2149 #[cfg(target_arch = "x86_64")] 2150 fn setup_vfio_network_interfaces() { 2151 // 'vfio-br0' 2152 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 2153 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 2154 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 2155 // 'vfio-tap0' 2156 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 2157 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 2158 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 2159 // 'vfio-tap1' 2160 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 2161 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 2162 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 2163 // 'vfio-tap2' 2164 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 2165 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 2166 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 2167 // 'vfio-tap3' 2168 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 2169 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 2170 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 2171 } 2172 2173 // Tear VFIO test network down 2174 #[cfg(target_arch = "x86_64")] 2175 fn cleanup_vfio_network_interfaces() { 2176 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 2177 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 2178 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 2179 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 2180 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 2181 } 2182 2183 fn balloon_size(api_socket: &str) -> u64 { 2184 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2185 assert!(cmd_success); 2186 2187 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2188 let total_mem = &info["config"]["memory"]["size"] 2189 .to_string() 2190 .parse::<u64>() 2191 .unwrap(); 2192 let actual_mem = &info["memory_actual_size"] 2193 .to_string() 2194 .parse::<u64>() 2195 .unwrap(); 2196 total_mem - actual_mem 2197 } 2198 2199 fn vm_state(api_socket: &str) -> String { 2200 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2201 assert!(cmd_success); 2202 2203 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2204 let state = &info["state"].as_str().unwrap(); 2205 2206 state.to_string() 2207 } 2208 2209 // This test validates that it can find the virtio-iommu device at first. 2210 // It also verifies that both disks and the network card are attached to 2211 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 2212 // The last interesting part of this test is that it exercises the network 2213 // interface attached to the virtual IOMMU since this is the one used to 2214 // send all commands through SSH. 2215 fn _test_virtio_iommu(acpi: bool) { 2216 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 2217 // Focal image is still old. 2218 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 2219 // the kernel binary has been updated. 2220 #[cfg(target_arch = "aarch64")] 2221 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 2222 #[cfg(target_arch = "x86_64")] 2223 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2224 let focal = UbuntuDiskConfig::new(focal_image); 2225 let guest = Guest::new(Box::new(focal)); 2226 2227 #[cfg(target_arch = "x86_64")] 2228 let kernel_path = direct_kernel_boot_path(); 2229 #[cfg(target_arch = "aarch64")] 2230 let kernel_path = if acpi { 2231 edk2_path() 2232 } else { 2233 direct_kernel_boot_path() 2234 }; 2235 2236 let mut child = GuestCommand::new(&guest) 2237 .args(["--cpus", "boot=1"]) 2238 .args(["--memory", "size=512M"]) 2239 .args(["--kernel", kernel_path.to_str().unwrap()]) 2240 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2241 .args([ 2242 "--disk", 2243 format!( 2244 "path={},iommu=on", 2245 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2246 ) 2247 .as_str(), 2248 format!( 2249 "path={},iommu=on", 2250 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2251 ) 2252 .as_str(), 2253 ]) 2254 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 2255 .capture_output() 2256 .spawn() 2257 .unwrap(); 2258 2259 let r = std::panic::catch_unwind(|| { 2260 guest.wait_vm_boot(None).unwrap(); 2261 2262 // Verify the virtio-iommu device is present. 2263 assert!(guest 2264 .does_device_vendor_pair_match("0x1057", "0x1af4") 2265 .unwrap_or_default()); 2266 2267 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 2268 // different with ACPI. 2269 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 2270 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 2271 // and appear under folder '/sys/kernel/iommu_groups/'. 2272 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 2273 // which is the console. The first disk "0000:00:02.0" is in group '1'. 2274 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 2275 // contains "0000:00:02.0" which is the first disk. 2276 // 2277 // Verify the iommu group of the first disk. 2278 let iommu_group = !acpi as i32; 2279 assert_eq!( 2280 guest 2281 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2282 .unwrap() 2283 .trim(), 2284 "0000:00:02.0" 2285 ); 2286 2287 // Verify the iommu group of the second disk. 2288 let iommu_group = if acpi { 1 } else { 2 }; 2289 assert_eq!( 2290 guest 2291 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2292 .unwrap() 2293 .trim(), 2294 "0000:00:03.0" 2295 ); 2296 2297 // Verify the iommu group of the network card. 2298 let iommu_group = if acpi { 2 } else { 3 }; 2299 assert_eq!( 2300 guest 2301 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2302 .unwrap() 2303 .trim(), 2304 "0000:00:04.0" 2305 ); 2306 }); 2307 2308 let _ = child.kill(); 2309 let output = child.wait_with_output().unwrap(); 2310 2311 handle_child_output(r, &output); 2312 } 2313 2314 fn get_reboot_count(guest: &Guest) -> u32 { 2315 guest 2316 .ssh_command("sudo last | grep -c reboot") 2317 .unwrap() 2318 .trim() 2319 .parse::<u32>() 2320 .unwrap_or_default() 2321 } 2322 2323 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 2324 // Check for PCI device 2325 assert!(guest 2326 .does_device_vendor_pair_match("0x1063", "0x1af4") 2327 .unwrap_or_default()); 2328 2329 // Enable systemd watchdog 2330 guest 2331 .ssh_command(&format!( 2332 "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" 2333 )) 2334 .unwrap(); 2335 } 2336 2337 fn make_guest_panic(guest: &Guest) { 2338 // Check for pvpanic device 2339 assert!(guest 2340 .does_device_vendor_pair_match("0x0011", "0x1b36") 2341 .unwrap_or_default()); 2342 2343 // Trigger guest a panic 2344 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 2345 } 2346 2347 mod common_parallel { 2348 use std::{fs::OpenOptions, io::SeekFrom}; 2349 2350 use crate::*; 2351 2352 #[test] 2353 #[cfg(target_arch = "x86_64")] 2354 fn test_focal_hypervisor_fw() { 2355 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 2356 } 2357 2358 #[test] 2359 #[cfg(target_arch = "x86_64")] 2360 fn test_focal_ovmf() { 2361 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 2362 } 2363 2364 #[cfg(target_arch = "x86_64")] 2365 fn test_simple_launch(fw_path: String, disk_path: &str) { 2366 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 2367 let guest = Guest::new(disk_config); 2368 let event_path = temp_event_monitor_path(&guest.tmp_dir); 2369 2370 let mut child = GuestCommand::new(&guest) 2371 .args(["--cpus", "boot=1"]) 2372 .args(["--memory", "size=512M"]) 2373 .args(["--kernel", fw_path.as_str()]) 2374 .default_disks() 2375 .default_net() 2376 .args(["--serial", "tty", "--console", "off"]) 2377 .args(["--event-monitor", format!("path={event_path}").as_str()]) 2378 .capture_output() 2379 .spawn() 2380 .unwrap(); 2381 2382 let r = std::panic::catch_unwind(|| { 2383 guest.wait_vm_boot(Some(120)).unwrap(); 2384 2385 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2386 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2387 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2388 2389 let expected_sequential_events = [ 2390 &MetaEvent { 2391 event: "starting".to_string(), 2392 device_id: None, 2393 }, 2394 &MetaEvent { 2395 event: "booting".to_string(), 2396 device_id: None, 2397 }, 2398 &MetaEvent { 2399 event: "booted".to_string(), 2400 device_id: None, 2401 }, 2402 &MetaEvent { 2403 event: "activated".to_string(), 2404 device_id: Some("_disk0".to_string()), 2405 }, 2406 &MetaEvent { 2407 event: "reset".to_string(), 2408 device_id: Some("_disk0".to_string()), 2409 }, 2410 ]; 2411 assert!(check_sequential_events( 2412 &expected_sequential_events, 2413 &event_path 2414 )); 2415 2416 // It's been observed on the Bionic image that udev and snapd 2417 // services can cause some delay in the VM's shutdown. Disabling 2418 // them improves the reliability of this test. 2419 let _ = guest.ssh_command("sudo systemctl disable udev"); 2420 let _ = guest.ssh_command("sudo systemctl stop udev"); 2421 let _ = guest.ssh_command("sudo systemctl disable snapd"); 2422 let _ = guest.ssh_command("sudo systemctl stop snapd"); 2423 2424 guest.ssh_command("sudo poweroff").unwrap(); 2425 thread::sleep(std::time::Duration::new(20, 0)); 2426 let latest_events = [ 2427 &MetaEvent { 2428 event: "shutdown".to_string(), 2429 device_id: None, 2430 }, 2431 &MetaEvent { 2432 event: "deleted".to_string(), 2433 device_id: None, 2434 }, 2435 &MetaEvent { 2436 event: "shutdown".to_string(), 2437 device_id: None, 2438 }, 2439 ]; 2440 assert!(check_latest_events_exact(&latest_events, &event_path)); 2441 }); 2442 2443 let _ = child.kill(); 2444 let output = child.wait_with_output().unwrap(); 2445 2446 handle_child_output(r, &output); 2447 } 2448 2449 #[test] 2450 fn test_multi_cpu() { 2451 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2452 let jammy = UbuntuDiskConfig::new(jammy_image); 2453 let guest = Guest::new(Box::new(jammy)); 2454 2455 let mut cmd = GuestCommand::new(&guest); 2456 cmd.args(["--cpus", "boot=2,max=4"]) 2457 .args(["--memory", "size=512M"]) 2458 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2459 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2460 .capture_output() 2461 .default_disks() 2462 .default_net(); 2463 2464 let mut child = cmd.spawn().unwrap(); 2465 2466 let r = std::panic::catch_unwind(|| { 2467 guest.wait_vm_boot(Some(120)).unwrap(); 2468 2469 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2470 2471 assert_eq!( 2472 guest 2473 .ssh_command( 2474 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2475 ) 2476 .unwrap() 2477 .trim(), 2478 "smp: Brought up 1 node, 2 CPUs" 2479 ); 2480 }); 2481 2482 let _ = child.kill(); 2483 let output = child.wait_with_output().unwrap(); 2484 2485 handle_child_output(r, &output); 2486 } 2487 2488 #[test] 2489 fn test_cpu_topology_421() { 2490 test_cpu_topology(4, 2, 1, false); 2491 } 2492 2493 #[test] 2494 fn test_cpu_topology_142() { 2495 test_cpu_topology(1, 4, 2, false); 2496 } 2497 2498 #[test] 2499 fn test_cpu_topology_262() { 2500 test_cpu_topology(2, 6, 2, false); 2501 } 2502 2503 #[test] 2504 #[cfg(target_arch = "x86_64")] 2505 #[cfg(not(feature = "mshv"))] 2506 fn test_cpu_physical_bits() { 2507 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2508 let guest = Guest::new(Box::new(focal)); 2509 let max_phys_bits: u8 = 36; 2510 let mut child = GuestCommand::new(&guest) 2511 .args(["--cpus", &format!("max_phys_bits={max_phys_bits}")]) 2512 .args(["--memory", "size=512M"]) 2513 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2514 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2515 .default_disks() 2516 .default_net() 2517 .capture_output() 2518 .spawn() 2519 .unwrap(); 2520 2521 let r = std::panic::catch_unwind(|| { 2522 guest.wait_vm_boot(None).unwrap(); 2523 2524 assert!( 2525 guest 2526 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2527 .unwrap() 2528 .trim() 2529 .parse::<u8>() 2530 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2531 ); 2532 }); 2533 2534 let _ = child.kill(); 2535 let output = child.wait_with_output().unwrap(); 2536 2537 handle_child_output(r, &output); 2538 } 2539 2540 #[test] 2541 fn test_cpu_affinity() { 2542 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2543 let guest = Guest::new(Box::new(focal)); 2544 2545 // We need the host to have at least 4 CPUs if we want to be able 2546 // to run this test. 2547 let host_cpus_count = exec_host_command_output("nproc"); 2548 assert!( 2549 String::from_utf8_lossy(&host_cpus_count.stdout) 2550 .trim() 2551 .parse::<u16>() 2552 .unwrap_or(0) 2553 >= 4 2554 ); 2555 2556 let mut child = GuestCommand::new(&guest) 2557 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2558 .args(["--memory", "size=512M"]) 2559 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2560 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2561 .default_disks() 2562 .default_net() 2563 .capture_output() 2564 .spawn() 2565 .unwrap(); 2566 2567 let r = std::panic::catch_unwind(|| { 2568 guest.wait_vm_boot(None).unwrap(); 2569 let pid = child.id(); 2570 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2571 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2572 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2573 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2574 }); 2575 2576 let _ = child.kill(); 2577 let output = child.wait_with_output().unwrap(); 2578 handle_child_output(r, &output); 2579 } 2580 2581 #[test] 2582 fn test_virtio_queue_affinity() { 2583 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2584 let guest = Guest::new(Box::new(focal)); 2585 2586 // We need the host to have at least 4 CPUs if we want to be able 2587 // to run this test. 2588 let host_cpus_count = exec_host_command_output("nproc"); 2589 assert!( 2590 String::from_utf8_lossy(&host_cpus_count.stdout) 2591 .trim() 2592 .parse::<u16>() 2593 .unwrap_or(0) 2594 >= 4 2595 ); 2596 2597 let mut child = GuestCommand::new(&guest) 2598 .args(["--cpus", "boot=4"]) 2599 .args(["--memory", "size=512M"]) 2600 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2601 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2602 .args([ 2603 "--disk", 2604 format!( 2605 "path={}", 2606 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2607 ) 2608 .as_str(), 2609 format!( 2610 "path={},num_queues=4,queue_affinity=[0@[0,2],1@[1,3],2@[1],3@[3]]", 2611 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2612 ) 2613 .as_str(), 2614 ]) 2615 .default_net() 2616 .capture_output() 2617 .spawn() 2618 .unwrap(); 2619 2620 let r = std::panic::catch_unwind(|| { 2621 guest.wait_vm_boot(None).unwrap(); 2622 let pid = child.id(); 2623 let taskset_q0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2624 assert_eq!(String::from_utf8_lossy(&taskset_q0.stdout).trim(), "0,2"); 2625 let taskset_q1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2626 assert_eq!(String::from_utf8_lossy(&taskset_q1.stdout).trim(), "1,3"); 2627 let taskset_q2 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q2 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2628 assert_eq!(String::from_utf8_lossy(&taskset_q2.stdout).trim(), "1"); 2629 let taskset_q3 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q3 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2630 assert_eq!(String::from_utf8_lossy(&taskset_q3.stdout).trim(), "3"); 2631 }); 2632 2633 let _ = child.kill(); 2634 let output = child.wait_with_output().unwrap(); 2635 handle_child_output(r, &output); 2636 } 2637 2638 #[test] 2639 #[cfg(not(feature = "mshv"))] 2640 fn test_large_vm() { 2641 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2642 let guest = Guest::new(Box::new(focal)); 2643 let mut cmd = GuestCommand::new(&guest); 2644 cmd.args(["--cpus", "boot=48"]) 2645 .args(["--memory", "size=5120M"]) 2646 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2647 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2648 .args(["--serial", "tty"]) 2649 .args(["--console", "off"]) 2650 .capture_output() 2651 .default_disks() 2652 .default_net(); 2653 2654 let mut child = cmd.spawn().unwrap(); 2655 2656 guest.wait_vm_boot(None).unwrap(); 2657 2658 let r = std::panic::catch_unwind(|| { 2659 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2660 assert_eq!( 2661 guest 2662 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2663 .unwrap() 2664 .trim(), 2665 "0-47" 2666 ); 2667 2668 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2669 }); 2670 2671 let _ = child.kill(); 2672 let output = child.wait_with_output().unwrap(); 2673 2674 handle_child_output(r, &output); 2675 } 2676 2677 #[test] 2678 #[cfg(not(feature = "mshv"))] 2679 fn test_huge_memory() { 2680 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2681 let guest = Guest::new(Box::new(focal)); 2682 let mut cmd = GuestCommand::new(&guest); 2683 cmd.args(["--cpus", "boot=1"]) 2684 .args(["--memory", "size=128G"]) 2685 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2686 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2687 .capture_output() 2688 .default_disks() 2689 .default_net(); 2690 2691 let mut child = cmd.spawn().unwrap(); 2692 2693 guest.wait_vm_boot(Some(120)).unwrap(); 2694 2695 let r = std::panic::catch_unwind(|| { 2696 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2697 }); 2698 2699 let _ = child.kill(); 2700 let output = child.wait_with_output().unwrap(); 2701 2702 handle_child_output(r, &output); 2703 } 2704 2705 #[test] 2706 fn test_power_button() { 2707 _test_power_button(false); 2708 } 2709 2710 #[test] 2711 #[cfg(not(feature = "mshv"))] 2712 fn test_user_defined_memory_regions() { 2713 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2714 let guest = Guest::new(Box::new(focal)); 2715 let api_socket = temp_api_path(&guest.tmp_dir); 2716 2717 let kernel_path = direct_kernel_boot_path(); 2718 2719 let mut child = GuestCommand::new(&guest) 2720 .args(["--cpus", "boot=1"]) 2721 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2722 .args([ 2723 "--memory-zone", 2724 "id=mem0,size=1G,hotplug_size=2G", 2725 "id=mem1,size=1G,shared=on", 2726 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2727 ]) 2728 .args(["--kernel", kernel_path.to_str().unwrap()]) 2729 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2730 .args(["--api-socket", &api_socket]) 2731 .capture_output() 2732 .default_disks() 2733 .default_net() 2734 .spawn() 2735 .unwrap(); 2736 2737 let r = std::panic::catch_unwind(|| { 2738 guest.wait_vm_boot(None).unwrap(); 2739 2740 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2741 2742 guest.enable_memory_hotplug(); 2743 2744 resize_zone_command(&api_socket, "mem0", "3G"); 2745 thread::sleep(std::time::Duration::new(5, 0)); 2746 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2747 resize_zone_command(&api_socket, "mem2", "3G"); 2748 thread::sleep(std::time::Duration::new(5, 0)); 2749 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2750 resize_zone_command(&api_socket, "mem0", "2G"); 2751 thread::sleep(std::time::Duration::new(5, 0)); 2752 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2753 resize_zone_command(&api_socket, "mem2", "2G"); 2754 thread::sleep(std::time::Duration::new(5, 0)); 2755 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2756 2757 guest.reboot_linux(0, None); 2758 2759 // Check the amount of RAM after reboot 2760 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2761 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2762 2763 // Check if we can still resize down to the initial 'boot'size 2764 resize_zone_command(&api_socket, "mem0", "1G"); 2765 thread::sleep(std::time::Duration::new(5, 0)); 2766 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2767 resize_zone_command(&api_socket, "mem2", "1G"); 2768 thread::sleep(std::time::Duration::new(5, 0)); 2769 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2770 }); 2771 2772 let _ = child.kill(); 2773 let output = child.wait_with_output().unwrap(); 2774 2775 handle_child_output(r, &output); 2776 } 2777 2778 #[test] 2779 #[cfg(not(feature = "mshv"))] 2780 fn test_guest_numa_nodes() { 2781 _test_guest_numa_nodes(false); 2782 } 2783 2784 #[test] 2785 #[cfg(target_arch = "x86_64")] 2786 fn test_iommu_segments() { 2787 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2788 let focal = UbuntuDiskConfig::new(focal_image); 2789 let guest = Guest::new(Box::new(focal)); 2790 2791 // Prepare another disk file for the virtio-disk device 2792 let test_disk_path = String::from( 2793 guest 2794 .tmp_dir 2795 .as_path() 2796 .join("test-disk.raw") 2797 .to_str() 2798 .unwrap(), 2799 ); 2800 assert!( 2801 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2802 ); 2803 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2804 2805 let api_socket = temp_api_path(&guest.tmp_dir); 2806 let mut cmd = GuestCommand::new(&guest); 2807 2808 cmd.args(["--cpus", "boot=1"]) 2809 .args(["--api-socket", &api_socket]) 2810 .args(["--memory", "size=512M"]) 2811 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2812 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2813 .args([ 2814 "--platform", 2815 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS},iommu_segments=[1]"), 2816 ]) 2817 .default_disks() 2818 .capture_output() 2819 .default_net(); 2820 2821 let mut child = cmd.spawn().unwrap(); 2822 2823 guest.wait_vm_boot(None).unwrap(); 2824 2825 let r = std::panic::catch_unwind(|| { 2826 let (cmd_success, cmd_output) = remote_command_w_output( 2827 &api_socket, 2828 "add-disk", 2829 Some( 2830 format!( 2831 "path={},id=test0,pci_segment=1,iommu=on", 2832 test_disk_path.as_str() 2833 ) 2834 .as_str(), 2835 ), 2836 ); 2837 assert!(cmd_success); 2838 assert!(String::from_utf8_lossy(&cmd_output) 2839 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2840 2841 // Check IOMMU setup 2842 assert!(guest 2843 .does_device_vendor_pair_match("0x1057", "0x1af4") 2844 .unwrap_or_default()); 2845 assert_eq!( 2846 guest 2847 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2848 .unwrap() 2849 .trim(), 2850 "0001:00:01.0" 2851 ); 2852 }); 2853 2854 let _ = child.kill(); 2855 let output = child.wait_with_output().unwrap(); 2856 2857 handle_child_output(r, &output); 2858 } 2859 2860 #[test] 2861 fn test_pci_msi() { 2862 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2863 let guest = Guest::new(Box::new(focal)); 2864 let mut cmd = GuestCommand::new(&guest); 2865 cmd.args(["--cpus", "boot=1"]) 2866 .args(["--memory", "size=512M"]) 2867 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2868 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2869 .capture_output() 2870 .default_disks() 2871 .default_net(); 2872 2873 let mut child = cmd.spawn().unwrap(); 2874 2875 guest.wait_vm_boot(None).unwrap(); 2876 2877 #[cfg(target_arch = "x86_64")] 2878 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2879 #[cfg(target_arch = "aarch64")] 2880 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2881 2882 let r = std::panic::catch_unwind(|| { 2883 assert_eq!( 2884 guest 2885 .ssh_command(grep_cmd) 2886 .unwrap() 2887 .trim() 2888 .parse::<u32>() 2889 .unwrap_or_default(), 2890 12 2891 ); 2892 }); 2893 2894 let _ = child.kill(); 2895 let output = child.wait_with_output().unwrap(); 2896 2897 handle_child_output(r, &output); 2898 } 2899 2900 #[test] 2901 fn test_virtio_net_ctrl_queue() { 2902 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2903 let guest = Guest::new(Box::new(focal)); 2904 let mut cmd = GuestCommand::new(&guest); 2905 cmd.args(["--cpus", "boot=1"]) 2906 .args(["--memory", "size=512M"]) 2907 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2908 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2909 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2910 .capture_output() 2911 .default_disks(); 2912 2913 let mut child = cmd.spawn().unwrap(); 2914 2915 guest.wait_vm_boot(None).unwrap(); 2916 2917 #[cfg(target_arch = "aarch64")] 2918 let iface = "enp0s4"; 2919 #[cfg(target_arch = "x86_64")] 2920 let iface = "ens4"; 2921 2922 let r = std::panic::catch_unwind(|| { 2923 assert_eq!( 2924 guest 2925 .ssh_command( 2926 format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() 2927 ) 2928 .unwrap() 2929 .trim(), 2930 "success" 2931 ); 2932 assert_eq!( 2933 guest 2934 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 2935 .unwrap() 2936 .trim(), 2937 "3000" 2938 ); 2939 }); 2940 2941 let _ = child.kill(); 2942 let output = child.wait_with_output().unwrap(); 2943 2944 handle_child_output(r, &output); 2945 } 2946 2947 #[test] 2948 fn test_pci_multiple_segments() { 2949 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2950 let guest = Guest::new(Box::new(focal)); 2951 2952 // Prepare another disk file for the virtio-disk device 2953 let test_disk_path = String::from( 2954 guest 2955 .tmp_dir 2956 .as_path() 2957 .join("test-disk.raw") 2958 .to_str() 2959 .unwrap(), 2960 ); 2961 assert!( 2962 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2963 ); 2964 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2965 2966 let mut cmd = GuestCommand::new(&guest); 2967 cmd.args(["--cpus", "boot=1"]) 2968 .args(["--memory", "size=512M"]) 2969 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2970 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2971 .args([ 2972 "--platform", 2973 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 2974 ]) 2975 .args([ 2976 "--disk", 2977 format!( 2978 "path={}", 2979 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2980 ) 2981 .as_str(), 2982 format!( 2983 "path={}", 2984 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2985 ) 2986 .as_str(), 2987 format!("path={test_disk_path},pci_segment=15").as_str(), 2988 ]) 2989 .capture_output() 2990 .default_net(); 2991 2992 let mut child = cmd.spawn().unwrap(); 2993 2994 guest.wait_vm_boot(None).unwrap(); 2995 2996 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2997 2998 let r = std::panic::catch_unwind(|| { 2999 // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. 3000 assert_eq!( 3001 guest 3002 .ssh_command(grep_cmd) 3003 .unwrap() 3004 .trim() 3005 .parse::<u16>() 3006 .unwrap_or_default(), 3007 MAX_NUM_PCI_SEGMENTS 3008 ); 3009 3010 // Check both if /dev/vdc exists and if the block size is 4M. 3011 assert_eq!( 3012 guest 3013 .ssh_command("lsblk | grep vdc | grep -c 4M") 3014 .unwrap() 3015 .trim() 3016 .parse::<u32>() 3017 .unwrap_or_default(), 3018 1 3019 ); 3020 3021 // Mount the device. 3022 guest.ssh_command("mkdir mount_image").unwrap(); 3023 guest 3024 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 3025 .unwrap(); 3026 // Grant all users with write permission. 3027 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 3028 3029 // Write something to the device. 3030 guest 3031 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 3032 .unwrap(); 3033 3034 // Check the content of the block device. The file "foo" should 3035 // contain "bar". 3036 assert_eq!( 3037 guest 3038 .ssh_command("sudo cat mount_image/foo") 3039 .unwrap() 3040 .trim(), 3041 "bar" 3042 ); 3043 }); 3044 3045 let _ = child.kill(); 3046 let output = child.wait_with_output().unwrap(); 3047 3048 handle_child_output(r, &output); 3049 } 3050 3051 #[test] 3052 fn test_pci_multiple_segments_numa_node() { 3053 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3054 let guest = Guest::new(Box::new(focal)); 3055 let api_socket = temp_api_path(&guest.tmp_dir); 3056 #[cfg(target_arch = "x86_64")] 3057 let kernel_path = direct_kernel_boot_path(); 3058 #[cfg(target_arch = "aarch64")] 3059 let kernel_path = edk2_path(); 3060 3061 // Prepare another disk file for the virtio-disk device 3062 let test_disk_path = String::from( 3063 guest 3064 .tmp_dir 3065 .as_path() 3066 .join("test-disk.raw") 3067 .to_str() 3068 .unwrap(), 3069 ); 3070 assert!( 3071 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 3072 ); 3073 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 3074 const TEST_DISK_NODE: u16 = 1; 3075 3076 let mut child = GuestCommand::new(&guest) 3077 .args(["--platform", "num_pci_segments=2"]) 3078 .args(["--cpus", "boot=2"]) 3079 .args(["--memory", "size=0"]) 3080 .args(["--memory-zone", "id=mem0,size=256M", "id=mem1,size=256M"]) 3081 .args([ 3082 "--numa", 3083 "guest_numa_id=0,cpus=[0],distances=[1@20],memory_zones=mem0,pci_segments=[0]", 3084 "guest_numa_id=1,cpus=[1],distances=[0@20],memory_zones=mem1,pci_segments=[1]", 3085 ]) 3086 .args(["--kernel", kernel_path.to_str().unwrap()]) 3087 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3088 .args(["--api-socket", &api_socket]) 3089 .capture_output() 3090 .args([ 3091 "--disk", 3092 format!( 3093 "path={}", 3094 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3095 ) 3096 .as_str(), 3097 format!( 3098 "path={}", 3099 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3100 ) 3101 .as_str(), 3102 format!("path={test_disk_path},pci_segment={TEST_DISK_NODE}").as_str(), 3103 ]) 3104 .default_net() 3105 .spawn() 3106 .unwrap(); 3107 3108 let cmd = "cat /sys/block/vdc/device/../numa_node"; 3109 3110 let r = std::panic::catch_unwind(|| { 3111 guest.wait_vm_boot(None).unwrap(); 3112 3113 assert_eq!( 3114 guest 3115 .ssh_command(cmd) 3116 .unwrap() 3117 .trim() 3118 .parse::<u16>() 3119 .unwrap_or_default(), 3120 TEST_DISK_NODE 3121 ); 3122 }); 3123 3124 let _ = child.kill(); 3125 let output = child.wait_with_output().unwrap(); 3126 3127 handle_child_output(r, &output); 3128 } 3129 3130 #[test] 3131 fn test_direct_kernel_boot() { 3132 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3133 let guest = Guest::new(Box::new(focal)); 3134 3135 let kernel_path = direct_kernel_boot_path(); 3136 3137 let mut child = GuestCommand::new(&guest) 3138 .args(["--cpus", "boot=1"]) 3139 .args(["--memory", "size=512M"]) 3140 .args(["--kernel", kernel_path.to_str().unwrap()]) 3141 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3142 .default_disks() 3143 .default_net() 3144 .capture_output() 3145 .spawn() 3146 .unwrap(); 3147 3148 let r = std::panic::catch_unwind(|| { 3149 guest.wait_vm_boot(None).unwrap(); 3150 3151 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3152 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3153 3154 let grep_cmd = if cfg!(target_arch = "x86_64") { 3155 "grep -c PCI-MSI /proc/interrupts" 3156 } else { 3157 "grep -c ITS-MSI /proc/interrupts" 3158 }; 3159 assert_eq!( 3160 guest 3161 .ssh_command(grep_cmd) 3162 .unwrap() 3163 .trim() 3164 .parse::<u32>() 3165 .unwrap_or_default(), 3166 12 3167 ); 3168 }); 3169 3170 let _ = child.kill(); 3171 let output = child.wait_with_output().unwrap(); 3172 3173 handle_child_output(r, &output); 3174 } 3175 3176 #[test] 3177 #[cfg(target_arch = "x86_64")] 3178 fn test_direct_kernel_boot_bzimage() { 3179 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3180 let guest = Guest::new(Box::new(focal)); 3181 3182 let mut kernel_path = direct_kernel_boot_path(); 3183 // Replace the default kernel with the bzImage. 3184 kernel_path.pop(); 3185 kernel_path.push("bzImage"); 3186 3187 let mut child = GuestCommand::new(&guest) 3188 .args(["--cpus", "boot=1"]) 3189 .args(["--memory", "size=512M"]) 3190 .args(["--kernel", kernel_path.to_str().unwrap()]) 3191 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3192 .default_disks() 3193 .default_net() 3194 .capture_output() 3195 .spawn() 3196 .unwrap(); 3197 3198 let r = std::panic::catch_unwind(|| { 3199 guest.wait_vm_boot(None).unwrap(); 3200 3201 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3202 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3203 3204 let grep_cmd = if cfg!(target_arch = "x86_64") { 3205 "grep -c PCI-MSI /proc/interrupts" 3206 } else { 3207 "grep -c ITS-MSI /proc/interrupts" 3208 }; 3209 assert_eq!( 3210 guest 3211 .ssh_command(grep_cmd) 3212 .unwrap() 3213 .trim() 3214 .parse::<u32>() 3215 .unwrap_or_default(), 3216 12 3217 ); 3218 }); 3219 3220 let _ = child.kill(); 3221 let output = child.wait_with_output().unwrap(); 3222 3223 handle_child_output(r, &output); 3224 } 3225 3226 fn _test_virtio_block(image_name: &str, disable_io_uring: bool, disable_aio: bool) { 3227 let focal = UbuntuDiskConfig::new(image_name.to_string()); 3228 let guest = Guest::new(Box::new(focal)); 3229 3230 let mut workload_path = dirs::home_dir().unwrap(); 3231 workload_path.push("workloads"); 3232 3233 let mut blk_file_path = workload_path; 3234 blk_file_path.push("blk.img"); 3235 3236 let kernel_path = direct_kernel_boot_path(); 3237 3238 let mut cloud_child = GuestCommand::new(&guest) 3239 .args(["--cpus", "boot=4"]) 3240 .args(["--memory", "size=512M,shared=on"]) 3241 .args(["--kernel", kernel_path.to_str().unwrap()]) 3242 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3243 .args([ 3244 "--disk", 3245 format!( 3246 "path={}", 3247 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3248 ) 3249 .as_str(), 3250 format!( 3251 "path={}", 3252 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3253 ) 3254 .as_str(), 3255 format!( 3256 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", 3257 blk_file_path.to_str().unwrap(), 3258 disable_io_uring, 3259 disable_aio, 3260 ) 3261 .as_str(), 3262 ]) 3263 .default_net() 3264 .capture_output() 3265 .spawn() 3266 .unwrap(); 3267 3268 let r = std::panic::catch_unwind(|| { 3269 guest.wait_vm_boot(None).unwrap(); 3270 3271 // Check both if /dev/vdc exists and if the block size is 16M. 3272 assert_eq!( 3273 guest 3274 .ssh_command("lsblk | grep vdc | grep -c 16M") 3275 .unwrap() 3276 .trim() 3277 .parse::<u32>() 3278 .unwrap_or_default(), 3279 1 3280 ); 3281 3282 // Check both if /dev/vdc exists and if this block is RO. 3283 assert_eq!( 3284 guest 3285 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 3286 .unwrap() 3287 .trim() 3288 .parse::<u32>() 3289 .unwrap_or_default(), 3290 1 3291 ); 3292 3293 // Check if the number of queues is 4. 3294 assert_eq!( 3295 guest 3296 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 3297 .unwrap() 3298 .trim() 3299 .parse::<u32>() 3300 .unwrap_or_default(), 3301 4 3302 ); 3303 }); 3304 3305 let _ = cloud_child.kill(); 3306 let output = cloud_child.wait_with_output().unwrap(); 3307 3308 handle_child_output(r, &output); 3309 } 3310 3311 #[test] 3312 fn test_virtio_block_io_uring() { 3313 _test_virtio_block(FOCAL_IMAGE_NAME, false, true) 3314 } 3315 3316 #[test] 3317 fn test_virtio_block_aio() { 3318 _test_virtio_block(FOCAL_IMAGE_NAME, true, false) 3319 } 3320 3321 #[test] 3322 fn test_virtio_block_sync() { 3323 _test_virtio_block(FOCAL_IMAGE_NAME, true, true) 3324 } 3325 3326 #[test] 3327 fn test_virtio_block_qcow2() { 3328 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false, false) 3329 } 3330 3331 #[test] 3332 fn test_virtio_block_qcow2_backing_file() { 3333 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE, false, false) 3334 } 3335 3336 #[test] 3337 fn test_virtio_block_vhd() { 3338 let mut workload_path = dirs::home_dir().unwrap(); 3339 workload_path.push("workloads"); 3340 3341 let mut raw_file_path = workload_path.clone(); 3342 let mut vhd_file_path = workload_path; 3343 raw_file_path.push(FOCAL_IMAGE_NAME); 3344 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 3345 3346 // Generate VHD file from RAW file 3347 std::process::Command::new("qemu-img") 3348 .arg("convert") 3349 .arg("-p") 3350 .args(["-f", "raw"]) 3351 .args(["-O", "vpc"]) 3352 .args(["-o", "subformat=fixed"]) 3353 .arg(raw_file_path.to_str().unwrap()) 3354 .arg(vhd_file_path.to_str().unwrap()) 3355 .output() 3356 .expect("Expect generating VHD image from RAW image"); 3357 3358 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false, false) 3359 } 3360 3361 #[test] 3362 fn test_virtio_block_vhdx() { 3363 let mut workload_path = dirs::home_dir().unwrap(); 3364 workload_path.push("workloads"); 3365 3366 let mut raw_file_path = workload_path.clone(); 3367 let mut vhdx_file_path = workload_path; 3368 raw_file_path.push(FOCAL_IMAGE_NAME); 3369 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 3370 3371 // Generate dynamic VHDX file from RAW file 3372 std::process::Command::new("qemu-img") 3373 .arg("convert") 3374 .arg("-p") 3375 .args(["-f", "raw"]) 3376 .args(["-O", "vhdx"]) 3377 .arg(raw_file_path.to_str().unwrap()) 3378 .arg(vhdx_file_path.to_str().unwrap()) 3379 .output() 3380 .expect("Expect generating dynamic VHDx image from RAW image"); 3381 3382 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false, false) 3383 } 3384 3385 #[test] 3386 fn test_virtio_block_dynamic_vhdx_expand() { 3387 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 3388 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 3389 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 3390 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 3391 3392 let mut workload_path = dirs::home_dir().unwrap(); 3393 workload_path.push("workloads"); 3394 3395 let mut vhdx_file_path = workload_path; 3396 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 3397 let vhdx_path = vhdx_file_path.to_str().unwrap(); 3398 3399 // Generate a 100 MiB dynamic VHDX file 3400 std::process::Command::new("qemu-img") 3401 .arg("create") 3402 .args(["-f", "vhdx"]) 3403 .arg(vhdx_path) 3404 .arg(VIRTUAL_DISK_SIZE.to_string()) 3405 .output() 3406 .expect("Expect generating dynamic VHDx image from RAW image"); 3407 3408 // Check if the size matches with empty VHDx file size 3409 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 3410 3411 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3412 let guest = Guest::new(Box::new(focal)); 3413 let kernel_path = direct_kernel_boot_path(); 3414 3415 let mut cloud_child = GuestCommand::new(&guest) 3416 .args(["--cpus", "boot=1"]) 3417 .args(["--memory", "size=512M"]) 3418 .args(["--kernel", kernel_path.to_str().unwrap()]) 3419 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3420 .args([ 3421 "--disk", 3422 format!( 3423 "path={}", 3424 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3425 ) 3426 .as_str(), 3427 format!( 3428 "path={}", 3429 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3430 ) 3431 .as_str(), 3432 format!("path={vhdx_path}").as_str(), 3433 ]) 3434 .default_net() 3435 .capture_output() 3436 .spawn() 3437 .unwrap(); 3438 3439 let r = std::panic::catch_unwind(|| { 3440 guest.wait_vm_boot(None).unwrap(); 3441 3442 // Check both if /dev/vdc exists and if the block size is 100 MiB. 3443 assert_eq!( 3444 guest 3445 .ssh_command("lsblk | grep vdc | grep -c 100M") 3446 .unwrap() 3447 .trim() 3448 .parse::<u32>() 3449 .unwrap_or_default(), 3450 1 3451 ); 3452 3453 // Write 100 MB of data to the VHDx disk 3454 guest 3455 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 3456 .unwrap(); 3457 }); 3458 3459 // Check if the size matches with expected expanded VHDx file size 3460 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 3461 3462 let _ = cloud_child.kill(); 3463 let output = cloud_child.wait_with_output().unwrap(); 3464 3465 handle_child_output(r, &output); 3466 } 3467 3468 fn vhdx_image_size(disk_name: &str) -> u64 { 3469 std::fs::File::open(disk_name) 3470 .unwrap() 3471 .seek(SeekFrom::End(0)) 3472 .unwrap() 3473 } 3474 3475 #[test] 3476 fn test_virtio_block_direct_and_firmware() { 3477 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3478 let guest = Guest::new(Box::new(focal)); 3479 3480 // The OS disk must be copied to a location that is not backed by 3481 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 3482 // with EINVAL because tmpfs doesn't support this flag. 3483 let mut workloads_path = dirs::home_dir().unwrap(); 3484 workloads_path.push("workloads"); 3485 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 3486 let mut os_path = os_dir.as_path().to_path_buf(); 3487 os_path.push("osdisk.img"); 3488 rate_limited_copy( 3489 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3490 os_path.as_path(), 3491 ) 3492 .expect("copying of OS disk failed"); 3493 3494 let mut child = GuestCommand::new(&guest) 3495 .args(["--cpus", "boot=1"]) 3496 .args(["--memory", "size=512M"]) 3497 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 3498 .args([ 3499 "--disk", 3500 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 3501 format!( 3502 "path={}", 3503 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3504 ) 3505 .as_str(), 3506 ]) 3507 .default_net() 3508 .capture_output() 3509 .spawn() 3510 .unwrap(); 3511 3512 let r = std::panic::catch_unwind(|| { 3513 guest.wait_vm_boot(Some(120)).unwrap(); 3514 }); 3515 3516 let _ = child.kill(); 3517 let output = child.wait_with_output().unwrap(); 3518 3519 handle_child_output(r, &output); 3520 } 3521 3522 #[test] 3523 fn test_vhost_user_net_default() { 3524 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 3525 } 3526 3527 #[test] 3528 fn test_vhost_user_net_named_tap() { 3529 test_vhost_user_net( 3530 Some("mytap0"), 3531 2, 3532 &prepare_vhost_user_net_daemon, 3533 false, 3534 false, 3535 ) 3536 } 3537 3538 #[test] 3539 fn test_vhost_user_net_existing_tap() { 3540 test_vhost_user_net( 3541 Some("vunet-tap0"), 3542 2, 3543 &prepare_vhost_user_net_daemon, 3544 false, 3545 false, 3546 ) 3547 } 3548 3549 #[test] 3550 fn test_vhost_user_net_multiple_queues() { 3551 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 3552 } 3553 3554 #[test] 3555 fn test_vhost_user_net_tap_multiple_queues() { 3556 test_vhost_user_net( 3557 Some("vunet-tap1"), 3558 4, 3559 &prepare_vhost_user_net_daemon, 3560 false, 3561 false, 3562 ) 3563 } 3564 3565 #[test] 3566 fn test_vhost_user_net_host_mac() { 3567 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 3568 } 3569 3570 #[test] 3571 fn test_vhost_user_net_client_mode() { 3572 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 3573 } 3574 3575 #[test] 3576 #[cfg(not(target_arch = "aarch64"))] 3577 fn test_vhost_user_blk_default() { 3578 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 3579 } 3580 3581 #[test] 3582 #[cfg(not(target_arch = "aarch64"))] 3583 fn test_vhost_user_blk_readonly() { 3584 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3585 } 3586 3587 #[test] 3588 #[cfg(not(target_arch = "aarch64"))] 3589 fn test_vhost_user_blk_direct() { 3590 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3591 } 3592 3593 #[test] 3594 fn test_boot_from_vhost_user_blk_default() { 3595 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3596 } 3597 3598 #[test] 3599 #[cfg(target_arch = "x86_64")] 3600 fn test_split_irqchip() { 3601 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3602 let guest = Guest::new(Box::new(focal)); 3603 3604 let mut child = GuestCommand::new(&guest) 3605 .args(["--cpus", "boot=1"]) 3606 .args(["--memory", "size=512M"]) 3607 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3608 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3609 .default_disks() 3610 .default_net() 3611 .capture_output() 3612 .spawn() 3613 .unwrap(); 3614 3615 let r = std::panic::catch_unwind(|| { 3616 guest.wait_vm_boot(None).unwrap(); 3617 3618 assert_eq!( 3619 guest 3620 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3621 .unwrap() 3622 .trim() 3623 .parse::<u32>() 3624 .unwrap_or(1), 3625 0 3626 ); 3627 assert_eq!( 3628 guest 3629 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3630 .unwrap() 3631 .trim() 3632 .parse::<u32>() 3633 .unwrap_or(1), 3634 0 3635 ); 3636 }); 3637 3638 let _ = child.kill(); 3639 let output = child.wait_with_output().unwrap(); 3640 3641 handle_child_output(r, &output); 3642 } 3643 3644 #[test] 3645 #[cfg(target_arch = "x86_64")] 3646 fn test_dmi_serial_number() { 3647 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3648 let guest = Guest::new(Box::new(focal)); 3649 3650 let mut child = GuestCommand::new(&guest) 3651 .args(["--cpus", "boot=1"]) 3652 .args(["--memory", "size=512M"]) 3653 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3654 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3655 .args(["--platform", "serial_number=a=b;c=d"]) 3656 .default_disks() 3657 .default_net() 3658 .capture_output() 3659 .spawn() 3660 .unwrap(); 3661 3662 let r = std::panic::catch_unwind(|| { 3663 guest.wait_vm_boot(None).unwrap(); 3664 3665 assert_eq!( 3666 guest 3667 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3668 .unwrap() 3669 .trim(), 3670 "a=b;c=d" 3671 ); 3672 }); 3673 3674 let _ = child.kill(); 3675 let output = child.wait_with_output().unwrap(); 3676 3677 handle_child_output(r, &output); 3678 } 3679 3680 #[test] 3681 #[cfg(target_arch = "x86_64")] 3682 fn test_dmi_uuid() { 3683 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3684 let guest = Guest::new(Box::new(focal)); 3685 3686 let mut child = GuestCommand::new(&guest) 3687 .args(["--cpus", "boot=1"]) 3688 .args(["--memory", "size=512M"]) 3689 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3690 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3691 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3692 .default_disks() 3693 .default_net() 3694 .capture_output() 3695 .spawn() 3696 .unwrap(); 3697 3698 let r = std::panic::catch_unwind(|| { 3699 guest.wait_vm_boot(None).unwrap(); 3700 3701 assert_eq!( 3702 guest 3703 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3704 .unwrap() 3705 .trim(), 3706 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3707 ); 3708 }); 3709 3710 let _ = child.kill(); 3711 let output = child.wait_with_output().unwrap(); 3712 3713 handle_child_output(r, &output); 3714 } 3715 3716 #[test] 3717 #[cfg(target_arch = "x86_64")] 3718 fn test_dmi_oem_strings() { 3719 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3720 let guest = Guest::new(Box::new(focal)); 3721 3722 let s1 = "io.systemd.credential:xx=yy"; 3723 let s2 = "This is a test string"; 3724 3725 let oem_strings = format!("oem_strings=[{s1},{s2}]"); 3726 3727 let mut child = GuestCommand::new(&guest) 3728 .args(["--cpus", "boot=1"]) 3729 .args(["--memory", "size=512M"]) 3730 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3731 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3732 .args(["--platform", &oem_strings]) 3733 .default_disks() 3734 .default_net() 3735 .capture_output() 3736 .spawn() 3737 .unwrap(); 3738 3739 let r = std::panic::catch_unwind(|| { 3740 guest.wait_vm_boot(None).unwrap(); 3741 3742 assert_eq!( 3743 guest 3744 .ssh_command("sudo dmidecode --oem-string count") 3745 .unwrap() 3746 .trim(), 3747 "2" 3748 ); 3749 3750 assert_eq!( 3751 guest 3752 .ssh_command("sudo dmidecode --oem-string 1") 3753 .unwrap() 3754 .trim(), 3755 s1 3756 ); 3757 3758 assert_eq!( 3759 guest 3760 .ssh_command("sudo dmidecode --oem-string 2") 3761 .unwrap() 3762 .trim(), 3763 s2 3764 ); 3765 }); 3766 3767 let _ = child.kill(); 3768 let output = child.wait_with_output().unwrap(); 3769 3770 handle_child_output(r, &output); 3771 } 3772 3773 #[test] 3774 fn test_virtio_fs() { 3775 _test_virtio_fs(&prepare_virtiofsd, false, None) 3776 } 3777 3778 #[test] 3779 fn test_virtio_fs_hotplug() { 3780 _test_virtio_fs(&prepare_virtiofsd, true, None) 3781 } 3782 3783 #[test] 3784 #[cfg(not(feature = "mshv"))] 3785 fn test_virtio_fs_multi_segment_hotplug() { 3786 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3787 } 3788 3789 #[test] 3790 #[cfg(not(feature = "mshv"))] 3791 fn test_virtio_fs_multi_segment() { 3792 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3793 } 3794 3795 #[test] 3796 fn test_virtio_pmem_persist_writes() { 3797 test_virtio_pmem(false, false) 3798 } 3799 3800 #[test] 3801 fn test_virtio_pmem_discard_writes() { 3802 test_virtio_pmem(true, false) 3803 } 3804 3805 #[test] 3806 fn test_virtio_pmem_with_size() { 3807 test_virtio_pmem(true, true) 3808 } 3809 3810 #[test] 3811 fn test_boot_from_virtio_pmem() { 3812 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3813 let guest = Guest::new(Box::new(focal)); 3814 3815 let kernel_path = direct_kernel_boot_path(); 3816 3817 let mut child = GuestCommand::new(&guest) 3818 .args(["--cpus", "boot=1"]) 3819 .args(["--memory", "size=512M"]) 3820 .args(["--kernel", kernel_path.to_str().unwrap()]) 3821 .args([ 3822 "--disk", 3823 format!( 3824 "path={}", 3825 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3826 ) 3827 .as_str(), 3828 ]) 3829 .default_net() 3830 .args([ 3831 "--pmem", 3832 format!( 3833 "file={},size={}", 3834 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3835 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3836 .unwrap() 3837 .len() 3838 ) 3839 .as_str(), 3840 ]) 3841 .args([ 3842 "--cmdline", 3843 DIRECT_KERNEL_BOOT_CMDLINE 3844 .replace("vda1", "pmem0p1") 3845 .as_str(), 3846 ]) 3847 .capture_output() 3848 .spawn() 3849 .unwrap(); 3850 3851 let r = std::panic::catch_unwind(|| { 3852 guest.wait_vm_boot(None).unwrap(); 3853 3854 // Simple checks to validate the VM booted properly 3855 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3856 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3857 }); 3858 3859 let _ = child.kill(); 3860 let output = child.wait_with_output().unwrap(); 3861 3862 handle_child_output(r, &output); 3863 } 3864 3865 #[test] 3866 fn test_multiple_network_interfaces() { 3867 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3868 let guest = Guest::new(Box::new(focal)); 3869 3870 let kernel_path = direct_kernel_boot_path(); 3871 3872 let mut child = GuestCommand::new(&guest) 3873 .args(["--cpus", "boot=1"]) 3874 .args(["--memory", "size=512M"]) 3875 .args(["--kernel", kernel_path.to_str().unwrap()]) 3876 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3877 .default_disks() 3878 .args([ 3879 "--net", 3880 guest.default_net_string().as_str(), 3881 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3882 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3883 ]) 3884 .capture_output() 3885 .spawn() 3886 .unwrap(); 3887 3888 let r = std::panic::catch_unwind(|| { 3889 guest.wait_vm_boot(None).unwrap(); 3890 3891 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3892 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3893 3894 // 3 network interfaces + default localhost ==> 4 interfaces 3895 assert_eq!( 3896 guest 3897 .ssh_command("ip -o link | wc -l") 3898 .unwrap() 3899 .trim() 3900 .parse::<u32>() 3901 .unwrap_or_default(), 3902 4 3903 ); 3904 }); 3905 3906 let _ = child.kill(); 3907 let output = child.wait_with_output().unwrap(); 3908 3909 handle_child_output(r, &output); 3910 } 3911 3912 #[test] 3913 #[cfg(target_arch = "aarch64")] 3914 fn test_pmu_on() { 3915 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3916 let guest = Guest::new(Box::new(focal)); 3917 let mut child = GuestCommand::new(&guest) 3918 .args(["--cpus", "boot=1"]) 3919 .args(["--memory", "size=512M"]) 3920 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3921 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3922 .default_disks() 3923 .default_net() 3924 .capture_output() 3925 .spawn() 3926 .unwrap(); 3927 3928 let r = std::panic::catch_unwind(|| { 3929 guest.wait_vm_boot(None).unwrap(); 3930 3931 // Test that PMU exists. 3932 assert_eq!( 3933 guest 3934 .ssh_command(GREP_PMU_IRQ_CMD) 3935 .unwrap() 3936 .trim() 3937 .parse::<u32>() 3938 .unwrap_or_default(), 3939 1 3940 ); 3941 }); 3942 3943 let _ = child.kill(); 3944 let output = child.wait_with_output().unwrap(); 3945 3946 handle_child_output(r, &output); 3947 } 3948 3949 #[test] 3950 fn test_serial_off() { 3951 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3952 let guest = Guest::new(Box::new(focal)); 3953 let mut child = GuestCommand::new(&guest) 3954 .args(["--cpus", "boot=1"]) 3955 .args(["--memory", "size=512M"]) 3956 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3957 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3958 .default_disks() 3959 .default_net() 3960 .args(["--serial", "off"]) 3961 .capture_output() 3962 .spawn() 3963 .unwrap(); 3964 3965 let r = std::panic::catch_unwind(|| { 3966 guest.wait_vm_boot(None).unwrap(); 3967 3968 // Test that there is no ttyS0 3969 assert_eq!( 3970 guest 3971 .ssh_command(GREP_SERIAL_IRQ_CMD) 3972 .unwrap() 3973 .trim() 3974 .parse::<u32>() 3975 .unwrap_or(1), 3976 0 3977 ); 3978 }); 3979 3980 let _ = child.kill(); 3981 let output = child.wait_with_output().unwrap(); 3982 3983 handle_child_output(r, &output); 3984 } 3985 3986 #[test] 3987 fn test_serial_null() { 3988 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3989 let guest = Guest::new(Box::new(focal)); 3990 let mut cmd = GuestCommand::new(&guest); 3991 #[cfg(target_arch = "x86_64")] 3992 let console_str: &str = "console=ttyS0"; 3993 #[cfg(target_arch = "aarch64")] 3994 let console_str: &str = "console=ttyAMA0"; 3995 3996 cmd.args(["--cpus", "boot=1"]) 3997 .args(["--memory", "size=512M"]) 3998 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3999 .args([ 4000 "--cmdline", 4001 DIRECT_KERNEL_BOOT_CMDLINE 4002 .replace("console=hvc0 ", console_str) 4003 .as_str(), 4004 ]) 4005 .default_disks() 4006 .default_net() 4007 .args(["--serial", "null"]) 4008 .args(["--console", "off"]) 4009 .capture_output(); 4010 4011 let mut child = cmd.spawn().unwrap(); 4012 4013 let r = std::panic::catch_unwind(|| { 4014 guest.wait_vm_boot(None).unwrap(); 4015 4016 // Test that there is a ttyS0 4017 assert_eq!( 4018 guest 4019 .ssh_command(GREP_SERIAL_IRQ_CMD) 4020 .unwrap() 4021 .trim() 4022 .parse::<u32>() 4023 .unwrap_or_default(), 4024 1 4025 ); 4026 }); 4027 4028 let _ = child.kill(); 4029 let output = child.wait_with_output().unwrap(); 4030 handle_child_output(r, &output); 4031 4032 let r = std::panic::catch_unwind(|| { 4033 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4034 }); 4035 4036 handle_child_output(r, &output); 4037 } 4038 4039 #[test] 4040 fn test_serial_tty() { 4041 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4042 let guest = Guest::new(Box::new(focal)); 4043 4044 let kernel_path = direct_kernel_boot_path(); 4045 4046 #[cfg(target_arch = "x86_64")] 4047 let console_str: &str = "console=ttyS0"; 4048 #[cfg(target_arch = "aarch64")] 4049 let console_str: &str = "console=ttyAMA0"; 4050 4051 let mut child = GuestCommand::new(&guest) 4052 .args(["--cpus", "boot=1"]) 4053 .args(["--memory", "size=512M"]) 4054 .args(["--kernel", kernel_path.to_str().unwrap()]) 4055 .args([ 4056 "--cmdline", 4057 DIRECT_KERNEL_BOOT_CMDLINE 4058 .replace("console=hvc0 ", console_str) 4059 .as_str(), 4060 ]) 4061 .default_disks() 4062 .default_net() 4063 .args(["--serial", "tty"]) 4064 .args(["--console", "off"]) 4065 .capture_output() 4066 .spawn() 4067 .unwrap(); 4068 4069 let r = std::panic::catch_unwind(|| { 4070 guest.wait_vm_boot(None).unwrap(); 4071 4072 // Test that there is a ttyS0 4073 assert_eq!( 4074 guest 4075 .ssh_command(GREP_SERIAL_IRQ_CMD) 4076 .unwrap() 4077 .trim() 4078 .parse::<u32>() 4079 .unwrap_or_default(), 4080 1 4081 ); 4082 }); 4083 4084 // This sleep is needed to wait for the login prompt 4085 thread::sleep(std::time::Duration::new(2, 0)); 4086 4087 let _ = child.kill(); 4088 let output = child.wait_with_output().unwrap(); 4089 handle_child_output(r, &output); 4090 4091 let r = std::panic::catch_unwind(|| { 4092 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4093 }); 4094 4095 handle_child_output(r, &output); 4096 } 4097 4098 #[test] 4099 fn test_serial_file() { 4100 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4101 let guest = Guest::new(Box::new(focal)); 4102 4103 let serial_path = guest.tmp_dir.as_path().join("serial-output"); 4104 #[cfg(target_arch = "x86_64")] 4105 let console_str: &str = "console=ttyS0"; 4106 #[cfg(target_arch = "aarch64")] 4107 let console_str: &str = "console=ttyAMA0"; 4108 4109 let mut child = GuestCommand::new(&guest) 4110 .args(["--cpus", "boot=1"]) 4111 .args(["--memory", "size=512M"]) 4112 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4113 .args([ 4114 "--cmdline", 4115 DIRECT_KERNEL_BOOT_CMDLINE 4116 .replace("console=hvc0 ", console_str) 4117 .as_str(), 4118 ]) 4119 .default_disks() 4120 .default_net() 4121 .args([ 4122 "--serial", 4123 format!("file={}", serial_path.to_str().unwrap()).as_str(), 4124 ]) 4125 .capture_output() 4126 .spawn() 4127 .unwrap(); 4128 4129 let r = std::panic::catch_unwind(|| { 4130 guest.wait_vm_boot(None).unwrap(); 4131 4132 // Test that there is a ttyS0 4133 assert_eq!( 4134 guest 4135 .ssh_command(GREP_SERIAL_IRQ_CMD) 4136 .unwrap() 4137 .trim() 4138 .parse::<u32>() 4139 .unwrap_or_default(), 4140 1 4141 ); 4142 4143 guest.ssh_command("sudo shutdown -h now").unwrap(); 4144 }); 4145 4146 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4147 let _ = child.kill(); 4148 let output = child.wait_with_output().unwrap(); 4149 handle_child_output(r, &output); 4150 4151 let r = std::panic::catch_unwind(|| { 4152 // Check that the cloud-hypervisor binary actually terminated 4153 assert!(output.status.success()); 4154 4155 // Do this check after shutdown of the VM as an easy way to ensure 4156 // all writes are flushed to disk 4157 let mut f = std::fs::File::open(serial_path).unwrap(); 4158 let mut buf = String::new(); 4159 f.read_to_string(&mut buf).unwrap(); 4160 assert!(buf.contains(CONSOLE_TEST_STRING)); 4161 }); 4162 4163 handle_child_output(r, &output); 4164 } 4165 4166 #[test] 4167 fn test_pty_interaction() { 4168 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4169 let guest = Guest::new(Box::new(focal)); 4170 let api_socket = temp_api_path(&guest.tmp_dir); 4171 let serial_option = if cfg!(target_arch = "x86_64") { 4172 " console=ttyS0" 4173 } else { 4174 " console=ttyAMA0" 4175 }; 4176 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4177 4178 let mut child = GuestCommand::new(&guest) 4179 .args(["--cpus", "boot=1"]) 4180 .args(["--memory", "size=512M"]) 4181 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4182 .args(["--cmdline", &cmdline]) 4183 .default_disks() 4184 .default_net() 4185 .args(["--serial", "null"]) 4186 .args(["--console", "pty"]) 4187 .args(["--api-socket", &api_socket]) 4188 .spawn() 4189 .unwrap(); 4190 4191 let r = std::panic::catch_unwind(|| { 4192 guest.wait_vm_boot(None).unwrap(); 4193 // Get pty fd for console 4194 let console_path = get_pty_path(&api_socket, "console"); 4195 _test_pty_interaction(console_path); 4196 4197 guest.ssh_command("sudo shutdown -h now").unwrap(); 4198 }); 4199 4200 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4201 let _ = child.kill(); 4202 let output = child.wait_with_output().unwrap(); 4203 handle_child_output(r, &output); 4204 4205 let r = std::panic::catch_unwind(|| { 4206 // Check that the cloud-hypervisor binary actually terminated 4207 assert!(output.status.success()) 4208 }); 4209 handle_child_output(r, &output); 4210 } 4211 4212 #[test] 4213 fn test_serial_socket_interaction() { 4214 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4215 let guest = Guest::new(Box::new(focal)); 4216 let serial_socket = guest.tmp_dir.as_path().join("serial.socket"); 4217 let serial_socket_pty = guest.tmp_dir.as_path().join("serial.pty"); 4218 let serial_option = if cfg!(target_arch = "x86_64") { 4219 " console=ttyS0" 4220 } else { 4221 " console=ttyAMA0" 4222 }; 4223 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4224 4225 let mut child = GuestCommand::new(&guest) 4226 .args(["--cpus", "boot=1"]) 4227 .args(["--memory", "size=512M"]) 4228 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4229 .args(["--cmdline", &cmdline]) 4230 .default_disks() 4231 .default_net() 4232 .args(["--console", "null"]) 4233 .args([ 4234 "--serial", 4235 format!("socket={}", serial_socket.to_str().unwrap()).as_str(), 4236 ]) 4237 .spawn() 4238 .unwrap(); 4239 4240 let _ = std::panic::catch_unwind(|| { 4241 guest.wait_vm_boot(None).unwrap(); 4242 }); 4243 4244 let mut socat_command = Command::new("socat"); 4245 let socat_args = [ 4246 &format!("pty,link={},raw", serial_socket_pty.display()), 4247 &format!("UNIX-CONNECT:{}", serial_socket.display()), 4248 ]; 4249 socat_command.args(socat_args); 4250 4251 let mut socat_child = socat_command.spawn().unwrap(); 4252 thread::sleep(std::time::Duration::new(1, 0)); 4253 4254 let _ = std::panic::catch_unwind(|| { 4255 _test_pty_interaction(serial_socket_pty); 4256 }); 4257 4258 let _ = socat_child.kill(); 4259 4260 let r = std::panic::catch_unwind(|| { 4261 guest.ssh_command("sudo shutdown -h now").unwrap(); 4262 }); 4263 4264 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4265 let _ = child.kill(); 4266 let output = child.wait_with_output().unwrap(); 4267 handle_child_output(r, &output); 4268 4269 let r = std::panic::catch_unwind(|| { 4270 // Check that the cloud-hypervisor binary actually terminated 4271 if !output.status.success() { 4272 panic!( 4273 "Cloud Hypervisor process failed to terminate gracefully: {:?}", 4274 output.status 4275 ); 4276 } 4277 }); 4278 handle_child_output(r, &output); 4279 } 4280 4281 #[test] 4282 fn test_virtio_console() { 4283 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4284 let guest = Guest::new(Box::new(focal)); 4285 4286 let kernel_path = direct_kernel_boot_path(); 4287 4288 let mut child = GuestCommand::new(&guest) 4289 .args(["--cpus", "boot=1"]) 4290 .args(["--memory", "size=512M"]) 4291 .args(["--kernel", kernel_path.to_str().unwrap()]) 4292 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4293 .default_disks() 4294 .default_net() 4295 .args(["--console", "tty"]) 4296 .args(["--serial", "null"]) 4297 .capture_output() 4298 .spawn() 4299 .unwrap(); 4300 4301 let text = String::from("On a branch floating down river a cricket, singing."); 4302 let cmd = format!("echo {text} | sudo tee /dev/hvc0"); 4303 4304 let r = std::panic::catch_unwind(|| { 4305 guest.wait_vm_boot(None).unwrap(); 4306 4307 assert!(guest 4308 .does_device_vendor_pair_match("0x1043", "0x1af4") 4309 .unwrap_or_default()); 4310 4311 guest.ssh_command(&cmd).unwrap(); 4312 }); 4313 4314 let _ = child.kill(); 4315 let output = child.wait_with_output().unwrap(); 4316 handle_child_output(r, &output); 4317 4318 let r = std::panic::catch_unwind(|| { 4319 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 4320 }); 4321 4322 handle_child_output(r, &output); 4323 } 4324 4325 #[test] 4326 fn test_console_file() { 4327 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4328 let guest = Guest::new(Box::new(focal)); 4329 4330 let console_path = guest.tmp_dir.as_path().join("console-output"); 4331 let mut child = GuestCommand::new(&guest) 4332 .args(["--cpus", "boot=1"]) 4333 .args(["--memory", "size=512M"]) 4334 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4335 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4336 .default_disks() 4337 .default_net() 4338 .args([ 4339 "--console", 4340 format!("file={}", console_path.to_str().unwrap()).as_str(), 4341 ]) 4342 .capture_output() 4343 .spawn() 4344 .unwrap(); 4345 4346 guest.wait_vm_boot(None).unwrap(); 4347 4348 guest.ssh_command("sudo shutdown -h now").unwrap(); 4349 4350 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4351 let _ = child.kill(); 4352 let output = child.wait_with_output().unwrap(); 4353 4354 let r = std::panic::catch_unwind(|| { 4355 // Check that the cloud-hypervisor binary actually terminated 4356 assert!(output.status.success()); 4357 4358 // Do this check after shutdown of the VM as an easy way to ensure 4359 // all writes are flushed to disk 4360 let mut f = std::fs::File::open(console_path).unwrap(); 4361 let mut buf = String::new(); 4362 f.read_to_string(&mut buf).unwrap(); 4363 4364 if !buf.contains(CONSOLE_TEST_STRING) { 4365 eprintln!( 4366 "\n\n==== Console file output ====\n\n{buf}\n\n==== End console file output ====" 4367 ); 4368 } 4369 assert!(buf.contains(CONSOLE_TEST_STRING)); 4370 }); 4371 4372 handle_child_output(r, &output); 4373 } 4374 4375 #[test] 4376 #[cfg(target_arch = "x86_64")] 4377 #[cfg(not(feature = "mshv"))] 4378 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 4379 // backed networking interfaces, bound through a simple bridge on the host. 4380 // So if the nested cloud-hypervisor succeeds in getting a directly 4381 // assigned interface from its cloud-hypervisor host, we should be able to 4382 // ssh into it, and verify that it's running with the right kernel command 4383 // line (We tag the command line from cloud-hypervisor for that purpose). 4384 // The third device is added to validate that hotplug works correctly since 4385 // it is being added to the L2 VM through hotplugging mechanism. 4386 // Also, we pass-through a virtio-blk device to the L2 VM to test the 32-bit 4387 // vfio device support 4388 fn test_vfio() { 4389 setup_vfio_network_interfaces(); 4390 4391 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 4392 let guest = Guest::new_from_ip_range(Box::new(jammy), "172.18", 0); 4393 4394 let mut workload_path = dirs::home_dir().unwrap(); 4395 workload_path.push("workloads"); 4396 4397 let kernel_path = direct_kernel_boot_path(); 4398 4399 let mut vfio_path = workload_path.clone(); 4400 vfio_path.push("vfio"); 4401 4402 let mut cloud_init_vfio_base_path = vfio_path.clone(); 4403 cloud_init_vfio_base_path.push("cloudinit.img"); 4404 4405 // We copy our cloudinit into the vfio mount point, for the nested 4406 // cloud-hypervisor guest to use. 4407 rate_limited_copy( 4408 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 4409 &cloud_init_vfio_base_path, 4410 ) 4411 .expect("copying of cloud-init disk failed"); 4412 4413 let mut vfio_disk_path = workload_path.clone(); 4414 vfio_disk_path.push("vfio.img"); 4415 4416 // Create the vfio disk image 4417 let output = Command::new("mkfs.ext4") 4418 .arg("-d") 4419 .arg(vfio_path.to_str().unwrap()) 4420 .arg(vfio_disk_path.to_str().unwrap()) 4421 .arg("2g") 4422 .output() 4423 .unwrap(); 4424 if !output.status.success() { 4425 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 4426 panic!("mkfs.ext4 command generated an error"); 4427 } 4428 4429 let mut blk_file_path = workload_path; 4430 blk_file_path.push("blk.img"); 4431 4432 let vfio_tap0 = "vfio-tap0"; 4433 let vfio_tap1 = "vfio-tap1"; 4434 let vfio_tap2 = "vfio-tap2"; 4435 let vfio_tap3 = "vfio-tap3"; 4436 4437 let mut child = GuestCommand::new(&guest) 4438 .args(["--cpus", "boot=4"]) 4439 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 4440 .args(["--kernel", kernel_path.to_str().unwrap()]) 4441 .args([ 4442 "--disk", 4443 format!( 4444 "path={}", 4445 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4446 ) 4447 .as_str(), 4448 format!( 4449 "path={}", 4450 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4451 ) 4452 .as_str(), 4453 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 4454 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 4455 ]) 4456 .args([ 4457 "--cmdline", 4458 format!( 4459 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 4460 ) 4461 .as_str(), 4462 ]) 4463 .args([ 4464 "--net", 4465 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 4466 format!( 4467 "tap={},mac={},iommu=on", 4468 vfio_tap1, guest.network.l2_guest_mac1 4469 ) 4470 .as_str(), 4471 format!( 4472 "tap={},mac={},iommu=on", 4473 vfio_tap2, guest.network.l2_guest_mac2 4474 ) 4475 .as_str(), 4476 format!( 4477 "tap={},mac={},iommu=on", 4478 vfio_tap3, guest.network.l2_guest_mac3 4479 ) 4480 .as_str(), 4481 ]) 4482 .capture_output() 4483 .spawn() 4484 .unwrap(); 4485 4486 thread::sleep(std::time::Duration::new(30, 0)); 4487 4488 let r = std::panic::catch_unwind(|| { 4489 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 4490 thread::sleep(std::time::Duration::new(120, 0)); 4491 4492 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 4493 // added to its kernel command line. 4494 // Let's ssh into it and verify that it's there. If it is it means 4495 // we're in the right guest (The L2 one) because the QEMU L1 guest 4496 // does not have this command line tag. 4497 assert!(check_matched_lines_count( 4498 guest.ssh_command_l2_1("cat /proc/cmdline").unwrap().trim(), 4499 vec!["VFIOTAG"], 4500 1 4501 )); 4502 4503 // Let's also verify from the second virtio-net device passed to 4504 // the L2 VM. 4505 assert!(check_matched_lines_count( 4506 guest.ssh_command_l2_2("cat /proc/cmdline").unwrap().trim(), 4507 vec!["VFIOTAG"], 4508 1 4509 )); 4510 4511 // Check the amount of PCI devices appearing in L2 VM. 4512 assert!(check_lines_count( 4513 guest 4514 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4515 .unwrap() 4516 .trim(), 4517 8 4518 )); 4519 4520 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 4521 assert!(check_matched_lines_count( 4522 guest.ssh_command_l2_1("lsblk").unwrap().trim(), 4523 vec!["vdc", "16M"], 4524 1 4525 )); 4526 4527 // Hotplug an extra virtio-net device through L2 VM. 4528 guest 4529 .ssh_command_l1( 4530 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 4531 ) 4532 .unwrap(); 4533 guest 4534 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 4535 .unwrap(); 4536 let vfio_hotplug_output = guest 4537 .ssh_command_l1( 4538 "sudo /mnt/ch-remote \ 4539 --api-socket=/tmp/ch_api.sock \ 4540 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 4541 ) 4542 .unwrap(); 4543 assert!(check_matched_lines_count( 4544 vfio_hotplug_output.trim(), 4545 vec!["{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}"], 4546 1 4547 )); 4548 4549 thread::sleep(std::time::Duration::new(10, 0)); 4550 4551 // Let's also verify from the third virtio-net device passed to 4552 // the L2 VM. This third device has been hotplugged through the L2 4553 // VM, so this is our way to validate hotplug works for VFIO PCI. 4554 assert!(check_matched_lines_count( 4555 guest.ssh_command_l2_3("cat /proc/cmdline").unwrap().trim(), 4556 vec!["VFIOTAG"], 4557 1 4558 )); 4559 4560 // Check the amount of PCI devices appearing in L2 VM. 4561 // There should be one more device than before, raising the count 4562 // up to 9 PCI devices. 4563 assert!(check_lines_count( 4564 guest 4565 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4566 .unwrap() 4567 .trim(), 4568 9 4569 )); 4570 4571 // Let's now verify that we can correctly remove the virtio-net 4572 // device through the "remove-device" command responsible for 4573 // unplugging VFIO devices. 4574 guest 4575 .ssh_command_l1( 4576 "sudo /mnt/ch-remote \ 4577 --api-socket=/tmp/ch_api.sock \ 4578 remove-device vfio123", 4579 ) 4580 .unwrap(); 4581 thread::sleep(std::time::Duration::new(10, 0)); 4582 4583 // Check the amount of PCI devices appearing in L2 VM is back down 4584 // to 8 devices. 4585 assert!(check_lines_count( 4586 guest 4587 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4588 .unwrap() 4589 .trim(), 4590 8 4591 )); 4592 4593 // Perform memory hotplug in L2 and validate the memory is showing 4594 // up as expected. In order to check, we will use the virtio-net 4595 // device already passed through L2 as a VFIO device, this will 4596 // verify that VFIO devices are functional with memory hotplug. 4597 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 4598 guest 4599 .ssh_command_l2_1( 4600 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 4601 ) 4602 .unwrap(); 4603 guest 4604 .ssh_command_l1( 4605 "sudo /mnt/ch-remote \ 4606 --api-socket=/tmp/ch_api.sock \ 4607 resize --memory=1073741824", 4608 ) 4609 .unwrap(); 4610 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 4611 }); 4612 4613 let _ = child.kill(); 4614 let output = child.wait_with_output().unwrap(); 4615 4616 cleanup_vfio_network_interfaces(); 4617 4618 handle_child_output(r, &output); 4619 } 4620 4621 #[test] 4622 fn test_direct_kernel_boot_noacpi() { 4623 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4624 let guest = Guest::new(Box::new(focal)); 4625 4626 let kernel_path = direct_kernel_boot_path(); 4627 4628 let mut child = GuestCommand::new(&guest) 4629 .args(["--cpus", "boot=1"]) 4630 .args(["--memory", "size=512M"]) 4631 .args(["--kernel", kernel_path.to_str().unwrap()]) 4632 .args([ 4633 "--cmdline", 4634 format!("{DIRECT_KERNEL_BOOT_CMDLINE} acpi=off").as_str(), 4635 ]) 4636 .default_disks() 4637 .default_net() 4638 .capture_output() 4639 .spawn() 4640 .unwrap(); 4641 4642 let r = std::panic::catch_unwind(|| { 4643 guest.wait_vm_boot(None).unwrap(); 4644 4645 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4646 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4647 }); 4648 4649 let _ = child.kill(); 4650 let output = child.wait_with_output().unwrap(); 4651 4652 handle_child_output(r, &output); 4653 } 4654 4655 #[test] 4656 fn test_virtio_vsock() { 4657 _test_virtio_vsock(false) 4658 } 4659 4660 #[test] 4661 fn test_virtio_vsock_hotplug() { 4662 _test_virtio_vsock(true); 4663 } 4664 4665 #[test] 4666 fn test_api_http_shutdown() { 4667 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4668 let guest = Guest::new(Box::new(focal)); 4669 4670 _test_api_shutdown(TargetApi::new_http_api(&guest.tmp_dir), guest) 4671 } 4672 4673 #[test] 4674 fn test_api_http_delete() { 4675 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4676 let guest = Guest::new(Box::new(focal)); 4677 4678 _test_api_delete(TargetApi::new_http_api(&guest.tmp_dir), guest); 4679 } 4680 4681 #[test] 4682 fn test_api_http_pause_resume() { 4683 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4684 let guest = Guest::new(Box::new(focal)); 4685 4686 _test_api_pause_resume(TargetApi::new_http_api(&guest.tmp_dir), guest) 4687 } 4688 4689 #[test] 4690 fn test_api_http_create_boot() { 4691 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4692 let guest = Guest::new(Box::new(focal)); 4693 4694 _test_api_create_boot(TargetApi::new_http_api(&guest.tmp_dir), guest) 4695 } 4696 4697 #[test] 4698 fn test_virtio_iommu() { 4699 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4700 } 4701 4702 #[test] 4703 // We cannot force the software running in the guest to reprogram the BAR 4704 // with some different addresses, but we have a reliable way of testing it 4705 // with a standard Linux kernel. 4706 // By removing a device from the PCI tree, and then rescanning the tree, 4707 // Linux consistently chooses to reorganize the PCI device BARs to other 4708 // locations in the guest address space. 4709 // This test creates a dedicated PCI network device to be checked as being 4710 // properly probed first, then removing it, and adding it again by doing a 4711 // rescan. 4712 fn test_pci_bar_reprogramming() { 4713 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4714 let guest = Guest::new(Box::new(focal)); 4715 4716 #[cfg(target_arch = "x86_64")] 4717 let kernel_path = direct_kernel_boot_path(); 4718 #[cfg(target_arch = "aarch64")] 4719 let kernel_path = edk2_path(); 4720 4721 let mut child = GuestCommand::new(&guest) 4722 .args(["--cpus", "boot=1"]) 4723 .args(["--memory", "size=512M"]) 4724 .args(["--kernel", kernel_path.to_str().unwrap()]) 4725 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4726 .default_disks() 4727 .args([ 4728 "--net", 4729 guest.default_net_string().as_str(), 4730 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4731 ]) 4732 .capture_output() 4733 .spawn() 4734 .unwrap(); 4735 4736 let r = std::panic::catch_unwind(|| { 4737 guest.wait_vm_boot(None).unwrap(); 4738 4739 // 2 network interfaces + default localhost ==> 3 interfaces 4740 assert_eq!( 4741 guest 4742 .ssh_command("ip -o link | wc -l") 4743 .unwrap() 4744 .trim() 4745 .parse::<u32>() 4746 .unwrap_or_default(), 4747 3 4748 ); 4749 4750 let init_bar_addr = guest 4751 .ssh_command( 4752 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4753 ) 4754 .unwrap(); 4755 4756 // Remove the PCI device 4757 guest 4758 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4759 .unwrap(); 4760 4761 // Only 1 network interface left + default localhost ==> 2 interfaces 4762 assert_eq!( 4763 guest 4764 .ssh_command("ip -o link | wc -l") 4765 .unwrap() 4766 .trim() 4767 .parse::<u32>() 4768 .unwrap_or_default(), 4769 2 4770 ); 4771 4772 // Remove the PCI device 4773 guest 4774 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4775 .unwrap(); 4776 4777 // Back to 2 network interface + default localhost ==> 3 interfaces 4778 assert_eq!( 4779 guest 4780 .ssh_command("ip -o link | wc -l") 4781 .unwrap() 4782 .trim() 4783 .parse::<u32>() 4784 .unwrap_or_default(), 4785 3 4786 ); 4787 4788 let new_bar_addr = guest 4789 .ssh_command( 4790 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4791 ) 4792 .unwrap(); 4793 4794 // Let's compare the BAR addresses for our virtio-net device. 4795 // They should be different as we expect the BAR reprogramming 4796 // to have happened. 4797 assert_ne!(init_bar_addr, new_bar_addr); 4798 }); 4799 4800 let _ = child.kill(); 4801 let output = child.wait_with_output().unwrap(); 4802 4803 handle_child_output(r, &output); 4804 } 4805 4806 #[test] 4807 fn test_memory_mergeable_off() { 4808 test_memory_mergeable(false) 4809 } 4810 4811 #[test] 4812 #[cfg(target_arch = "x86_64")] 4813 fn test_cpu_hotplug() { 4814 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4815 let guest = Guest::new(Box::new(focal)); 4816 let api_socket = temp_api_path(&guest.tmp_dir); 4817 4818 let kernel_path = direct_kernel_boot_path(); 4819 4820 let mut child = GuestCommand::new(&guest) 4821 .args(["--cpus", "boot=2,max=4"]) 4822 .args(["--memory", "size=512M"]) 4823 .args(["--kernel", kernel_path.to_str().unwrap()]) 4824 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4825 .default_disks() 4826 .default_net() 4827 .args(["--api-socket", &api_socket]) 4828 .capture_output() 4829 .spawn() 4830 .unwrap(); 4831 4832 let r = std::panic::catch_unwind(|| { 4833 guest.wait_vm_boot(None).unwrap(); 4834 4835 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4836 4837 // Resize the VM 4838 let desired_vcpus = 4; 4839 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4840 4841 guest 4842 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4843 .unwrap(); 4844 guest 4845 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4846 .unwrap(); 4847 thread::sleep(std::time::Duration::new(10, 0)); 4848 assert_eq!( 4849 guest.get_cpu_count().unwrap_or_default(), 4850 u32::from(desired_vcpus) 4851 ); 4852 4853 guest.reboot_linux(0, None); 4854 4855 assert_eq!( 4856 guest.get_cpu_count().unwrap_or_default(), 4857 u32::from(desired_vcpus) 4858 ); 4859 4860 // Resize the VM 4861 let desired_vcpus = 2; 4862 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4863 4864 thread::sleep(std::time::Duration::new(10, 0)); 4865 assert_eq!( 4866 guest.get_cpu_count().unwrap_or_default(), 4867 u32::from(desired_vcpus) 4868 ); 4869 4870 // Resize the VM back up to 4 4871 let desired_vcpus = 4; 4872 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4873 4874 guest 4875 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4876 .unwrap(); 4877 guest 4878 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4879 .unwrap(); 4880 thread::sleep(std::time::Duration::new(10, 0)); 4881 assert_eq!( 4882 guest.get_cpu_count().unwrap_or_default(), 4883 u32::from(desired_vcpus) 4884 ); 4885 }); 4886 4887 let _ = child.kill(); 4888 let output = child.wait_with_output().unwrap(); 4889 4890 handle_child_output(r, &output); 4891 } 4892 4893 #[test] 4894 fn test_memory_hotplug() { 4895 #[cfg(target_arch = "aarch64")] 4896 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4897 #[cfg(target_arch = "x86_64")] 4898 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4899 let focal = UbuntuDiskConfig::new(focal_image); 4900 let guest = Guest::new(Box::new(focal)); 4901 let api_socket = temp_api_path(&guest.tmp_dir); 4902 4903 #[cfg(target_arch = "aarch64")] 4904 let kernel_path = edk2_path(); 4905 #[cfg(target_arch = "x86_64")] 4906 let kernel_path = direct_kernel_boot_path(); 4907 4908 let mut child = GuestCommand::new(&guest) 4909 .args(["--cpus", "boot=2,max=4"]) 4910 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4911 .args(["--kernel", kernel_path.to_str().unwrap()]) 4912 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4913 .default_disks() 4914 .default_net() 4915 .args(["--balloon", "size=0"]) 4916 .args(["--api-socket", &api_socket]) 4917 .capture_output() 4918 .spawn() 4919 .unwrap(); 4920 4921 let r = std::panic::catch_unwind(|| { 4922 guest.wait_vm_boot(None).unwrap(); 4923 4924 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4925 4926 guest.enable_memory_hotplug(); 4927 4928 // Add RAM to the VM 4929 let desired_ram = 1024 << 20; 4930 resize_command(&api_socket, None, Some(desired_ram), None, None); 4931 4932 thread::sleep(std::time::Duration::new(10, 0)); 4933 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4934 4935 // Use balloon to remove RAM from the VM 4936 let desired_balloon = 512 << 20; 4937 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4938 4939 thread::sleep(std::time::Duration::new(10, 0)); 4940 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4941 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4942 4943 guest.reboot_linux(0, None); 4944 4945 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4946 4947 // Use balloon add RAM to the VM 4948 let desired_balloon = 0; 4949 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4950 4951 thread::sleep(std::time::Duration::new(10, 0)); 4952 4953 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4954 4955 guest.enable_memory_hotplug(); 4956 4957 // Add RAM to the VM 4958 let desired_ram = 2048 << 20; 4959 resize_command(&api_socket, None, Some(desired_ram), None, None); 4960 4961 thread::sleep(std::time::Duration::new(10, 0)); 4962 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4963 4964 // Remove RAM to the VM (only applies after reboot) 4965 let desired_ram = 1024 << 20; 4966 resize_command(&api_socket, None, Some(desired_ram), None, None); 4967 4968 guest.reboot_linux(1, None); 4969 4970 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4971 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4972 }); 4973 4974 let _ = child.kill(); 4975 let output = child.wait_with_output().unwrap(); 4976 4977 handle_child_output(r, &output); 4978 } 4979 4980 #[test] 4981 #[cfg(not(feature = "mshv"))] 4982 fn test_virtio_mem() { 4983 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4984 let guest = Guest::new(Box::new(focal)); 4985 let api_socket = temp_api_path(&guest.tmp_dir); 4986 4987 let kernel_path = direct_kernel_boot_path(); 4988 4989 let mut child = GuestCommand::new(&guest) 4990 .args(["--cpus", "boot=2,max=4"]) 4991 .args([ 4992 "--memory", 4993 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4994 ]) 4995 .args(["--kernel", kernel_path.to_str().unwrap()]) 4996 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4997 .default_disks() 4998 .default_net() 4999 .args(["--api-socket", &api_socket]) 5000 .capture_output() 5001 .spawn() 5002 .unwrap(); 5003 5004 let r = std::panic::catch_unwind(|| { 5005 guest.wait_vm_boot(None).unwrap(); 5006 5007 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5008 5009 guest.enable_memory_hotplug(); 5010 5011 // Add RAM to the VM 5012 let desired_ram = 1024 << 20; 5013 resize_command(&api_socket, None, Some(desired_ram), None, None); 5014 5015 thread::sleep(std::time::Duration::new(10, 0)); 5016 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5017 5018 // Add RAM to the VM 5019 let desired_ram = 2048 << 20; 5020 resize_command(&api_socket, None, Some(desired_ram), None, None); 5021 5022 thread::sleep(std::time::Duration::new(10, 0)); 5023 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 5024 5025 // Remove RAM from the VM 5026 let desired_ram = 1024 << 20; 5027 resize_command(&api_socket, None, Some(desired_ram), None, None); 5028 5029 thread::sleep(std::time::Duration::new(10, 0)); 5030 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5031 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5032 5033 guest.reboot_linux(0, None); 5034 5035 // Check the amount of memory after reboot is 1GiB 5036 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5037 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5038 5039 // Check we can still resize to 512MiB 5040 let desired_ram = 512 << 20; 5041 resize_command(&api_socket, None, Some(desired_ram), None, None); 5042 thread::sleep(std::time::Duration::new(10, 0)); 5043 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5044 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 5045 }); 5046 5047 let _ = child.kill(); 5048 let output = child.wait_with_output().unwrap(); 5049 5050 handle_child_output(r, &output); 5051 } 5052 5053 #[test] 5054 #[cfg(target_arch = "x86_64")] 5055 #[cfg(not(feature = "mshv"))] 5056 // Test both vCPU and memory resizing together 5057 fn test_resize() { 5058 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5059 let guest = Guest::new(Box::new(focal)); 5060 let api_socket = temp_api_path(&guest.tmp_dir); 5061 5062 let kernel_path = direct_kernel_boot_path(); 5063 5064 let mut child = GuestCommand::new(&guest) 5065 .args(["--cpus", "boot=2,max=4"]) 5066 .args(["--memory", "size=512M,hotplug_size=8192M"]) 5067 .args(["--kernel", kernel_path.to_str().unwrap()]) 5068 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5069 .default_disks() 5070 .default_net() 5071 .args(["--api-socket", &api_socket]) 5072 .capture_output() 5073 .spawn() 5074 .unwrap(); 5075 5076 let r = std::panic::catch_unwind(|| { 5077 guest.wait_vm_boot(None).unwrap(); 5078 5079 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 5080 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5081 5082 guest.enable_memory_hotplug(); 5083 5084 // Resize the VM 5085 let desired_vcpus = 4; 5086 let desired_ram = 1024 << 20; 5087 resize_command( 5088 &api_socket, 5089 Some(desired_vcpus), 5090 Some(desired_ram), 5091 None, 5092 None, 5093 ); 5094 5095 guest 5096 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 5097 .unwrap(); 5098 guest 5099 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 5100 .unwrap(); 5101 thread::sleep(std::time::Duration::new(10, 0)); 5102 assert_eq!( 5103 guest.get_cpu_count().unwrap_or_default(), 5104 u32::from(desired_vcpus) 5105 ); 5106 5107 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5108 }); 5109 5110 let _ = child.kill(); 5111 let output = child.wait_with_output().unwrap(); 5112 5113 handle_child_output(r, &output); 5114 } 5115 5116 #[test] 5117 fn test_memory_overhead() { 5118 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5119 let guest = Guest::new(Box::new(focal)); 5120 5121 let kernel_path = direct_kernel_boot_path(); 5122 5123 let guest_memory_size_kb = 512 * 1024; 5124 5125 let mut child = GuestCommand::new(&guest) 5126 .args(["--cpus", "boot=1"]) 5127 .args(["--memory", format!("size={guest_memory_size_kb}K").as_str()]) 5128 .args(["--kernel", kernel_path.to_str().unwrap()]) 5129 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5130 .default_net() 5131 .default_disks() 5132 .capture_output() 5133 .spawn() 5134 .unwrap(); 5135 5136 guest.wait_vm_boot(None).unwrap(); 5137 5138 let r = std::panic::catch_unwind(|| { 5139 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 5140 eprintln!("Guest memory overhead: {overhead} vs {MAXIMUM_VMM_OVERHEAD_KB}"); 5141 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 5142 }); 5143 5144 let _ = child.kill(); 5145 let output = child.wait_with_output().unwrap(); 5146 5147 handle_child_output(r, &output); 5148 } 5149 5150 #[test] 5151 fn test_disk_hotplug() { 5152 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5153 let guest = Guest::new(Box::new(focal)); 5154 5155 #[cfg(target_arch = "x86_64")] 5156 let kernel_path = direct_kernel_boot_path(); 5157 #[cfg(target_arch = "aarch64")] 5158 let kernel_path = edk2_path(); 5159 5160 let api_socket = temp_api_path(&guest.tmp_dir); 5161 5162 let mut child = GuestCommand::new(&guest) 5163 .args(["--api-socket", &api_socket]) 5164 .args(["--cpus", "boot=1"]) 5165 .args(["--memory", "size=512M"]) 5166 .args(["--kernel", kernel_path.to_str().unwrap()]) 5167 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5168 .default_disks() 5169 .default_net() 5170 .capture_output() 5171 .spawn() 5172 .unwrap(); 5173 5174 let r = std::panic::catch_unwind(|| { 5175 guest.wait_vm_boot(None).unwrap(); 5176 5177 // Check /dev/vdc is not there 5178 assert_eq!( 5179 guest 5180 .ssh_command("lsblk | grep -c vdc.*16M || true") 5181 .unwrap() 5182 .trim() 5183 .parse::<u32>() 5184 .unwrap_or(1), 5185 0 5186 ); 5187 5188 // Now let's add the extra disk. 5189 let mut blk_file_path = dirs::home_dir().unwrap(); 5190 blk_file_path.push("workloads"); 5191 blk_file_path.push("blk.img"); 5192 let (cmd_success, cmd_output) = remote_command_w_output( 5193 &api_socket, 5194 "add-disk", 5195 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5196 ); 5197 assert!(cmd_success); 5198 assert!(String::from_utf8_lossy(&cmd_output) 5199 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5200 5201 thread::sleep(std::time::Duration::new(10, 0)); 5202 5203 // Check that /dev/vdc exists and the block size is 16M. 5204 assert_eq!( 5205 guest 5206 .ssh_command("lsblk | grep vdc | grep -c 16M") 5207 .unwrap() 5208 .trim() 5209 .parse::<u32>() 5210 .unwrap_or_default(), 5211 1 5212 ); 5213 // And check the block device can be read. 5214 guest 5215 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5216 .unwrap(); 5217 5218 // Let's remove it the extra disk. 5219 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5220 thread::sleep(std::time::Duration::new(5, 0)); 5221 // And check /dev/vdc is not there 5222 assert_eq!( 5223 guest 5224 .ssh_command("lsblk | grep -c vdc.*16M || true") 5225 .unwrap() 5226 .trim() 5227 .parse::<u32>() 5228 .unwrap_or(1), 5229 0 5230 ); 5231 5232 // And add it back to validate unplug did work correctly. 5233 let (cmd_success, cmd_output) = remote_command_w_output( 5234 &api_socket, 5235 "add-disk", 5236 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5237 ); 5238 assert!(cmd_success); 5239 assert!(String::from_utf8_lossy(&cmd_output) 5240 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5241 5242 thread::sleep(std::time::Duration::new(10, 0)); 5243 5244 // Check that /dev/vdc exists and the block size is 16M. 5245 assert_eq!( 5246 guest 5247 .ssh_command("lsblk | grep vdc | grep -c 16M") 5248 .unwrap() 5249 .trim() 5250 .parse::<u32>() 5251 .unwrap_or_default(), 5252 1 5253 ); 5254 // And check the block device can be read. 5255 guest 5256 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5257 .unwrap(); 5258 5259 // Reboot the VM. 5260 guest.reboot_linux(0, None); 5261 5262 // Check still there after reboot 5263 assert_eq!( 5264 guest 5265 .ssh_command("lsblk | grep vdc | grep -c 16M") 5266 .unwrap() 5267 .trim() 5268 .parse::<u32>() 5269 .unwrap_or_default(), 5270 1 5271 ); 5272 5273 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5274 5275 thread::sleep(std::time::Duration::new(20, 0)); 5276 5277 // Check device has gone away 5278 assert_eq!( 5279 guest 5280 .ssh_command("lsblk | grep -c vdc.*16M || true") 5281 .unwrap() 5282 .trim() 5283 .parse::<u32>() 5284 .unwrap_or(1), 5285 0 5286 ); 5287 5288 guest.reboot_linux(1, None); 5289 5290 // Check device still absent 5291 assert_eq!( 5292 guest 5293 .ssh_command("lsblk | grep -c vdc.*16M || true") 5294 .unwrap() 5295 .trim() 5296 .parse::<u32>() 5297 .unwrap_or(1), 5298 0 5299 ); 5300 }); 5301 5302 let _ = child.kill(); 5303 let output = child.wait_with_output().unwrap(); 5304 5305 handle_child_output(r, &output); 5306 } 5307 5308 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 5309 const LOOP_CONFIGURE: u64 = 0x4c0a; 5310 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 5311 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 5312 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 5313 5314 #[repr(C)] 5315 struct LoopInfo64 { 5316 lo_device: u64, 5317 lo_inode: u64, 5318 lo_rdevice: u64, 5319 lo_offset: u64, 5320 lo_sizelimit: u64, 5321 lo_number: u32, 5322 lo_encrypt_type: u32, 5323 lo_encrypt_key_size: u32, 5324 lo_flags: u32, 5325 lo_file_name: [u8; 64], 5326 lo_crypt_name: [u8; 64], 5327 lo_encrypt_key: [u8; 32], 5328 lo_init: [u64; 2], 5329 } 5330 5331 impl Default for LoopInfo64 { 5332 fn default() -> Self { 5333 LoopInfo64 { 5334 lo_device: 0, 5335 lo_inode: 0, 5336 lo_rdevice: 0, 5337 lo_offset: 0, 5338 lo_sizelimit: 0, 5339 lo_number: 0, 5340 lo_encrypt_type: 0, 5341 lo_encrypt_key_size: 0, 5342 lo_flags: 0, 5343 lo_file_name: [0; 64], 5344 lo_crypt_name: [0; 64], 5345 lo_encrypt_key: [0; 32], 5346 lo_init: [0; 2], 5347 } 5348 } 5349 } 5350 5351 #[derive(Default)] 5352 #[repr(C)] 5353 struct LoopConfig { 5354 fd: u32, 5355 block_size: u32, 5356 info: LoopInfo64, 5357 _reserved: [u64; 8], 5358 } 5359 5360 // Open loop-control device 5361 let loop_ctl_file = OpenOptions::new() 5362 .read(true) 5363 .write(true) 5364 .open(LOOP_CTL_PATH) 5365 .unwrap(); 5366 5367 // Request a free loop device 5368 let loop_device_number = 5369 unsafe { libc::ioctl(loop_ctl_file.as_raw_fd(), LOOP_CTL_GET_FREE as _) }; 5370 5371 if loop_device_number < 0 { 5372 panic!("Couldn't find a free loop device"); 5373 } 5374 5375 // Create loop device path 5376 let loop_device_path = format!("{LOOP_DEVICE_PREFIX}{loop_device_number}"); 5377 5378 // Open loop device 5379 let loop_device_file = OpenOptions::new() 5380 .read(true) 5381 .write(true) 5382 .open(&loop_device_path) 5383 .unwrap(); 5384 5385 // Open backing file 5386 let backing_file = OpenOptions::new() 5387 .read(true) 5388 .write(true) 5389 .open(backing_file_path) 5390 .unwrap(); 5391 5392 let loop_config = LoopConfig { 5393 fd: backing_file.as_raw_fd() as u32, 5394 block_size, 5395 ..Default::default() 5396 }; 5397 5398 for i in 0..num_retries { 5399 let ret = unsafe { 5400 libc::ioctl( 5401 loop_device_file.as_raw_fd(), 5402 LOOP_CONFIGURE as _, 5403 &loop_config, 5404 ) 5405 }; 5406 if ret != 0 { 5407 if i < num_retries - 1 { 5408 println!( 5409 "Iteration {}: Failed to configure the loop device {}: {}", 5410 i, 5411 loop_device_path, 5412 std::io::Error::last_os_error() 5413 ); 5414 } else { 5415 panic!( 5416 "Failed {} times trying to configure the loop device {}: {}", 5417 num_retries, 5418 loop_device_path, 5419 std::io::Error::last_os_error() 5420 ); 5421 } 5422 } else { 5423 break; 5424 } 5425 5426 // Wait for a bit before retrying 5427 thread::sleep(std::time::Duration::new(5, 0)); 5428 } 5429 5430 loop_device_path 5431 } 5432 5433 #[test] 5434 fn test_virtio_block_topology() { 5435 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5436 let guest = Guest::new(Box::new(focal)); 5437 5438 let kernel_path = direct_kernel_boot_path(); 5439 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5440 5441 let output = exec_host_command_output( 5442 format!( 5443 "qemu-img create -f raw {} 16M", 5444 test_disk_path.to_str().unwrap() 5445 ) 5446 .as_str(), 5447 ); 5448 if !output.status.success() { 5449 let stdout = String::from_utf8_lossy(&output.stdout); 5450 let stderr = String::from_utf8_lossy(&output.stderr); 5451 panic!("qemu-img command failed\nstdout\n{stdout}\nstderr\n{stderr}"); 5452 } 5453 5454 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5455 5456 let mut child = GuestCommand::new(&guest) 5457 .args(["--cpus", "boot=1"]) 5458 .args(["--memory", "size=512M"]) 5459 .args(["--kernel", kernel_path.to_str().unwrap()]) 5460 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5461 .args([ 5462 "--disk", 5463 format!( 5464 "path={}", 5465 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5466 ) 5467 .as_str(), 5468 format!( 5469 "path={}", 5470 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5471 ) 5472 .as_str(), 5473 format!("path={}", &loop_dev).as_str(), 5474 ]) 5475 .default_net() 5476 .capture_output() 5477 .spawn() 5478 .unwrap(); 5479 5480 let r = std::panic::catch_unwind(|| { 5481 guest.wait_vm_boot(None).unwrap(); 5482 5483 // MIN-IO column 5484 assert_eq!( 5485 guest 5486 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5487 .unwrap() 5488 .trim() 5489 .parse::<u32>() 5490 .unwrap_or_default(), 5491 4096 5492 ); 5493 // PHY-SEC column 5494 assert_eq!( 5495 guest 5496 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5497 .unwrap() 5498 .trim() 5499 .parse::<u32>() 5500 .unwrap_or_default(), 5501 4096 5502 ); 5503 // LOG-SEC column 5504 assert_eq!( 5505 guest 5506 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5507 .unwrap() 5508 .trim() 5509 .parse::<u32>() 5510 .unwrap_or_default(), 5511 4096 5512 ); 5513 }); 5514 5515 let _ = child.kill(); 5516 let output = child.wait_with_output().unwrap(); 5517 5518 handle_child_output(r, &output); 5519 5520 Command::new("losetup") 5521 .args(["-d", &loop_dev]) 5522 .output() 5523 .expect("loop device not found"); 5524 } 5525 5526 #[test] 5527 fn test_virtio_balloon_deflate_on_oom() { 5528 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5529 let guest = Guest::new(Box::new(focal)); 5530 5531 let kernel_path = direct_kernel_boot_path(); 5532 5533 let api_socket = temp_api_path(&guest.tmp_dir); 5534 5535 //Let's start a 4G guest with balloon occupied 2G memory 5536 let mut child = GuestCommand::new(&guest) 5537 .args(["--api-socket", &api_socket]) 5538 .args(["--cpus", "boot=1"]) 5539 .args(["--memory", "size=4G"]) 5540 .args(["--kernel", kernel_path.to_str().unwrap()]) 5541 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5542 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5543 .default_disks() 5544 .default_net() 5545 .capture_output() 5546 .spawn() 5547 .unwrap(); 5548 5549 let r = std::panic::catch_unwind(|| { 5550 guest.wait_vm_boot(None).unwrap(); 5551 5552 // Wait for balloon memory's initialization and check its size. 5553 // The virtio-balloon driver might take a few seconds to report the 5554 // balloon effective size back to the VMM. 5555 thread::sleep(std::time::Duration::new(20, 0)); 5556 5557 let orig_balloon = balloon_size(&api_socket); 5558 println!("The original balloon memory size is {orig_balloon} bytes"); 5559 assert!(orig_balloon == 2147483648); 5560 5561 // Two steps to verify if the 'deflate_on_oom' parameter works. 5562 // 1st: run a command to trigger an OOM in the guest. 5563 guest 5564 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5565 .unwrap(); 5566 5567 // Give some time for the OOM to happen in the guest and be reported 5568 // back to the host. 5569 thread::sleep(std::time::Duration::new(20, 0)); 5570 5571 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5572 let deflated_balloon = balloon_size(&api_socket); 5573 println!("After deflating, balloon memory size is {deflated_balloon} bytes"); 5574 // Verify the balloon size deflated 5575 assert!(deflated_balloon < 2147483648); 5576 }); 5577 5578 let _ = child.kill(); 5579 let output = child.wait_with_output().unwrap(); 5580 5581 handle_child_output(r, &output); 5582 } 5583 5584 #[test] 5585 #[cfg(not(feature = "mshv"))] 5586 fn test_virtio_balloon_free_page_reporting() { 5587 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5588 let guest = Guest::new(Box::new(focal)); 5589 5590 //Let's start a 4G guest with balloon occupied 2G memory 5591 let mut child = GuestCommand::new(&guest) 5592 .args(["--cpus", "boot=1"]) 5593 .args(["--memory", "size=4G"]) 5594 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5595 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5596 .args(["--balloon", "size=0,free_page_reporting=on"]) 5597 .default_disks() 5598 .default_net() 5599 .capture_output() 5600 .spawn() 5601 .unwrap(); 5602 5603 let pid = child.id(); 5604 let r = std::panic::catch_unwind(|| { 5605 guest.wait_vm_boot(None).unwrap(); 5606 5607 // Check the initial RSS is less than 1GiB 5608 let rss = process_rss_kib(pid); 5609 println!("RSS {rss} < 1048576"); 5610 assert!(rss < 1048576); 5611 5612 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5613 // seconds 5614 let guest_ip = guest.network.guest_ip.clone(); 5615 thread::spawn(move || { 5616 ssh_command_ip( 5617 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5618 &guest_ip, 5619 DEFAULT_SSH_RETRIES, 5620 DEFAULT_SSH_TIMEOUT, 5621 ) 5622 .unwrap(); 5623 }); 5624 5625 // Wait for 50 seconds to make sure the stress command is consuming 5626 // the expected amount of memory. 5627 thread::sleep(std::time::Duration::new(50, 0)); 5628 let rss = process_rss_kib(pid); 5629 println!("RSS {rss} >= 2097152"); 5630 assert!(rss >= 2097152); 5631 5632 // Wait for an extra minute to make sure the stress command has 5633 // completed and that the guest reported the free pages to the VMM 5634 // through the virtio-balloon device. We expect the RSS to be under 5635 // 2GiB. 5636 thread::sleep(std::time::Duration::new(60, 0)); 5637 let rss = process_rss_kib(pid); 5638 println!("RSS {rss} < 2097152"); 5639 assert!(rss < 2097152); 5640 }); 5641 5642 let _ = child.kill(); 5643 let output = child.wait_with_output().unwrap(); 5644 5645 handle_child_output(r, &output); 5646 } 5647 5648 #[test] 5649 fn test_pmem_hotplug() { 5650 _test_pmem_hotplug(None) 5651 } 5652 5653 #[test] 5654 fn test_pmem_multi_segment_hotplug() { 5655 _test_pmem_hotplug(Some(15)) 5656 } 5657 5658 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5659 #[cfg(target_arch = "aarch64")] 5660 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5661 #[cfg(target_arch = "x86_64")] 5662 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5663 let focal = UbuntuDiskConfig::new(focal_image); 5664 let guest = Guest::new(Box::new(focal)); 5665 5666 #[cfg(target_arch = "x86_64")] 5667 let kernel_path = direct_kernel_boot_path(); 5668 #[cfg(target_arch = "aarch64")] 5669 let kernel_path = edk2_path(); 5670 5671 let api_socket = temp_api_path(&guest.tmp_dir); 5672 5673 let mut cmd = GuestCommand::new(&guest); 5674 5675 cmd.args(["--api-socket", &api_socket]) 5676 .args(["--cpus", "boot=1"]) 5677 .args(["--memory", "size=512M"]) 5678 .args(["--kernel", kernel_path.to_str().unwrap()]) 5679 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5680 .default_disks() 5681 .default_net() 5682 .capture_output(); 5683 5684 if pci_segment.is_some() { 5685 cmd.args([ 5686 "--platform", 5687 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5688 ]); 5689 } 5690 5691 let mut child = cmd.spawn().unwrap(); 5692 5693 let r = std::panic::catch_unwind(|| { 5694 guest.wait_vm_boot(None).unwrap(); 5695 5696 // Check /dev/pmem0 is not there 5697 assert_eq!( 5698 guest 5699 .ssh_command("lsblk | grep -c pmem0 || true") 5700 .unwrap() 5701 .trim() 5702 .parse::<u32>() 5703 .unwrap_or(1), 5704 0 5705 ); 5706 5707 let pmem_temp_file = TempFile::new().unwrap(); 5708 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5709 let (cmd_success, cmd_output) = remote_command_w_output( 5710 &api_socket, 5711 "add-pmem", 5712 Some(&format!( 5713 "file={},id=test0{}", 5714 pmem_temp_file.as_path().to_str().unwrap(), 5715 if let Some(pci_segment) = pci_segment { 5716 format!(",pci_segment={pci_segment}") 5717 } else { 5718 "".to_owned() 5719 } 5720 )), 5721 ); 5722 assert!(cmd_success); 5723 if let Some(pci_segment) = pci_segment { 5724 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5725 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5726 ))); 5727 } else { 5728 assert!(String::from_utf8_lossy(&cmd_output) 5729 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5730 } 5731 5732 // Check that /dev/pmem0 exists and the block size is 128M 5733 assert_eq!( 5734 guest 5735 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5736 .unwrap() 5737 .trim() 5738 .parse::<u32>() 5739 .unwrap_or_default(), 5740 1 5741 ); 5742 5743 guest.reboot_linux(0, None); 5744 5745 // Check still there after reboot 5746 assert_eq!( 5747 guest 5748 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5749 .unwrap() 5750 .trim() 5751 .parse::<u32>() 5752 .unwrap_or_default(), 5753 1 5754 ); 5755 5756 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5757 5758 thread::sleep(std::time::Duration::new(20, 0)); 5759 5760 // Check device has gone away 5761 assert_eq!( 5762 guest 5763 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5764 .unwrap() 5765 .trim() 5766 .parse::<u32>() 5767 .unwrap_or(1), 5768 0 5769 ); 5770 5771 guest.reboot_linux(1, None); 5772 5773 // Check still absent after reboot 5774 assert_eq!( 5775 guest 5776 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5777 .unwrap() 5778 .trim() 5779 .parse::<u32>() 5780 .unwrap_or(1), 5781 0 5782 ); 5783 }); 5784 5785 let _ = child.kill(); 5786 let output = child.wait_with_output().unwrap(); 5787 5788 handle_child_output(r, &output); 5789 } 5790 5791 #[test] 5792 fn test_net_hotplug() { 5793 _test_net_hotplug(None) 5794 } 5795 5796 #[test] 5797 fn test_net_multi_segment_hotplug() { 5798 _test_net_hotplug(Some(15)) 5799 } 5800 5801 fn _test_net_hotplug(pci_segment: Option<u16>) { 5802 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5803 let guest = Guest::new(Box::new(focal)); 5804 5805 #[cfg(target_arch = "x86_64")] 5806 let kernel_path = direct_kernel_boot_path(); 5807 #[cfg(target_arch = "aarch64")] 5808 let kernel_path = edk2_path(); 5809 5810 let api_socket = temp_api_path(&guest.tmp_dir); 5811 5812 // Boot without network 5813 let mut cmd = GuestCommand::new(&guest); 5814 5815 cmd.args(["--api-socket", &api_socket]) 5816 .args(["--cpus", "boot=1"]) 5817 .args(["--memory", "size=512M"]) 5818 .args(["--kernel", kernel_path.to_str().unwrap()]) 5819 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5820 .default_disks() 5821 .capture_output(); 5822 5823 if pci_segment.is_some() { 5824 cmd.args([ 5825 "--platform", 5826 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5827 ]); 5828 } 5829 5830 let mut child = cmd.spawn().unwrap(); 5831 5832 thread::sleep(std::time::Duration::new(20, 0)); 5833 5834 let r = std::panic::catch_unwind(|| { 5835 // Add network 5836 let (cmd_success, cmd_output) = remote_command_w_output( 5837 &api_socket, 5838 "add-net", 5839 Some( 5840 format!( 5841 "{}{},id=test0", 5842 guest.default_net_string(), 5843 if let Some(pci_segment) = pci_segment { 5844 format!(",pci_segment={pci_segment}") 5845 } else { 5846 "".to_owned() 5847 } 5848 ) 5849 .as_str(), 5850 ), 5851 ); 5852 assert!(cmd_success); 5853 5854 if let Some(pci_segment) = pci_segment { 5855 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5856 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5857 ))); 5858 } else { 5859 assert!(String::from_utf8_lossy(&cmd_output) 5860 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5861 } 5862 5863 thread::sleep(std::time::Duration::new(5, 0)); 5864 5865 // 1 network interfaces + default localhost ==> 2 interfaces 5866 assert_eq!( 5867 guest 5868 .ssh_command("ip -o link | wc -l") 5869 .unwrap() 5870 .trim() 5871 .parse::<u32>() 5872 .unwrap_or_default(), 5873 2 5874 ); 5875 5876 // Remove network 5877 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5878 thread::sleep(std::time::Duration::new(5, 0)); 5879 5880 let (cmd_success, cmd_output) = remote_command_w_output( 5881 &api_socket, 5882 "add-net", 5883 Some( 5884 format!( 5885 "{}{},id=test1", 5886 guest.default_net_string(), 5887 if let Some(pci_segment) = pci_segment { 5888 format!(",pci_segment={pci_segment}") 5889 } else { 5890 "".to_owned() 5891 } 5892 ) 5893 .as_str(), 5894 ), 5895 ); 5896 assert!(cmd_success); 5897 5898 if let Some(pci_segment) = pci_segment { 5899 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5900 "{{\"id\":\"test1\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5901 ))); 5902 } else { 5903 assert!(String::from_utf8_lossy(&cmd_output) 5904 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5905 } 5906 5907 thread::sleep(std::time::Duration::new(5, 0)); 5908 5909 // 1 network interfaces + default localhost ==> 2 interfaces 5910 assert_eq!( 5911 guest 5912 .ssh_command("ip -o link | wc -l") 5913 .unwrap() 5914 .trim() 5915 .parse::<u32>() 5916 .unwrap_or_default(), 5917 2 5918 ); 5919 5920 guest.reboot_linux(0, None); 5921 5922 // Check still there after reboot 5923 // 1 network interfaces + default localhost ==> 2 interfaces 5924 assert_eq!( 5925 guest 5926 .ssh_command("ip -o link | wc -l") 5927 .unwrap() 5928 .trim() 5929 .parse::<u32>() 5930 .unwrap_or_default(), 5931 2 5932 ); 5933 }); 5934 5935 let _ = child.kill(); 5936 let output = child.wait_with_output().unwrap(); 5937 5938 handle_child_output(r, &output); 5939 } 5940 5941 #[test] 5942 fn test_initramfs() { 5943 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5944 let guest = Guest::new(Box::new(focal)); 5945 let mut workload_path = dirs::home_dir().unwrap(); 5946 workload_path.push("workloads"); 5947 5948 #[cfg(target_arch = "x86_64")] 5949 let mut kernels = vec![direct_kernel_boot_path()]; 5950 #[cfg(target_arch = "aarch64")] 5951 let kernels = [direct_kernel_boot_path()]; 5952 5953 #[cfg(target_arch = "x86_64")] 5954 { 5955 let mut pvh_kernel_path = workload_path.clone(); 5956 pvh_kernel_path.push("vmlinux"); 5957 kernels.push(pvh_kernel_path); 5958 } 5959 5960 let mut initramfs_path = workload_path; 5961 initramfs_path.push("alpine_initramfs.img"); 5962 5963 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5964 let cmdline = format!("console=hvc0 quiet TEST_STRING={test_string}"); 5965 5966 kernels.iter().for_each(|k_path| { 5967 let mut child = GuestCommand::new(&guest) 5968 .args(["--kernel", k_path.to_str().unwrap()]) 5969 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 5970 .args(["--cmdline", &cmdline]) 5971 .capture_output() 5972 .spawn() 5973 .unwrap(); 5974 5975 thread::sleep(std::time::Duration::new(20, 0)); 5976 5977 let _ = child.kill(); 5978 let output = child.wait_with_output().unwrap(); 5979 5980 let r = std::panic::catch_unwind(|| { 5981 let s = String::from_utf8_lossy(&output.stdout); 5982 5983 assert_ne!(s.lines().position(|line| line == test_string), None); 5984 }); 5985 5986 handle_child_output(r, &output); 5987 }); 5988 } 5989 5990 // One thing to note about this test. The virtio-net device is heavily used 5991 // through each ssh command. There's no need to perform a dedicated test to 5992 // verify the migration went well for virtio-net. 5993 #[test] 5994 #[cfg(not(feature = "mshv"))] 5995 fn test_snapshot_restore_hotplug_virtiomem() { 5996 _test_snapshot_restore(true); 5997 } 5998 5999 #[test] 6000 fn test_snapshot_restore_basic() { 6001 _test_snapshot_restore(false); 6002 } 6003 6004 fn _test_snapshot_restore(use_hotplug: bool) { 6005 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6006 let guest = Guest::new(Box::new(focal)); 6007 let kernel_path = direct_kernel_boot_path(); 6008 6009 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 6010 6011 let net_id = "net123"; 6012 let net_params = format!( 6013 "id={},tap=,mac={},ip={},mask=255.255.255.0", 6014 net_id, guest.network.guest_mac, guest.network.host_ip 6015 ); 6016 let mut mem_params = "size=4G"; 6017 6018 if use_hotplug { 6019 mem_params = "size=4G,hotplug_method=virtio-mem,hotplug_size=32G" 6020 } 6021 6022 let cloudinit_params = format!( 6023 "path={},iommu=on", 6024 guest.disk_config.disk(DiskType::CloudInit).unwrap() 6025 ); 6026 6027 let socket = temp_vsock_path(&guest.tmp_dir); 6028 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6029 6030 let mut child = GuestCommand::new(&guest) 6031 .args(["--api-socket", &api_socket_source]) 6032 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6033 .args(["--cpus", "boot=4"]) 6034 .args(["--memory", mem_params]) 6035 .args(["--balloon", "size=0"]) 6036 .args(["--kernel", kernel_path.to_str().unwrap()]) 6037 .args([ 6038 "--disk", 6039 format!( 6040 "path={}", 6041 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 6042 ) 6043 .as_str(), 6044 cloudinit_params.as_str(), 6045 ]) 6046 .args(["--net", net_params.as_str()]) 6047 .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) 6048 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6049 .capture_output() 6050 .spawn() 6051 .unwrap(); 6052 6053 let console_text = String::from("On a branch floating down river a cricket, singing."); 6054 // Create the snapshot directory 6055 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 6056 6057 let r = std::panic::catch_unwind(|| { 6058 guest.wait_vm_boot(None).unwrap(); 6059 6060 // Check the number of vCPUs 6061 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 6062 // Check the guest RAM 6063 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6064 if use_hotplug { 6065 // Increase guest RAM with virtio-mem 6066 resize_command( 6067 &api_socket_source, 6068 None, 6069 Some(6 << 30), 6070 None, 6071 Some(&event_path), 6072 ); 6073 thread::sleep(std::time::Duration::new(5, 0)); 6074 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6075 // Use balloon to remove RAM from the VM 6076 resize_command( 6077 &api_socket_source, 6078 None, 6079 None, 6080 Some(1 << 30), 6081 Some(&event_path), 6082 ); 6083 thread::sleep(std::time::Duration::new(5, 0)); 6084 let total_memory = guest.get_total_memory().unwrap_or_default(); 6085 assert!(total_memory > 4_800_000); 6086 assert!(total_memory < 5_760_000); 6087 } 6088 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 6089 guest.check_devices_common(Some(&socket), Some(&console_text), None); 6090 6091 // x86_64: We check that removing and adding back the virtio-net device 6092 // does not break the snapshot/restore support for virtio-pci. 6093 // This is an important thing to test as the hotplug will 6094 // trigger a PCI BAR reprogramming, which is a good way of 6095 // checking if the stored resources are correctly restored. 6096 // Unplug the virtio-net device 6097 // AArch64: Device hotplug is currently not supported, skipping here. 6098 #[cfg(target_arch = "x86_64")] 6099 { 6100 assert!(remote_command( 6101 &api_socket_source, 6102 "remove-device", 6103 Some(net_id), 6104 )); 6105 thread::sleep(std::time::Duration::new(10, 0)); 6106 let latest_events = [&MetaEvent { 6107 event: "device-removed".to_string(), 6108 device_id: Some(net_id.to_string()), 6109 }]; 6110 // See: #5938 6111 thread::sleep(std::time::Duration::new(1, 0)); 6112 assert!(check_latest_events_exact(&latest_events, &event_path)); 6113 6114 // Plug the virtio-net device again 6115 assert!(remote_command( 6116 &api_socket_source, 6117 "add-net", 6118 Some(net_params.as_str()), 6119 )); 6120 thread::sleep(std::time::Duration::new(10, 0)); 6121 } 6122 6123 // Pause the VM 6124 assert!(remote_command(&api_socket_source, "pause", None)); 6125 let latest_events = [ 6126 &MetaEvent { 6127 event: "pausing".to_string(), 6128 device_id: None, 6129 }, 6130 &MetaEvent { 6131 event: "paused".to_string(), 6132 device_id: None, 6133 }, 6134 ]; 6135 // See: #5938 6136 thread::sleep(std::time::Duration::new(1, 0)); 6137 assert!(check_latest_events_exact(&latest_events, &event_path)); 6138 6139 // Take a snapshot from the VM 6140 assert!(remote_command( 6141 &api_socket_source, 6142 "snapshot", 6143 Some(format!("file://{snapshot_dir}").as_str()), 6144 )); 6145 6146 // Wait to make sure the snapshot is completed 6147 thread::sleep(std::time::Duration::new(10, 0)); 6148 6149 let latest_events = [ 6150 &MetaEvent { 6151 event: "snapshotting".to_string(), 6152 device_id: None, 6153 }, 6154 &MetaEvent { 6155 event: "snapshotted".to_string(), 6156 device_id: None, 6157 }, 6158 ]; 6159 // See: #5938 6160 thread::sleep(std::time::Duration::new(1, 0)); 6161 assert!(check_latest_events_exact(&latest_events, &event_path)); 6162 }); 6163 6164 // Shutdown the source VM and check console output 6165 let _ = child.kill(); 6166 let output = child.wait_with_output().unwrap(); 6167 handle_child_output(r, &output); 6168 6169 let r = std::panic::catch_unwind(|| { 6170 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 6171 }); 6172 6173 handle_child_output(r, &output); 6174 6175 // Remove the vsock socket file. 6176 Command::new("rm") 6177 .arg("-f") 6178 .arg(socket.as_str()) 6179 .output() 6180 .unwrap(); 6181 6182 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 6183 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 6184 6185 // Restore the VM from the snapshot 6186 let mut child = GuestCommand::new(&guest) 6187 .args(["--api-socket", &api_socket_restored]) 6188 .args([ 6189 "--event-monitor", 6190 format!("path={event_path_restored}").as_str(), 6191 ]) 6192 .args([ 6193 "--restore", 6194 format!("source_url=file://{snapshot_dir}").as_str(), 6195 ]) 6196 .capture_output() 6197 .spawn() 6198 .unwrap(); 6199 6200 // Wait for the VM to be restored 6201 thread::sleep(std::time::Duration::new(10, 0)); 6202 let expected_events = [ 6203 &MetaEvent { 6204 event: "starting".to_string(), 6205 device_id: None, 6206 }, 6207 &MetaEvent { 6208 event: "activated".to_string(), 6209 device_id: Some("__console".to_string()), 6210 }, 6211 &MetaEvent { 6212 event: "activated".to_string(), 6213 device_id: Some("__rng".to_string()), 6214 }, 6215 &MetaEvent { 6216 event: "restoring".to_string(), 6217 device_id: None, 6218 }, 6219 ]; 6220 assert!(check_sequential_events( 6221 &expected_events, 6222 &event_path_restored 6223 )); 6224 let latest_events = [&MetaEvent { 6225 event: "restored".to_string(), 6226 device_id: None, 6227 }]; 6228 assert!(check_latest_events_exact( 6229 &latest_events, 6230 &event_path_restored 6231 )); 6232 6233 let r = std::panic::catch_unwind(|| { 6234 // Resume the VM 6235 assert!(remote_command(&api_socket_restored, "resume", None)); 6236 // There is no way that we can ensure the 'write()' to the 6237 // event file is completed when the 'resume' request is 6238 // returned successfully, because the 'write()' was done 6239 // asynchronously from a different thread of Cloud 6240 // Hypervisor (e.g. the event-monitor thread). 6241 thread::sleep(std::time::Duration::new(1, 0)); 6242 let latest_events = [ 6243 &MetaEvent { 6244 event: "resuming".to_string(), 6245 device_id: None, 6246 }, 6247 &MetaEvent { 6248 event: "resumed".to_string(), 6249 device_id: None, 6250 }, 6251 ]; 6252 assert!(check_latest_events_exact( 6253 &latest_events, 6254 &event_path_restored 6255 )); 6256 6257 // Perform same checks to validate VM has been properly restored 6258 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 6259 let total_memory = guest.get_total_memory().unwrap_or_default(); 6260 if !use_hotplug { 6261 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6262 } else { 6263 assert!(total_memory > 4_800_000); 6264 assert!(total_memory < 5_760_000); 6265 // Deflate balloon to restore entire RAM to the VM 6266 resize_command(&api_socket_restored, None, None, Some(0), None); 6267 thread::sleep(std::time::Duration::new(5, 0)); 6268 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6269 // Decrease guest RAM with virtio-mem 6270 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 6271 thread::sleep(std::time::Duration::new(5, 0)); 6272 let total_memory = guest.get_total_memory().unwrap_or_default(); 6273 assert!(total_memory > 4_800_000); 6274 assert!(total_memory < 5_760_000); 6275 } 6276 6277 guest.check_devices_common(Some(&socket), Some(&console_text), None); 6278 }); 6279 // Shutdown the target VM and check console output 6280 let _ = child.kill(); 6281 let output = child.wait_with_output().unwrap(); 6282 handle_child_output(r, &output); 6283 6284 let r = std::panic::catch_unwind(|| { 6285 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 6286 }); 6287 6288 handle_child_output(r, &output); 6289 } 6290 6291 #[test] 6292 fn test_counters() { 6293 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6294 let guest = Guest::new(Box::new(focal)); 6295 let api_socket = temp_api_path(&guest.tmp_dir); 6296 6297 let mut cmd = GuestCommand::new(&guest); 6298 cmd.args(["--cpus", "boot=1"]) 6299 .args(["--memory", "size=512M"]) 6300 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 6301 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6302 .default_disks() 6303 .args(["--net", guest.default_net_string().as_str()]) 6304 .args(["--api-socket", &api_socket]) 6305 .capture_output(); 6306 6307 let mut child = cmd.spawn().unwrap(); 6308 6309 let r = std::panic::catch_unwind(|| { 6310 guest.wait_vm_boot(None).unwrap(); 6311 6312 let orig_counters = get_counters(&api_socket); 6313 guest 6314 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 6315 .unwrap(); 6316 6317 let new_counters = get_counters(&api_socket); 6318 6319 // Check that all the counters have increased 6320 assert!(new_counters > orig_counters); 6321 }); 6322 6323 let _ = child.kill(); 6324 let output = child.wait_with_output().unwrap(); 6325 6326 handle_child_output(r, &output); 6327 } 6328 6329 #[test] 6330 #[cfg(feature = "guest_debug")] 6331 fn test_coredump() { 6332 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6333 let guest = Guest::new(Box::new(focal)); 6334 let api_socket = temp_api_path(&guest.tmp_dir); 6335 6336 let mut cmd = GuestCommand::new(&guest); 6337 cmd.args(["--cpus", "boot=4"]) 6338 .args(["--memory", "size=4G"]) 6339 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6340 .default_disks() 6341 .args(["--net", guest.default_net_string().as_str()]) 6342 .args(["--api-socket", &api_socket]) 6343 .capture_output(); 6344 6345 let mut child = cmd.spawn().unwrap(); 6346 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6347 6348 let r = std::panic::catch_unwind(|| { 6349 guest.wait_vm_boot(None).unwrap(); 6350 6351 assert!(remote_command(&api_socket, "pause", None)); 6352 6353 assert!(remote_command( 6354 &api_socket, 6355 "coredump", 6356 Some(format!("file://{vmcore_file}").as_str()), 6357 )); 6358 6359 // the num of CORE notes should equals to vcpu 6360 let readelf_core_num_cmd = 6361 format!("readelf --all {vmcore_file} |grep CORE |grep -v Type |wc -l"); 6362 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 6363 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 6364 6365 // the num of QEMU notes should equals to vcpu 6366 let readelf_vmm_num_cmd = format!("readelf --all {vmcore_file} |grep QEMU |wc -l"); 6367 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 6368 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 6369 }); 6370 6371 let _ = child.kill(); 6372 let output = child.wait_with_output().unwrap(); 6373 6374 handle_child_output(r, &output); 6375 } 6376 6377 #[test] 6378 #[cfg(feature = "guest_debug")] 6379 fn test_coredump_no_pause() { 6380 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6381 let guest = Guest::new(Box::new(focal)); 6382 let api_socket = temp_api_path(&guest.tmp_dir); 6383 6384 let mut cmd = GuestCommand::new(&guest); 6385 cmd.args(["--cpus", "boot=4"]) 6386 .args(["--memory", "size=4G"]) 6387 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6388 .default_disks() 6389 .args(["--net", guest.default_net_string().as_str()]) 6390 .args(["--api-socket", &api_socket]) 6391 .capture_output(); 6392 6393 let mut child = cmd.spawn().unwrap(); 6394 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6395 6396 let r = std::panic::catch_unwind(|| { 6397 guest.wait_vm_boot(None).unwrap(); 6398 6399 assert!(remote_command( 6400 &api_socket, 6401 "coredump", 6402 Some(format!("file://{vmcore_file}").as_str()), 6403 )); 6404 6405 assert_eq!(vm_state(&api_socket), "Running"); 6406 }); 6407 6408 let _ = child.kill(); 6409 let output = child.wait_with_output().unwrap(); 6410 6411 handle_child_output(r, &output); 6412 } 6413 6414 #[test] 6415 fn test_watchdog() { 6416 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6417 let guest = Guest::new(Box::new(focal)); 6418 let api_socket = temp_api_path(&guest.tmp_dir); 6419 6420 let kernel_path = direct_kernel_boot_path(); 6421 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6422 6423 let mut cmd = GuestCommand::new(&guest); 6424 cmd.args(["--cpus", "boot=1"]) 6425 .args(["--memory", "size=512M"]) 6426 .args(["--kernel", kernel_path.to_str().unwrap()]) 6427 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6428 .default_disks() 6429 .args(["--net", guest.default_net_string().as_str()]) 6430 .args(["--watchdog"]) 6431 .args(["--api-socket", &api_socket]) 6432 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6433 .capture_output(); 6434 6435 let mut child = cmd.spawn().unwrap(); 6436 6437 let r = std::panic::catch_unwind(|| { 6438 guest.wait_vm_boot(None).unwrap(); 6439 6440 let mut expected_reboot_count = 1; 6441 6442 // Enable the watchdog with a 15s timeout 6443 enable_guest_watchdog(&guest, 15); 6444 6445 // Reboot and check that systemd has activated the watchdog 6446 guest.ssh_command("sudo reboot").unwrap(); 6447 guest.wait_vm_boot(None).unwrap(); 6448 expected_reboot_count += 1; 6449 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6450 assert_eq!( 6451 guest 6452 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6453 .unwrap() 6454 .trim() 6455 .parse::<u32>() 6456 .unwrap_or_default(), 6457 2 6458 ); 6459 6460 // Allow some normal time to elapse to check we don't get spurious reboots 6461 thread::sleep(std::time::Duration::new(40, 0)); 6462 // Check no reboot 6463 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6464 6465 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6466 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6467 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6468 guest.wait_vm_boot(Some(50)).unwrap(); 6469 // Check a reboot is triggered by the watchdog 6470 expected_reboot_count += 1; 6471 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6472 6473 #[cfg(target_arch = "x86_64")] 6474 { 6475 // Now pause the VM and remain offline for 30s 6476 assert!(remote_command(&api_socket, "pause", None)); 6477 let latest_events = [ 6478 &MetaEvent { 6479 event: "pausing".to_string(), 6480 device_id: None, 6481 }, 6482 &MetaEvent { 6483 event: "paused".to_string(), 6484 device_id: None, 6485 }, 6486 ]; 6487 assert!(check_latest_events_exact(&latest_events, &event_path)); 6488 assert!(remote_command(&api_socket, "resume", None)); 6489 6490 // Check no reboot 6491 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6492 } 6493 }); 6494 6495 let _ = child.kill(); 6496 let output = child.wait_with_output().unwrap(); 6497 6498 handle_child_output(r, &output); 6499 } 6500 6501 #[test] 6502 fn test_pvpanic() { 6503 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 6504 let guest = Guest::new(Box::new(jammy)); 6505 let api_socket = temp_api_path(&guest.tmp_dir); 6506 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6507 6508 let kernel_path = direct_kernel_boot_path(); 6509 6510 let mut cmd = GuestCommand::new(&guest); 6511 cmd.args(["--cpus", "boot=1"]) 6512 .args(["--memory", "size=512M"]) 6513 .args(["--kernel", kernel_path.to_str().unwrap()]) 6514 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6515 .default_disks() 6516 .args(["--net", guest.default_net_string().as_str()]) 6517 .args(["--pvpanic"]) 6518 .args(["--api-socket", &api_socket]) 6519 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6520 .capture_output(); 6521 6522 let mut child = cmd.spawn().unwrap(); 6523 6524 let r = std::panic::catch_unwind(|| { 6525 guest.wait_vm_boot(None).unwrap(); 6526 6527 // Trigger guest a panic 6528 make_guest_panic(&guest); 6529 6530 // Wait a while for guest 6531 thread::sleep(std::time::Duration::new(10, 0)); 6532 6533 let expected_sequential_events = [&MetaEvent { 6534 event: "panic".to_string(), 6535 device_id: None, 6536 }]; 6537 assert!(check_latest_events_exact( 6538 &expected_sequential_events, 6539 &event_path 6540 )); 6541 }); 6542 6543 let _ = child.kill(); 6544 let output = child.wait_with_output().unwrap(); 6545 6546 handle_child_output(r, &output); 6547 } 6548 6549 #[test] 6550 fn test_tap_from_fd() { 6551 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6552 let guest = Guest::new(Box::new(focal)); 6553 let kernel_path = direct_kernel_boot_path(); 6554 6555 // Create a TAP interface with multi-queue enabled 6556 let num_queue_pairs: usize = 2; 6557 6558 use std::str::FromStr; 6559 let taps = net_util::open_tap( 6560 Some("chtap0"), 6561 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6562 None, 6563 &mut None, 6564 None, 6565 num_queue_pairs, 6566 Some(libc::O_RDWR | libc::O_NONBLOCK), 6567 ) 6568 .unwrap(); 6569 6570 let mut child = GuestCommand::new(&guest) 6571 .args(["--cpus", &format!("boot={num_queue_pairs}")]) 6572 .args(["--memory", "size=512M"]) 6573 .args(["--kernel", kernel_path.to_str().unwrap()]) 6574 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6575 .default_disks() 6576 .args([ 6577 "--net", 6578 &format!( 6579 "fd=[{},{}],mac={},num_queues={}", 6580 taps[0].as_raw_fd(), 6581 taps[1].as_raw_fd(), 6582 guest.network.guest_mac, 6583 num_queue_pairs * 2 6584 ), 6585 ]) 6586 .capture_output() 6587 .spawn() 6588 .unwrap(); 6589 6590 let r = std::panic::catch_unwind(|| { 6591 guest.wait_vm_boot(None).unwrap(); 6592 6593 assert_eq!( 6594 guest 6595 .ssh_command("ip -o link | wc -l") 6596 .unwrap() 6597 .trim() 6598 .parse::<u32>() 6599 .unwrap_or_default(), 6600 2 6601 ); 6602 6603 guest.reboot_linux(0, None); 6604 6605 assert_eq!( 6606 guest 6607 .ssh_command("ip -o link | wc -l") 6608 .unwrap() 6609 .trim() 6610 .parse::<u32>() 6611 .unwrap_or_default(), 6612 2 6613 ); 6614 }); 6615 6616 let _ = child.kill(); 6617 let output = child.wait_with_output().unwrap(); 6618 6619 handle_child_output(r, &output); 6620 } 6621 6622 // By design, a guest VM won't be able to connect to the host 6623 // machine when using a macvtap network interface (while it can 6624 // communicate externally). As a workaround, this integration 6625 // test creates two macvtap interfaces in 'bridge' mode on the 6626 // same physical net interface, one for the guest and one for 6627 // the host. With additional setup on the IP address and the 6628 // routing table, it enables the communications between the 6629 // guest VM and the host machine. 6630 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6631 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6632 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6633 let guest = Guest::new(Box::new(focal)); 6634 let api_socket = temp_api_path(&guest.tmp_dir); 6635 6636 #[cfg(target_arch = "x86_64")] 6637 let kernel_path = direct_kernel_boot_path(); 6638 #[cfg(target_arch = "aarch64")] 6639 let kernel_path = edk2_path(); 6640 6641 let phy_net = "eth0"; 6642 6643 // Create a macvtap interface for the guest VM to use 6644 assert!(exec_host_command_status(&format!( 6645 "sudo ip link add link {phy_net} name {guest_macvtap_name} type macvtap mod bridge" 6646 )) 6647 .success()); 6648 assert!(exec_host_command_status(&format!( 6649 "sudo ip link set {} address {} up", 6650 guest_macvtap_name, guest.network.guest_mac 6651 )) 6652 .success()); 6653 assert!( 6654 exec_host_command_status(&format!("sudo ip link show {guest_macvtap_name}")).success() 6655 ); 6656 6657 let tap_index = 6658 fs::read_to_string(format!("/sys/class/net/{guest_macvtap_name}/ifindex")).unwrap(); 6659 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6660 6661 assert!(exec_host_command_status(&format!("sudo chown $UID.$UID {tap_device}")).success()); 6662 6663 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6664 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6665 assert!(tap_fd1 > 0); 6666 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6667 assert!(tap_fd2 > 0); 6668 6669 // Create a macvtap on the same physical net interface for 6670 // the host machine to use 6671 assert!(exec_host_command_status(&format!( 6672 "sudo ip link add link {phy_net} name {host_macvtap_name} type macvtap mod bridge" 6673 )) 6674 .success()); 6675 // Use default mask "255.255.255.0" 6676 assert!(exec_host_command_status(&format!( 6677 "sudo ip address add {}/24 dev {}", 6678 guest.network.host_ip, host_macvtap_name 6679 )) 6680 .success()); 6681 assert!( 6682 exec_host_command_status(&format!("sudo ip link set dev {host_macvtap_name} up")) 6683 .success() 6684 ); 6685 6686 let mut guest_command = GuestCommand::new(&guest); 6687 guest_command 6688 .args(["--cpus", "boot=2"]) 6689 .args(["--memory", "size=512M"]) 6690 .args(["--kernel", kernel_path.to_str().unwrap()]) 6691 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6692 .default_disks() 6693 .args(["--api-socket", &api_socket]); 6694 6695 let net_params = format!( 6696 "fd=[{},{}],mac={},num_queues=4", 6697 tap_fd1, tap_fd2, guest.network.guest_mac 6698 ); 6699 6700 if !hotplug { 6701 guest_command.args(["--net", &net_params]); 6702 } 6703 6704 let mut child = guest_command.capture_output().spawn().unwrap(); 6705 6706 if hotplug { 6707 // Give some time to the VMM process to listen to the API 6708 // socket. This is the only requirement to avoid the following 6709 // call to ch-remote from failing. 6710 thread::sleep(std::time::Duration::new(10, 0)); 6711 // Hotplug the virtio-net device 6712 let (cmd_success, cmd_output) = 6713 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6714 assert!(cmd_success); 6715 #[cfg(target_arch = "x86_64")] 6716 assert!(String::from_utf8_lossy(&cmd_output) 6717 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6718 #[cfg(target_arch = "aarch64")] 6719 assert!(String::from_utf8_lossy(&cmd_output) 6720 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6721 } 6722 6723 // The functional connectivity provided by the virtio-net device 6724 // gets tested through wait_vm_boot() as it expects to receive a 6725 // HTTP request, and through the SSH command as well. 6726 let r = std::panic::catch_unwind(|| { 6727 guest.wait_vm_boot(None).unwrap(); 6728 6729 assert_eq!( 6730 guest 6731 .ssh_command("ip -o link | wc -l") 6732 .unwrap() 6733 .trim() 6734 .parse::<u32>() 6735 .unwrap_or_default(), 6736 2 6737 ); 6738 6739 guest.reboot_linux(0, None); 6740 6741 assert_eq!( 6742 guest 6743 .ssh_command("ip -o link | wc -l") 6744 .unwrap() 6745 .trim() 6746 .parse::<u32>() 6747 .unwrap_or_default(), 6748 2 6749 ); 6750 }); 6751 6752 let _ = child.kill(); 6753 6754 exec_host_command_status(&format!("sudo ip link del {guest_macvtap_name}")); 6755 exec_host_command_status(&format!("sudo ip link del {host_macvtap_name}")); 6756 6757 let output = child.wait_with_output().unwrap(); 6758 6759 handle_child_output(r, &output); 6760 } 6761 6762 #[test] 6763 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6764 fn test_macvtap() { 6765 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6766 } 6767 6768 #[test] 6769 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6770 fn test_macvtap_hotplug() { 6771 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6772 } 6773 6774 #[test] 6775 #[cfg(not(feature = "mshv"))] 6776 fn test_ovs_dpdk() { 6777 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6778 let guest1 = Guest::new(Box::new(focal1)); 6779 6780 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6781 let guest2 = Guest::new(Box::new(focal2)); 6782 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6783 6784 let (mut child1, mut child2) = 6785 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6786 6787 // Create the snapshot directory 6788 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6789 6790 let r = std::panic::catch_unwind(|| { 6791 // Remove one of the two ports from the OVS bridge 6792 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6793 6794 // Spawn a new netcat listener in the first VM 6795 let guest_ip = guest1.network.guest_ip.clone(); 6796 thread::spawn(move || { 6797 ssh_command_ip( 6798 "nc -l 12345", 6799 &guest_ip, 6800 DEFAULT_SSH_RETRIES, 6801 DEFAULT_SSH_TIMEOUT, 6802 ) 6803 .unwrap(); 6804 }); 6805 6806 // Wait for the server to be listening 6807 thread::sleep(std::time::Duration::new(5, 0)); 6808 6809 // Check the connection fails this time 6810 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6811 6812 // Add the OVS port back 6813 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6814 6815 // And finally check the connection is functional again 6816 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6817 6818 // Pause the VM 6819 assert!(remote_command(&api_socket_source, "pause", None)); 6820 6821 // Take a snapshot from the VM 6822 assert!(remote_command( 6823 &api_socket_source, 6824 "snapshot", 6825 Some(format!("file://{snapshot_dir}").as_str()), 6826 )); 6827 6828 // Wait to make sure the snapshot is completed 6829 thread::sleep(std::time::Duration::new(10, 0)); 6830 }); 6831 6832 // Shutdown the source VM 6833 let _ = child2.kill(); 6834 let output = child2.wait_with_output().unwrap(); 6835 handle_child_output(r, &output); 6836 6837 // Remove the vhost-user socket file. 6838 Command::new("rm") 6839 .arg("-f") 6840 .arg("/tmp/dpdkvhostclient2") 6841 .output() 6842 .unwrap(); 6843 6844 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6845 // Restore the VM from the snapshot 6846 let mut child2 = GuestCommand::new(&guest2) 6847 .args(["--api-socket", &api_socket_restored]) 6848 .args([ 6849 "--restore", 6850 format!("source_url=file://{snapshot_dir}").as_str(), 6851 ]) 6852 .capture_output() 6853 .spawn() 6854 .unwrap(); 6855 6856 // Wait for the VM to be restored 6857 thread::sleep(std::time::Duration::new(10, 0)); 6858 6859 let r = std::panic::catch_unwind(|| { 6860 // Resume the VM 6861 assert!(remote_command(&api_socket_restored, "resume", None)); 6862 6863 // Spawn a new netcat listener in the first VM 6864 let guest_ip = guest1.network.guest_ip.clone(); 6865 thread::spawn(move || { 6866 ssh_command_ip( 6867 "nc -l 12345", 6868 &guest_ip, 6869 DEFAULT_SSH_RETRIES, 6870 DEFAULT_SSH_TIMEOUT, 6871 ) 6872 .unwrap(); 6873 }); 6874 6875 // Wait for the server to be listening 6876 thread::sleep(std::time::Duration::new(5, 0)); 6877 6878 // And check the connection is still functional after restore 6879 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6880 }); 6881 6882 let _ = child1.kill(); 6883 let _ = child2.kill(); 6884 6885 let output = child1.wait_with_output().unwrap(); 6886 child2.wait().unwrap(); 6887 6888 cleanup_ovs_dpdk(); 6889 6890 handle_child_output(r, &output); 6891 } 6892 6893 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6894 cleanup_spdk_nvme(); 6895 6896 assert!(exec_host_command_status(&format!( 6897 "mkdir -p {}", 6898 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6899 )) 6900 .success()); 6901 assert!(exec_host_command_status(&format!( 6902 "truncate {} -s 128M", 6903 nvme_dir.join("test-disk.raw").to_str().unwrap() 6904 )) 6905 .success()); 6906 assert!(exec_host_command_status(&format!( 6907 "mkfs.ext4 {}", 6908 nvme_dir.join("test-disk.raw").to_str().unwrap() 6909 )) 6910 .success()); 6911 6912 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6913 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6914 .args(["-i", "0", "-m", "0x1"]) 6915 .spawn() 6916 .unwrap(); 6917 thread::sleep(std::time::Duration::new(2, 0)); 6918 6919 assert!(exec_host_command_with_retries( 6920 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER", 6921 3, 6922 std::time::Duration::new(5, 0), 6923 )); 6924 assert!(exec_host_command_status(&format!( 6925 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6926 nvme_dir.join("test-disk.raw").to_str().unwrap() 6927 )) 6928 .success()); 6929 assert!(exec_host_command_status( 6930 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6931 ) 6932 .success()); 6933 assert!(exec_host_command_status( 6934 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6935 ) 6936 .success()); 6937 assert!(exec_host_command_status(&format!( 6938 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6939 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6940 )) 6941 .success()); 6942 } 6943 6944 fn cleanup_spdk_nvme() { 6945 exec_host_command_status("pkill -f nvmf_tgt"); 6946 } 6947 6948 #[test] 6949 fn test_vfio_user() { 6950 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6951 let jammy = UbuntuDiskConfig::new(jammy_image); 6952 let guest = Guest::new(Box::new(jammy)); 6953 6954 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6955 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6956 6957 let api_socket = temp_api_path(&guest.tmp_dir); 6958 let mut child = GuestCommand::new(&guest) 6959 .args(["--api-socket", &api_socket]) 6960 .args(["--cpus", "boot=1"]) 6961 .args(["--memory", "size=512M,shared=on,hugepages=on"]) 6962 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6963 .args(["--serial", "tty", "--console", "off"]) 6964 .default_disks() 6965 .default_net() 6966 .capture_output() 6967 .spawn() 6968 .unwrap(); 6969 6970 let r = std::panic::catch_unwind(|| { 6971 guest.wait_vm_boot(None).unwrap(); 6972 6973 // Hotplug the SPDK-NVMe device to the VM 6974 let (cmd_success, cmd_output) = remote_command_w_output( 6975 &api_socket, 6976 "add-user-device", 6977 Some(&format!( 6978 "socket={},id=vfio_user0", 6979 spdk_nvme_dir 6980 .as_path() 6981 .join("nvme-vfio-user/cntrl") 6982 .to_str() 6983 .unwrap(), 6984 )), 6985 ); 6986 assert!(cmd_success); 6987 assert!(String::from_utf8_lossy(&cmd_output) 6988 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6989 6990 thread::sleep(std::time::Duration::new(10, 0)); 6991 6992 // Check both if /dev/nvme exists and if the block size is 128M. 6993 assert_eq!( 6994 guest 6995 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6996 .unwrap() 6997 .trim() 6998 .parse::<u32>() 6999 .unwrap_or_default(), 7000 1 7001 ); 7002 7003 // Check changes persist after reboot 7004 assert_eq!( 7005 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 7006 "" 7007 ); 7008 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 7009 guest 7010 .ssh_command("echo test123 | sudo tee /mnt/test") 7011 .unwrap(); 7012 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 7013 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 7014 7015 guest.reboot_linux(0, None); 7016 assert_eq!( 7017 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 7018 "" 7019 ); 7020 assert_eq!( 7021 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 7022 "test123" 7023 ); 7024 }); 7025 7026 cleanup_spdk_nvme(); 7027 7028 let _ = child.kill(); 7029 let output = child.wait_with_output().unwrap(); 7030 7031 handle_child_output(r, &output); 7032 } 7033 7034 #[test] 7035 #[cfg(target_arch = "x86_64")] 7036 fn test_vdpa_block() { 7037 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 7038 assert!(exec_host_command_status("lsmod | grep vdpa_sim_blk").success()); 7039 7040 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7041 let guest = Guest::new(Box::new(focal)); 7042 let api_socket = temp_api_path(&guest.tmp_dir); 7043 7044 let kernel_path = direct_kernel_boot_path(); 7045 7046 let mut child = GuestCommand::new(&guest) 7047 .args(["--cpus", "boot=2"]) 7048 .args(["--memory", "size=512M,hugepages=on"]) 7049 .args(["--kernel", kernel_path.to_str().unwrap()]) 7050 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7051 .default_disks() 7052 .default_net() 7053 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 7054 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 7055 .args(["--api-socket", &api_socket]) 7056 .capture_output() 7057 .spawn() 7058 .unwrap(); 7059 7060 let r = std::panic::catch_unwind(|| { 7061 guest.wait_vm_boot(None).unwrap(); 7062 7063 // Check both if /dev/vdc exists and if the block size is 128M. 7064 assert_eq!( 7065 guest 7066 .ssh_command("lsblk | grep vdc | grep -c 128M") 7067 .unwrap() 7068 .trim() 7069 .parse::<u32>() 7070 .unwrap_or_default(), 7071 1 7072 ); 7073 7074 // Check the content of the block device after we wrote to it. 7075 // The vpda-sim-blk should let us read what we previously wrote. 7076 guest 7077 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 7078 .unwrap(); 7079 assert_eq!( 7080 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 7081 "foobar" 7082 ); 7083 7084 // Hotplug an extra vDPA block device behind the vIOMMU 7085 // Add a new vDPA device to the VM 7086 let (cmd_success, cmd_output) = remote_command_w_output( 7087 &api_socket, 7088 "add-vdpa", 7089 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 7090 ); 7091 assert!(cmd_success); 7092 assert!(String::from_utf8_lossy(&cmd_output) 7093 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 7094 7095 thread::sleep(std::time::Duration::new(10, 0)); 7096 7097 // Check IOMMU setup 7098 assert!(guest 7099 .does_device_vendor_pair_match("0x1057", "0x1af4") 7100 .unwrap_or_default()); 7101 assert_eq!( 7102 guest 7103 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 7104 .unwrap() 7105 .trim(), 7106 "0001:00:01.0" 7107 ); 7108 7109 // Check both if /dev/vdd exists and if the block size is 128M. 7110 assert_eq!( 7111 guest 7112 .ssh_command("lsblk | grep vdd | grep -c 128M") 7113 .unwrap() 7114 .trim() 7115 .parse::<u32>() 7116 .unwrap_or_default(), 7117 1 7118 ); 7119 7120 // Write some content to the block device we've just plugged. 7121 guest 7122 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 7123 .unwrap(); 7124 7125 // Check we can read the content back. 7126 assert_eq!( 7127 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 7128 "foobar" 7129 ); 7130 7131 // Unplug the device 7132 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 7133 assert!(cmd_success); 7134 thread::sleep(std::time::Duration::new(10, 0)); 7135 7136 // Check /dev/vdd doesn't exist anymore 7137 assert_eq!( 7138 guest 7139 .ssh_command("lsblk | grep -c vdd || true") 7140 .unwrap() 7141 .trim() 7142 .parse::<u32>() 7143 .unwrap_or(1), 7144 0 7145 ); 7146 }); 7147 7148 let _ = child.kill(); 7149 let output = child.wait_with_output().unwrap(); 7150 7151 handle_child_output(r, &output); 7152 } 7153 7154 #[test] 7155 #[cfg(target_arch = "x86_64")] 7156 #[ignore = "See #5756"] 7157 fn test_vdpa_net() { 7158 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 7159 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 7160 return; 7161 } 7162 7163 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7164 let guest = Guest::new(Box::new(focal)); 7165 7166 let kernel_path = direct_kernel_boot_path(); 7167 7168 let mut child = GuestCommand::new(&guest) 7169 .args(["--cpus", "boot=2"]) 7170 .args(["--memory", "size=512M,hugepages=on"]) 7171 .args(["--kernel", kernel_path.to_str().unwrap()]) 7172 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7173 .default_disks() 7174 .default_net() 7175 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 7176 .capture_output() 7177 .spawn() 7178 .unwrap(); 7179 7180 let r = std::panic::catch_unwind(|| { 7181 guest.wait_vm_boot(None).unwrap(); 7182 7183 // Check we can find network interface related to vDPA device 7184 assert_eq!( 7185 guest 7186 .ssh_command("ip -o link | grep -c ens6") 7187 .unwrap() 7188 .trim() 7189 .parse::<u32>() 7190 .unwrap_or(0), 7191 1 7192 ); 7193 7194 guest 7195 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 7196 .unwrap(); 7197 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 7198 7199 // Check there is no packet yet on both TX/RX of the network interface 7200 assert_eq!( 7201 guest 7202 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 7203 .unwrap() 7204 .trim() 7205 .parse::<u32>() 7206 .unwrap_or(0), 7207 2 7208 ); 7209 7210 // Send 6 packets with ping command 7211 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 7212 7213 // Check we can find 6 packets on both TX/RX of the network interface 7214 assert_eq!( 7215 guest 7216 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 7217 .unwrap() 7218 .trim() 7219 .parse::<u32>() 7220 .unwrap_or(0), 7221 2 7222 ); 7223 7224 // No need to check for hotplug as we already tested it through 7225 // test_vdpa_block() 7226 }); 7227 7228 let _ = child.kill(); 7229 let output = child.wait_with_output().unwrap(); 7230 7231 handle_child_output(r, &output); 7232 } 7233 7234 #[test] 7235 #[cfg(target_arch = "x86_64")] 7236 fn test_tpm() { 7237 let focal = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7238 let guest = Guest::new(Box::new(focal)); 7239 7240 let (mut swtpm_command, swtpm_socket_path) = prepare_swtpm_daemon(&guest.tmp_dir); 7241 7242 let mut guest_cmd = GuestCommand::new(&guest); 7243 guest_cmd 7244 .args(["--cpus", "boot=1"]) 7245 .args(["--memory", "size=512M"]) 7246 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7247 .args(["--tpm", &format!("socket={swtpm_socket_path}")]) 7248 .capture_output() 7249 .default_disks() 7250 .default_net(); 7251 7252 // Start swtpm daemon 7253 let mut swtpm_child = swtpm_command.spawn().unwrap(); 7254 thread::sleep(std::time::Duration::new(10, 0)); 7255 let mut child = guest_cmd.spawn().unwrap(); 7256 let r = std::panic::catch_unwind(|| { 7257 guest.wait_vm_boot(None).unwrap(); 7258 assert_eq!( 7259 guest.ssh_command("ls /dev/tpm0").unwrap().trim(), 7260 "/dev/tpm0" 7261 ); 7262 guest.ssh_command("sudo tpm2_selftest -f").unwrap(); 7263 guest 7264 .ssh_command("echo 'hello' > /tmp/checksum_test; ") 7265 .unwrap(); 7266 guest.ssh_command("cmp <(sudo tpm2_pcrevent /tmp/checksum_test | grep sha256 | awk '{print $2}') <(sha256sum /tmp/checksum_test| awk '{print $1}')").unwrap(); 7267 }); 7268 7269 let _ = swtpm_child.kill(); 7270 let _d_out = swtpm_child.wait_with_output().unwrap(); 7271 7272 let _ = child.kill(); 7273 let output = child.wait_with_output().unwrap(); 7274 7275 handle_child_output(r, &output); 7276 } 7277 7278 #[test] 7279 #[cfg(target_arch = "x86_64")] 7280 fn test_double_tty() { 7281 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7282 let guest = Guest::new(Box::new(focal)); 7283 let mut cmd = GuestCommand::new(&guest); 7284 let api_socket = temp_api_path(&guest.tmp_dir); 7285 let tty_str: &str = "console=hvc0 earlyprintk=ttyS0 "; 7286 // linux printk module enable console log. 7287 let con_dis_str: &str = "console [hvc0] enabled"; 7288 // linux printk module disable console log. 7289 let con_enb_str: &str = "bootconsole [earlyser0] disabled"; 7290 7291 let kernel_path = direct_kernel_boot_path(); 7292 7293 cmd.args(["--cpus", "boot=1"]) 7294 .args(["--memory", "size=512M"]) 7295 .args(["--kernel", kernel_path.to_str().unwrap()]) 7296 .args([ 7297 "--cmdline", 7298 DIRECT_KERNEL_BOOT_CMDLINE 7299 .replace("console=hvc0 ", tty_str) 7300 .as_str(), 7301 ]) 7302 .capture_output() 7303 .default_disks() 7304 .default_net() 7305 .args(["--serial", "tty"]) 7306 .args(["--console", "tty"]) 7307 .args(["--api-socket", &api_socket]); 7308 7309 let mut child = cmd.spawn().unwrap(); 7310 7311 let mut r = std::panic::catch_unwind(|| { 7312 guest.wait_vm_boot(None).unwrap(); 7313 }); 7314 7315 let _ = child.kill(); 7316 let output = child.wait_with_output().unwrap(); 7317 7318 if r.is_ok() { 7319 r = std::panic::catch_unwind(|| { 7320 let s = String::from_utf8_lossy(&output.stdout); 7321 assert!(s.contains(tty_str)); 7322 assert!(s.contains(con_dis_str)); 7323 assert!(s.contains(con_enb_str)); 7324 }); 7325 } 7326 7327 handle_child_output(r, &output); 7328 } 7329 } 7330 7331 mod dbus_api { 7332 use crate::*; 7333 7334 // Start cloud-hypervisor with no VM parameters, running both the HTTP 7335 // and DBus APIs. Alternate calls to the external APIs (HTTP and DBus) 7336 // to create a VM, boot it, and verify that it can be shut down and then 7337 // booted again. 7338 #[test] 7339 fn test_api_dbus_and_http_interleaved() { 7340 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7341 let guest = Guest::new(Box::new(focal)); 7342 let dbus_api = TargetApi::new_dbus_api(&guest.tmp_dir); 7343 let http_api = TargetApi::new_http_api(&guest.tmp_dir); 7344 7345 let mut child = GuestCommand::new(&guest) 7346 .args(dbus_api.guest_args()) 7347 .args(http_api.guest_args()) 7348 .capture_output() 7349 .spawn() 7350 .unwrap(); 7351 7352 thread::sleep(std::time::Duration::new(1, 0)); 7353 7354 // Verify API servers are running 7355 assert!(dbus_api.remote_command("ping", None)); 7356 assert!(http_api.remote_command("ping", None)); 7357 7358 // Create the VM first 7359 let cpu_count: u8 = 4; 7360 let request_body = guest.api_create_body( 7361 cpu_count, 7362 direct_kernel_boot_path().to_str().unwrap(), 7363 DIRECT_KERNEL_BOOT_CMDLINE, 7364 ); 7365 7366 let temp_config_path = guest.tmp_dir.as_path().join("config"); 7367 std::fs::write(&temp_config_path, request_body).unwrap(); 7368 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 7369 7370 let r = std::panic::catch_unwind(|| { 7371 // Create the VM 7372 assert!(dbus_api.remote_command("create", Some(create_config),)); 7373 7374 // Then boot it 7375 assert!(http_api.remote_command("boot", None)); 7376 guest.wait_vm_boot(None).unwrap(); 7377 7378 // Check that the VM booted as expected 7379 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7380 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7381 7382 // Sync and shutdown without powering off to prevent filesystem 7383 // corruption. 7384 guest.ssh_command("sync").unwrap(); 7385 guest.ssh_command("sudo shutdown -H now").unwrap(); 7386 7387 // Wait for the guest to be fully shutdown 7388 thread::sleep(std::time::Duration::new(20, 0)); 7389 7390 // Then shutdown the VM 7391 assert!(dbus_api.remote_command("shutdown", None)); 7392 7393 // Then boot it again 7394 assert!(http_api.remote_command("boot", None)); 7395 guest.wait_vm_boot(None).unwrap(); 7396 7397 // Check that the VM booted as expected 7398 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7399 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7400 }); 7401 7402 let _ = child.kill(); 7403 let output = child.wait_with_output().unwrap(); 7404 7405 handle_child_output(r, &output); 7406 } 7407 7408 #[test] 7409 fn test_api_dbus_create_boot() { 7410 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7411 let guest = Guest::new(Box::new(focal)); 7412 7413 _test_api_create_boot(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7414 } 7415 7416 #[test] 7417 fn test_api_dbus_shutdown() { 7418 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7419 let guest = Guest::new(Box::new(focal)); 7420 7421 _test_api_shutdown(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7422 } 7423 7424 #[test] 7425 fn test_api_dbus_delete() { 7426 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7427 let guest = Guest::new(Box::new(focal)); 7428 7429 _test_api_delete(TargetApi::new_dbus_api(&guest.tmp_dir), guest); 7430 } 7431 7432 #[test] 7433 fn test_api_dbus_pause_resume() { 7434 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7435 let guest = Guest::new(Box::new(focal)); 7436 7437 _test_api_pause_resume(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7438 } 7439 } 7440 7441 mod common_sequential { 7442 #[cfg(not(feature = "mshv"))] 7443 use crate::*; 7444 7445 #[test] 7446 #[cfg(not(feature = "mshv"))] 7447 fn test_memory_mergeable_on() { 7448 test_memory_mergeable(true) 7449 } 7450 } 7451 7452 mod windows { 7453 use crate::*; 7454 use once_cell::sync::Lazy; 7455 7456 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 7457 7458 struct WindowsGuest { 7459 guest: Guest, 7460 auth: PasswordAuth, 7461 } 7462 7463 trait FsType { 7464 const FS_FAT: u8; 7465 const FS_NTFS: u8; 7466 } 7467 impl FsType for WindowsGuest { 7468 const FS_FAT: u8 = 0; 7469 const FS_NTFS: u8 = 1; 7470 } 7471 7472 impl WindowsGuest { 7473 fn new() -> Self { 7474 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 7475 let guest = Guest::new(Box::new(disk)); 7476 let auth = PasswordAuth { 7477 username: String::from("administrator"), 7478 password: String::from("Admin123"), 7479 }; 7480 7481 WindowsGuest { guest, auth } 7482 } 7483 7484 fn guest(&self) -> &Guest { 7485 &self.guest 7486 } 7487 7488 fn ssh_cmd(&self, cmd: &str) -> String { 7489 ssh_command_ip_with_auth( 7490 cmd, 7491 &self.auth, 7492 &self.guest.network.guest_ip, 7493 DEFAULT_SSH_RETRIES, 7494 DEFAULT_SSH_TIMEOUT, 7495 ) 7496 .unwrap() 7497 } 7498 7499 fn cpu_count(&self) -> u8 { 7500 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 7501 .trim() 7502 .parse::<u8>() 7503 .unwrap_or(0) 7504 } 7505 7506 fn ram_size(&self) -> usize { 7507 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 7508 .trim() 7509 .parse::<usize>() 7510 .unwrap_or(0) 7511 } 7512 7513 fn netdev_count(&self) -> u8 { 7514 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7515 .trim() 7516 .parse::<u8>() 7517 .unwrap_or(0) 7518 } 7519 7520 fn disk_count(&self) -> u8 { 7521 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7522 .trim() 7523 .parse::<u8>() 7524 .unwrap_or(0) 7525 } 7526 7527 fn reboot(&self) { 7528 let _ = self.ssh_cmd("shutdown /r /t 0"); 7529 } 7530 7531 fn shutdown(&self) { 7532 let _ = self.ssh_cmd("shutdown /s /t 0"); 7533 } 7534 7535 fn run_dnsmasq(&self) -> std::process::Child { 7536 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 7537 let dhcp_host = format!( 7538 "--dhcp-host={},{}", 7539 self.guest.network.guest_mac, self.guest.network.guest_ip 7540 ); 7541 let dhcp_range = format!( 7542 "--dhcp-range=eth,{},{}", 7543 self.guest.network.guest_ip, self.guest.network.guest_ip 7544 ); 7545 7546 Command::new("dnsmasq") 7547 .arg("--no-daemon") 7548 .arg("--log-queries") 7549 .arg(listen_address.as_str()) 7550 .arg("--except-interface=lo") 7551 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 7552 .arg("--conf-file=/dev/null") 7553 .arg(dhcp_host.as_str()) 7554 .arg(dhcp_range.as_str()) 7555 .spawn() 7556 .unwrap() 7557 } 7558 7559 // TODO Cleanup image file explicitly after test, if there's some space issues. 7560 fn disk_new(&self, fs: u8, sz: usize) -> String { 7561 let mut guard = NEXT_DISK_ID.lock().unwrap(); 7562 let id = *guard; 7563 *guard = id + 1; 7564 7565 let img = PathBuf::from(format!("/tmp/test-hotplug-{id}.raw")); 7566 let _ = fs::remove_file(&img); 7567 7568 // Create an image file 7569 let out = Command::new("qemu-img") 7570 .args([ 7571 "create", 7572 "-f", 7573 "raw", 7574 img.to_str().unwrap(), 7575 format!("{sz}m").as_str(), 7576 ]) 7577 .output() 7578 .expect("qemu-img command failed") 7579 .stdout; 7580 println!("{out:?}"); 7581 7582 // Associate image to a loop device 7583 let out = Command::new("losetup") 7584 .args(["--show", "-f", img.to_str().unwrap()]) 7585 .output() 7586 .expect("failed to create loop device") 7587 .stdout; 7588 let _tmp = String::from_utf8_lossy(&out); 7589 let loop_dev = _tmp.trim(); 7590 println!("{out:?}"); 7591 7592 // Create a partition table 7593 // echo 'type=7' | sudo sfdisk "${LOOP}" 7594 let mut child = Command::new("sfdisk") 7595 .args([loop_dev]) 7596 .stdin(Stdio::piped()) 7597 .spawn() 7598 .unwrap(); 7599 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 7600 stdin 7601 .write_all("type=7".as_bytes()) 7602 .expect("failed to write stdin"); 7603 let out = child.wait_with_output().expect("sfdisk failed").stdout; 7604 println!("{out:?}"); 7605 7606 // Disengage the loop device 7607 let out = Command::new("losetup") 7608 .args(["-d", loop_dev]) 7609 .output() 7610 .expect("loop device not found") 7611 .stdout; 7612 println!("{out:?}"); 7613 7614 // Re-associate loop device pointing to the partition only 7615 let out = Command::new("losetup") 7616 .args([ 7617 "--show", 7618 "--offset", 7619 (512 * 2048).to_string().as_str(), 7620 "-f", 7621 img.to_str().unwrap(), 7622 ]) 7623 .output() 7624 .expect("failed to create loop device") 7625 .stdout; 7626 let _tmp = String::from_utf8_lossy(&out); 7627 let loop_dev = _tmp.trim(); 7628 println!("{out:?}"); 7629 7630 // Create filesystem. 7631 let fs_cmd = match fs { 7632 WindowsGuest::FS_FAT => "mkfs.msdos", 7633 WindowsGuest::FS_NTFS => "mkfs.ntfs", 7634 _ => panic!("Unknown filesystem type '{fs}'"), 7635 }; 7636 let out = Command::new(fs_cmd) 7637 .args([&loop_dev]) 7638 .output() 7639 .unwrap_or_else(|_| panic!("{fs_cmd} failed")) 7640 .stdout; 7641 println!("{out:?}"); 7642 7643 // Disengage the loop device 7644 let out = Command::new("losetup") 7645 .args(["-d", loop_dev]) 7646 .output() 7647 .unwrap_or_else(|_| panic!("loop device '{loop_dev}' not found")) 7648 .stdout; 7649 println!("{out:?}"); 7650 7651 img.to_str().unwrap().to_string() 7652 } 7653 7654 fn disks_set_rw(&self) { 7655 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 7656 } 7657 7658 fn disks_online(&self) { 7659 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 7660 } 7661 7662 fn disk_file_put(&self, fname: &str, data: &str) { 7663 let _ = self.ssh_cmd(&format!( 7664 "powershell -Command \"'{data}' | Set-Content -Path {fname}\"" 7665 )); 7666 } 7667 7668 fn disk_file_read(&self, fname: &str) -> String { 7669 self.ssh_cmd(&format!( 7670 "powershell -Command \"Get-Content -Path {fname}\"" 7671 )) 7672 } 7673 7674 fn wait_for_boot(&self) -> bool { 7675 let cmd = "dir /b c:\\ | find \"Windows\""; 7676 let tmo_max = 180; 7677 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 7678 // interval must be small. 7679 let tmo_int = 2; 7680 let out = ssh_command_ip_with_auth( 7681 cmd, 7682 &self.auth, 7683 &self.guest.network.guest_ip, 7684 { 7685 let mut ret = 1; 7686 let mut tmo_acc = 0; 7687 loop { 7688 tmo_acc += tmo_int * ret; 7689 if tmo_acc >= tmo_max { 7690 break; 7691 } 7692 ret += 1; 7693 } 7694 ret 7695 }, 7696 tmo_int, 7697 ) 7698 .unwrap(); 7699 7700 if "Windows" == out.trim() { 7701 return true; 7702 } 7703 7704 false 7705 } 7706 } 7707 7708 fn vcpu_threads_count(pid: u32) -> u8 { 7709 // ps -T -p 12345 | grep vcpu | wc -l 7710 let out = Command::new("ps") 7711 .args(["-T", "-p", format!("{pid}").as_str()]) 7712 .output() 7713 .expect("ps command failed") 7714 .stdout; 7715 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 7716 } 7717 7718 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 7719 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 7720 let out = Command::new("ps") 7721 .args(["-T", "-p", format!("{pid}").as_str()]) 7722 .output() 7723 .expect("ps command failed") 7724 .stdout; 7725 let mut n = 0; 7726 String::from_utf8_lossy(&out) 7727 .split_whitespace() 7728 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 7729 n 7730 } 7731 7732 fn disk_ctrl_threads_count(pid: u32) -> u8 { 7733 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 7734 let out = Command::new("ps") 7735 .args(["-T", "-p", format!("{pid}").as_str()]) 7736 .output() 7737 .expect("ps command failed") 7738 .stdout; 7739 let mut n = 0; 7740 String::from_utf8_lossy(&out) 7741 .split_whitespace() 7742 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 7743 n 7744 } 7745 7746 #[test] 7747 fn test_windows_guest() { 7748 let windows_guest = WindowsGuest::new(); 7749 7750 let mut child = GuestCommand::new(windows_guest.guest()) 7751 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7752 .args(["--memory", "size=4G"]) 7753 .args(["--kernel", edk2_path().to_str().unwrap()]) 7754 .args(["--serial", "tty"]) 7755 .args(["--console", "off"]) 7756 .default_disks() 7757 .default_net() 7758 .capture_output() 7759 .spawn() 7760 .unwrap(); 7761 7762 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7763 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7764 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7765 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7766 7767 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7768 7769 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7770 7771 let r = std::panic::catch_unwind(|| { 7772 // Wait to make sure Windows boots up 7773 assert!(windows_guest.wait_for_boot()); 7774 7775 windows_guest.shutdown(); 7776 }); 7777 7778 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7779 let _ = child.kill(); 7780 let output = child.wait_with_output().unwrap(); 7781 7782 let _ = child_dnsmasq.kill(); 7783 let _ = child_dnsmasq.wait(); 7784 7785 handle_child_output(r, &output); 7786 } 7787 7788 #[test] 7789 fn test_windows_guest_multiple_queues() { 7790 let windows_guest = WindowsGuest::new(); 7791 7792 let mut ovmf_path = dirs::home_dir().unwrap(); 7793 ovmf_path.push("workloads"); 7794 ovmf_path.push(OVMF_NAME); 7795 7796 let mut child = GuestCommand::new(windows_guest.guest()) 7797 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 7798 .args(["--memory", "size=4G"]) 7799 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7800 .args(["--serial", "tty"]) 7801 .args(["--console", "off"]) 7802 .args([ 7803 "--disk", 7804 format!( 7805 "path={},num_queues=4", 7806 windows_guest 7807 .guest() 7808 .disk_config 7809 .disk(DiskType::OperatingSystem) 7810 .unwrap() 7811 ) 7812 .as_str(), 7813 ]) 7814 .args([ 7815 "--net", 7816 format!( 7817 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 7818 windows_guest.guest().network.guest_mac, 7819 windows_guest.guest().network.host_ip 7820 ) 7821 .as_str(), 7822 ]) 7823 .capture_output() 7824 .spawn() 7825 .unwrap(); 7826 7827 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7828 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7829 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7830 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7831 7832 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7833 7834 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7835 7836 let r = std::panic::catch_unwind(|| { 7837 // Wait to make sure Windows boots up 7838 assert!(windows_guest.wait_for_boot()); 7839 7840 windows_guest.shutdown(); 7841 }); 7842 7843 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7844 let _ = child.kill(); 7845 let output = child.wait_with_output().unwrap(); 7846 7847 let _ = child_dnsmasq.kill(); 7848 let _ = child_dnsmasq.wait(); 7849 7850 handle_child_output(r, &output); 7851 } 7852 7853 #[test] 7854 #[cfg(not(feature = "mshv"))] 7855 #[ignore = "See #4327"] 7856 fn test_windows_guest_snapshot_restore() { 7857 let windows_guest = WindowsGuest::new(); 7858 7859 let mut ovmf_path = dirs::home_dir().unwrap(); 7860 ovmf_path.push("workloads"); 7861 ovmf_path.push(OVMF_NAME); 7862 7863 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7864 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 7865 7866 let mut child = GuestCommand::new(windows_guest.guest()) 7867 .args(["--api-socket", &api_socket_source]) 7868 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7869 .args(["--memory", "size=4G"]) 7870 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7871 .args(["--serial", "tty"]) 7872 .args(["--console", "off"]) 7873 .default_disks() 7874 .default_net() 7875 .capture_output() 7876 .spawn() 7877 .unwrap(); 7878 7879 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7880 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7881 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7882 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7883 7884 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7885 7886 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7887 7888 // Wait to make sure Windows boots up 7889 assert!(windows_guest.wait_for_boot()); 7890 7891 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 7892 7893 // Pause the VM 7894 assert!(remote_command(&api_socket_source, "pause", None)); 7895 7896 // Take a snapshot from the VM 7897 assert!(remote_command( 7898 &api_socket_source, 7899 "snapshot", 7900 Some(format!("file://{snapshot_dir}").as_str()), 7901 )); 7902 7903 // Wait to make sure the snapshot is completed 7904 thread::sleep(std::time::Duration::new(30, 0)); 7905 7906 let _ = child.kill(); 7907 child.wait().unwrap(); 7908 7909 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 7910 7911 // Restore the VM from the snapshot 7912 let mut child = GuestCommand::new(windows_guest.guest()) 7913 .args(["--api-socket", &api_socket_restored]) 7914 .args([ 7915 "--restore", 7916 format!("source_url=file://{snapshot_dir}").as_str(), 7917 ]) 7918 .capture_output() 7919 .spawn() 7920 .unwrap(); 7921 7922 // Wait for the VM to be restored 7923 thread::sleep(std::time::Duration::new(20, 0)); 7924 7925 let r = std::panic::catch_unwind(|| { 7926 // Resume the VM 7927 assert!(remote_command(&api_socket_restored, "resume", None)); 7928 7929 windows_guest.shutdown(); 7930 }); 7931 7932 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7933 let _ = child.kill(); 7934 let output = child.wait_with_output().unwrap(); 7935 7936 let _ = child_dnsmasq.kill(); 7937 let _ = child_dnsmasq.wait(); 7938 7939 handle_child_output(r, &output); 7940 } 7941 7942 #[test] 7943 #[cfg(not(feature = "mshv"))] 7944 #[cfg(not(target_arch = "aarch64"))] 7945 fn test_windows_guest_cpu_hotplug() { 7946 let windows_guest = WindowsGuest::new(); 7947 7948 let mut ovmf_path = dirs::home_dir().unwrap(); 7949 ovmf_path.push("workloads"); 7950 ovmf_path.push(OVMF_NAME); 7951 7952 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7953 let api_socket = temp_api_path(&tmp_dir); 7954 7955 let mut child = GuestCommand::new(windows_guest.guest()) 7956 .args(["--api-socket", &api_socket]) 7957 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 7958 .args(["--memory", "size=4G"]) 7959 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7960 .args(["--serial", "tty"]) 7961 .args(["--console", "off"]) 7962 .default_disks() 7963 .default_net() 7964 .capture_output() 7965 .spawn() 7966 .unwrap(); 7967 7968 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7969 7970 let r = std::panic::catch_unwind(|| { 7971 // Wait to make sure Windows boots up 7972 assert!(windows_guest.wait_for_boot()); 7973 7974 let vcpu_num = 2; 7975 // Check the initial number of CPUs the guest sees 7976 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7977 // Check the initial number of vcpu threads in the CH process 7978 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7979 7980 let vcpu_num = 6; 7981 // Hotplug some CPUs 7982 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7983 // Wait to make sure CPUs are added 7984 thread::sleep(std::time::Duration::new(10, 0)); 7985 // Check the guest sees the correct number 7986 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7987 // Check the CH process has the correct number of vcpu threads 7988 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7989 7990 let vcpu_num = 4; 7991 // Remove some CPUs. Note that Windows doesn't support hot-remove. 7992 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7993 // Wait to make sure CPUs are removed 7994 thread::sleep(std::time::Duration::new(10, 0)); 7995 // Reboot to let Windows catch up 7996 windows_guest.reboot(); 7997 // Wait to make sure Windows completely rebooted 7998 thread::sleep(std::time::Duration::new(60, 0)); 7999 // Check the guest sees the correct number 8000 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8001 // Check the CH process has the correct number of vcpu threads 8002 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8003 8004 windows_guest.shutdown(); 8005 }); 8006 8007 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8008 let _ = child.kill(); 8009 let output = child.wait_with_output().unwrap(); 8010 8011 let _ = child_dnsmasq.kill(); 8012 let _ = child_dnsmasq.wait(); 8013 8014 handle_child_output(r, &output); 8015 } 8016 8017 #[test] 8018 #[cfg(not(feature = "mshv"))] 8019 #[cfg(not(target_arch = "aarch64"))] 8020 fn test_windows_guest_ram_hotplug() { 8021 let windows_guest = WindowsGuest::new(); 8022 8023 let mut ovmf_path = dirs::home_dir().unwrap(); 8024 ovmf_path.push("workloads"); 8025 ovmf_path.push(OVMF_NAME); 8026 8027 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8028 let api_socket = temp_api_path(&tmp_dir); 8029 8030 let mut child = GuestCommand::new(windows_guest.guest()) 8031 .args(["--api-socket", &api_socket]) 8032 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8033 .args(["--memory", "size=2G,hotplug_size=5G"]) 8034 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8035 .args(["--serial", "tty"]) 8036 .args(["--console", "off"]) 8037 .default_disks() 8038 .default_net() 8039 .capture_output() 8040 .spawn() 8041 .unwrap(); 8042 8043 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8044 8045 let r = std::panic::catch_unwind(|| { 8046 // Wait to make sure Windows boots up 8047 assert!(windows_guest.wait_for_boot()); 8048 8049 let ram_size = 2 * 1024 * 1024 * 1024; 8050 // Check the initial number of RAM the guest sees 8051 let current_ram_size = windows_guest.ram_size(); 8052 // This size seems to be reserved by the system and thus the 8053 // reported amount differs by this constant value. 8054 let reserved_ram_size = ram_size - current_ram_size; 8055 // Verify that there's not more than 4mb constant diff wasted 8056 // by the reserved ram. 8057 assert!(reserved_ram_size < 4 * 1024 * 1024); 8058 8059 let ram_size = 4 * 1024 * 1024 * 1024; 8060 // Hotplug some RAM 8061 resize_command(&api_socket, None, Some(ram_size), None, None); 8062 // Wait to make sure RAM has been added 8063 thread::sleep(std::time::Duration::new(10, 0)); 8064 // Check the guest sees the correct number 8065 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8066 8067 let ram_size = 3 * 1024 * 1024 * 1024; 8068 // Unplug some RAM. Note that hot-remove most likely won't work. 8069 resize_command(&api_socket, None, Some(ram_size), None, None); 8070 // Wait to make sure RAM has been added 8071 thread::sleep(std::time::Duration::new(10, 0)); 8072 // Reboot to let Windows catch up 8073 windows_guest.reboot(); 8074 // Wait to make sure guest completely rebooted 8075 thread::sleep(std::time::Duration::new(60, 0)); 8076 // Check the guest sees the correct number 8077 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8078 8079 windows_guest.shutdown(); 8080 }); 8081 8082 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8083 let _ = child.kill(); 8084 let output = child.wait_with_output().unwrap(); 8085 8086 let _ = child_dnsmasq.kill(); 8087 let _ = child_dnsmasq.wait(); 8088 8089 handle_child_output(r, &output); 8090 } 8091 8092 #[test] 8093 #[cfg(not(feature = "mshv"))] 8094 fn test_windows_guest_netdev_hotplug() { 8095 let windows_guest = WindowsGuest::new(); 8096 8097 let mut ovmf_path = dirs::home_dir().unwrap(); 8098 ovmf_path.push("workloads"); 8099 ovmf_path.push(OVMF_NAME); 8100 8101 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8102 let api_socket = temp_api_path(&tmp_dir); 8103 8104 let mut child = GuestCommand::new(windows_guest.guest()) 8105 .args(["--api-socket", &api_socket]) 8106 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8107 .args(["--memory", "size=4G"]) 8108 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8109 .args(["--serial", "tty"]) 8110 .args(["--console", "off"]) 8111 .default_disks() 8112 .default_net() 8113 .capture_output() 8114 .spawn() 8115 .unwrap(); 8116 8117 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8118 8119 let r = std::panic::catch_unwind(|| { 8120 // Wait to make sure Windows boots up 8121 assert!(windows_guest.wait_for_boot()); 8122 8123 // Initially present network device 8124 let netdev_num = 1; 8125 assert_eq!(windows_guest.netdev_count(), netdev_num); 8126 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8127 8128 // Hotplug network device 8129 let (cmd_success, cmd_output) = remote_command_w_output( 8130 &api_socket, 8131 "add-net", 8132 Some(windows_guest.guest().default_net_string().as_str()), 8133 ); 8134 assert!(cmd_success); 8135 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 8136 thread::sleep(std::time::Duration::new(5, 0)); 8137 // Verify the device is on the system 8138 let netdev_num = 2; 8139 assert_eq!(windows_guest.netdev_count(), netdev_num); 8140 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8141 8142 // Remove network device 8143 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 8144 assert!(cmd_success); 8145 thread::sleep(std::time::Duration::new(5, 0)); 8146 // Verify the device has been removed 8147 let netdev_num = 1; 8148 assert_eq!(windows_guest.netdev_count(), netdev_num); 8149 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8150 8151 windows_guest.shutdown(); 8152 }); 8153 8154 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8155 let _ = child.kill(); 8156 let output = child.wait_with_output().unwrap(); 8157 8158 let _ = child_dnsmasq.kill(); 8159 let _ = child_dnsmasq.wait(); 8160 8161 handle_child_output(r, &output); 8162 } 8163 8164 #[test] 8165 #[ignore = "See #6037"] 8166 #[cfg(not(feature = "mshv"))] 8167 #[cfg(not(target_arch = "aarch64"))] 8168 fn test_windows_guest_disk_hotplug() { 8169 let windows_guest = WindowsGuest::new(); 8170 8171 let mut ovmf_path = dirs::home_dir().unwrap(); 8172 ovmf_path.push("workloads"); 8173 ovmf_path.push(OVMF_NAME); 8174 8175 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8176 let api_socket = temp_api_path(&tmp_dir); 8177 8178 let mut child = GuestCommand::new(windows_guest.guest()) 8179 .args(["--api-socket", &api_socket]) 8180 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8181 .args(["--memory", "size=4G"]) 8182 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8183 .args(["--serial", "tty"]) 8184 .args(["--console", "off"]) 8185 .default_disks() 8186 .default_net() 8187 .capture_output() 8188 .spawn() 8189 .unwrap(); 8190 8191 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8192 8193 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 8194 8195 let r = std::panic::catch_unwind(|| { 8196 // Wait to make sure Windows boots up 8197 assert!(windows_guest.wait_for_boot()); 8198 8199 // Initially present disk device 8200 let disk_num = 1; 8201 assert_eq!(windows_guest.disk_count(), disk_num); 8202 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8203 8204 // Hotplug disk device 8205 let (cmd_success, cmd_output) = remote_command_w_output( 8206 &api_socket, 8207 "add-disk", 8208 Some(format!("path={disk},readonly=off").as_str()), 8209 ); 8210 assert!(cmd_success); 8211 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 8212 thread::sleep(std::time::Duration::new(5, 0)); 8213 // Online disk device 8214 windows_guest.disks_set_rw(); 8215 windows_guest.disks_online(); 8216 // Verify the device is on the system 8217 let disk_num = 2; 8218 assert_eq!(windows_guest.disk_count(), disk_num); 8219 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8220 8221 let data = "hello"; 8222 let fname = "d:\\world"; 8223 windows_guest.disk_file_put(fname, data); 8224 8225 // Unmount disk device 8226 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 8227 assert!(cmd_success); 8228 thread::sleep(std::time::Duration::new(5, 0)); 8229 // Verify the device has been removed 8230 let disk_num = 1; 8231 assert_eq!(windows_guest.disk_count(), disk_num); 8232 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8233 8234 // Remount and check the file exists with the expected contents 8235 let (cmd_success, _cmd_output) = remote_command_w_output( 8236 &api_socket, 8237 "add-disk", 8238 Some(format!("path={disk},readonly=off").as_str()), 8239 ); 8240 assert!(cmd_success); 8241 thread::sleep(std::time::Duration::new(5, 0)); 8242 let out = windows_guest.disk_file_read(fname); 8243 assert_eq!(data, out.trim()); 8244 8245 // Intentionally no unmount, it'll happen at shutdown. 8246 8247 windows_guest.shutdown(); 8248 }); 8249 8250 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8251 let _ = child.kill(); 8252 let output = child.wait_with_output().unwrap(); 8253 8254 let _ = child_dnsmasq.kill(); 8255 let _ = child_dnsmasq.wait(); 8256 8257 handle_child_output(r, &output); 8258 } 8259 8260 #[test] 8261 #[ignore = "See #6037"] 8262 #[cfg(not(feature = "mshv"))] 8263 #[cfg(not(target_arch = "aarch64"))] 8264 fn test_windows_guest_disk_hotplug_multi() { 8265 let windows_guest = WindowsGuest::new(); 8266 8267 let mut ovmf_path = dirs::home_dir().unwrap(); 8268 ovmf_path.push("workloads"); 8269 ovmf_path.push(OVMF_NAME); 8270 8271 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8272 let api_socket = temp_api_path(&tmp_dir); 8273 8274 let mut child = GuestCommand::new(windows_guest.guest()) 8275 .args(["--api-socket", &api_socket]) 8276 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8277 .args(["--memory", "size=2G"]) 8278 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8279 .args(["--serial", "tty"]) 8280 .args(["--console", "off"]) 8281 .default_disks() 8282 .default_net() 8283 .capture_output() 8284 .spawn() 8285 .unwrap(); 8286 8287 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8288 8289 // Predefined data to used at various test stages 8290 let disk_test_data: [[String; 4]; 2] = [ 8291 [ 8292 "_disk2".to_string(), 8293 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 8294 "d:\\world".to_string(), 8295 "hello".to_string(), 8296 ], 8297 [ 8298 "_disk3".to_string(), 8299 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 8300 "e:\\hello".to_string(), 8301 "world".to_string(), 8302 ], 8303 ]; 8304 8305 let r = std::panic::catch_unwind(|| { 8306 // Wait to make sure Windows boots up 8307 assert!(windows_guest.wait_for_boot()); 8308 8309 // Initially present disk device 8310 let disk_num = 1; 8311 assert_eq!(windows_guest.disk_count(), disk_num); 8312 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8313 8314 for it in &disk_test_data { 8315 let disk_id = it[0].as_str(); 8316 let disk = it[1].as_str(); 8317 // Hotplug disk device 8318 let (cmd_success, cmd_output) = remote_command_w_output( 8319 &api_socket, 8320 "add-disk", 8321 Some(format!("path={disk},readonly=off").as_str()), 8322 ); 8323 assert!(cmd_success); 8324 assert!(String::from_utf8_lossy(&cmd_output) 8325 .contains(format!("\"id\":\"{disk_id}\"").as_str())); 8326 thread::sleep(std::time::Duration::new(5, 0)); 8327 // Online disk devices 8328 windows_guest.disks_set_rw(); 8329 windows_guest.disks_online(); 8330 } 8331 // Verify the devices are on the system 8332 let disk_num = (disk_test_data.len() + 1) as u8; 8333 assert_eq!(windows_guest.disk_count(), disk_num); 8334 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8335 8336 // Put test data 8337 for it in &disk_test_data { 8338 let fname = it[2].as_str(); 8339 let data = it[3].as_str(); 8340 windows_guest.disk_file_put(fname, data); 8341 } 8342 8343 // Unmount disk devices 8344 for it in &disk_test_data { 8345 let disk_id = it[0].as_str(); 8346 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 8347 assert!(cmd_success); 8348 thread::sleep(std::time::Duration::new(5, 0)); 8349 } 8350 8351 // Verify the devices have been removed 8352 let disk_num = 1; 8353 assert_eq!(windows_guest.disk_count(), disk_num); 8354 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8355 8356 // Remount 8357 for it in &disk_test_data { 8358 let disk = it[1].as_str(); 8359 let (cmd_success, _cmd_output) = remote_command_w_output( 8360 &api_socket, 8361 "add-disk", 8362 Some(format!("path={disk},readonly=off").as_str()), 8363 ); 8364 assert!(cmd_success); 8365 thread::sleep(std::time::Duration::new(5, 0)); 8366 } 8367 8368 // Check the files exists with the expected contents 8369 for it in &disk_test_data { 8370 let fname = it[2].as_str(); 8371 let data = it[3].as_str(); 8372 let out = windows_guest.disk_file_read(fname); 8373 assert_eq!(data, out.trim()); 8374 } 8375 8376 // Intentionally no unmount, it'll happen at shutdown. 8377 8378 windows_guest.shutdown(); 8379 }); 8380 8381 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8382 let _ = child.kill(); 8383 let output = child.wait_with_output().unwrap(); 8384 8385 let _ = child_dnsmasq.kill(); 8386 let _ = child_dnsmasq.wait(); 8387 8388 handle_child_output(r, &output); 8389 } 8390 8391 #[test] 8392 #[cfg(not(feature = "mshv"))] 8393 #[cfg(not(target_arch = "aarch64"))] 8394 fn test_windows_guest_netdev_multi() { 8395 let windows_guest = WindowsGuest::new(); 8396 8397 let mut ovmf_path = dirs::home_dir().unwrap(); 8398 ovmf_path.push("workloads"); 8399 ovmf_path.push(OVMF_NAME); 8400 8401 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8402 let api_socket = temp_api_path(&tmp_dir); 8403 8404 let mut child = GuestCommand::new(windows_guest.guest()) 8405 .args(["--api-socket", &api_socket]) 8406 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8407 .args(["--memory", "size=4G"]) 8408 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8409 .args(["--serial", "tty"]) 8410 .args(["--console", "off"]) 8411 .default_disks() 8412 // The multi net dev config is borrowed from test_multiple_network_interfaces 8413 .args([ 8414 "--net", 8415 windows_guest.guest().default_net_string().as_str(), 8416 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 8417 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 8418 ]) 8419 .capture_output() 8420 .spawn() 8421 .unwrap(); 8422 8423 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8424 8425 let r = std::panic::catch_unwind(|| { 8426 // Wait to make sure Windows boots up 8427 assert!(windows_guest.wait_for_boot()); 8428 8429 let netdev_num = 3; 8430 assert_eq!(windows_guest.netdev_count(), netdev_num); 8431 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8432 8433 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 8434 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 8435 8436 windows_guest.shutdown(); 8437 }); 8438 8439 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8440 let _ = child.kill(); 8441 let output = child.wait_with_output().unwrap(); 8442 8443 let _ = child_dnsmasq.kill(); 8444 let _ = child_dnsmasq.wait(); 8445 8446 handle_child_output(r, &output); 8447 } 8448 } 8449 8450 #[cfg(target_arch = "x86_64")] 8451 mod sgx { 8452 use crate::*; 8453 8454 #[test] 8455 fn test_sgx() { 8456 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 8457 let jammy = UbuntuDiskConfig::new(jammy_image); 8458 let guest = Guest::new(Box::new(jammy)); 8459 8460 let mut child = GuestCommand::new(&guest) 8461 .args(["--cpus", "boot=1"]) 8462 .args(["--memory", "size=512M"]) 8463 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8464 .default_disks() 8465 .default_net() 8466 .args(["--sgx-epc", "id=epc0,size=64M"]) 8467 .capture_output() 8468 .spawn() 8469 .unwrap(); 8470 8471 let r = std::panic::catch_unwind(|| { 8472 guest.wait_vm_boot(None).unwrap(); 8473 8474 // Check if SGX is correctly detected in the guest. 8475 guest.check_sgx_support().unwrap(); 8476 8477 // Validate the SGX EPC section is 64MiB. 8478 assert_eq!( 8479 guest 8480 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 8481 .unwrap() 8482 .trim(), 8483 "0x0000000004000000" 8484 ); 8485 }); 8486 8487 let _ = child.kill(); 8488 let output = child.wait_with_output().unwrap(); 8489 8490 handle_child_output(r, &output); 8491 } 8492 } 8493 8494 #[cfg(target_arch = "x86_64")] 8495 mod vfio { 8496 use crate::*; 8497 8498 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 8499 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8500 let guest = Guest::new(Box::new(jammy)); 8501 let api_socket = temp_api_path(&guest.tmp_dir); 8502 8503 let mut child = GuestCommand::new(&guest) 8504 .args(["--cpus", "boot=4"]) 8505 .args([ 8506 "--memory", 8507 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(), 8508 ]) 8509 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8510 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8511 .args(["--api-socket", &api_socket]) 8512 .default_disks() 8513 .default_net() 8514 .capture_output() 8515 .spawn() 8516 .unwrap(); 8517 8518 let r = std::panic::catch_unwind(|| { 8519 guest.wait_vm_boot(None).unwrap(); 8520 8521 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8522 8523 guest.enable_memory_hotplug(); 8524 8525 // Add RAM to the VM 8526 let desired_ram = 6 << 30; 8527 resize_command(&api_socket, None, Some(desired_ram), None, None); 8528 thread::sleep(std::time::Duration::new(30, 0)); 8529 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8530 8531 // Check the VFIO device works when RAM is increased to 6GiB 8532 guest.check_nvidia_gpu(); 8533 }); 8534 8535 let _ = child.kill(); 8536 let output = child.wait_with_output().unwrap(); 8537 8538 handle_child_output(r, &output); 8539 } 8540 8541 #[test] 8542 fn test_nvidia_card_memory_hotplug_acpi() { 8543 test_nvidia_card_memory_hotplug("acpi") 8544 } 8545 8546 #[test] 8547 fn test_nvidia_card_memory_hotplug_virtio_mem() { 8548 test_nvidia_card_memory_hotplug("virtio-mem") 8549 } 8550 8551 #[test] 8552 fn test_nvidia_card_pci_hotplug() { 8553 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8554 let guest = Guest::new(Box::new(jammy)); 8555 let api_socket = temp_api_path(&guest.tmp_dir); 8556 8557 let mut child = GuestCommand::new(&guest) 8558 .args(["--cpus", "boot=4"]) 8559 .args(["--memory", "size=4G"]) 8560 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8561 .args(["--api-socket", &api_socket]) 8562 .default_disks() 8563 .default_net() 8564 .capture_output() 8565 .spawn() 8566 .unwrap(); 8567 8568 let r = std::panic::catch_unwind(|| { 8569 guest.wait_vm_boot(None).unwrap(); 8570 8571 // Hotplug the card to the VM 8572 let (cmd_success, cmd_output) = remote_command_w_output( 8573 &api_socket, 8574 "add-device", 8575 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 8576 ); 8577 assert!(cmd_success); 8578 assert!(String::from_utf8_lossy(&cmd_output) 8579 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 8580 8581 thread::sleep(std::time::Duration::new(10, 0)); 8582 8583 // Check the VFIO device works after hotplug 8584 guest.check_nvidia_gpu(); 8585 }); 8586 8587 let _ = child.kill(); 8588 let output = child.wait_with_output().unwrap(); 8589 8590 handle_child_output(r, &output); 8591 } 8592 8593 #[test] 8594 fn test_nvidia_card_reboot() { 8595 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8596 let guest = Guest::new(Box::new(jammy)); 8597 let api_socket = temp_api_path(&guest.tmp_dir); 8598 8599 let mut child = GuestCommand::new(&guest) 8600 .args(["--cpus", "boot=4"]) 8601 .args(["--memory", "size=4G"]) 8602 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8603 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8604 .args(["--api-socket", &api_socket]) 8605 .default_disks() 8606 .default_net() 8607 .capture_output() 8608 .spawn() 8609 .unwrap(); 8610 8611 let r = std::panic::catch_unwind(|| { 8612 guest.wait_vm_boot(None).unwrap(); 8613 8614 // Check the VFIO device works after boot 8615 guest.check_nvidia_gpu(); 8616 8617 guest.reboot_linux(0, None); 8618 8619 // Check the VFIO device works after reboot 8620 guest.check_nvidia_gpu(); 8621 }); 8622 8623 let _ = child.kill(); 8624 let output = child.wait_with_output().unwrap(); 8625 8626 handle_child_output(r, &output); 8627 } 8628 } 8629 8630 mod live_migration { 8631 use crate::*; 8632 8633 fn start_live_migration( 8634 migration_socket: &str, 8635 src_api_socket: &str, 8636 dest_api_socket: &str, 8637 local: bool, 8638 ) -> bool { 8639 // Start to receive migration from the destintion VM 8640 let mut receive_migration = Command::new(clh_command("ch-remote")) 8641 .args([ 8642 &format!("--api-socket={dest_api_socket}"), 8643 "receive-migration", 8644 &format! {"unix:{migration_socket}"}, 8645 ]) 8646 .stderr(Stdio::piped()) 8647 .stdout(Stdio::piped()) 8648 .spawn() 8649 .unwrap(); 8650 // Give it '1s' to make sure the 'migration_socket' file is properly created 8651 thread::sleep(std::time::Duration::new(1, 0)); 8652 // Start to send migration from the source VM 8653 8654 let mut args = [ 8655 format!("--api-socket={}", &src_api_socket), 8656 "send-migration".to_string(), 8657 format! {"unix:{migration_socket}"}, 8658 ] 8659 .to_vec(); 8660 8661 if local { 8662 args.insert(2, "--local".to_string()); 8663 } 8664 8665 let mut send_migration = Command::new(clh_command("ch-remote")) 8666 .args(&args) 8667 .stderr(Stdio::piped()) 8668 .stdout(Stdio::piped()) 8669 .spawn() 8670 .unwrap(); 8671 8672 // The 'send-migration' command should be executed successfully within the given timeout 8673 let send_success = if let Some(status) = send_migration 8674 .wait_timeout(std::time::Duration::from_secs(30)) 8675 .unwrap() 8676 { 8677 status.success() 8678 } else { 8679 false 8680 }; 8681 8682 if !send_success { 8683 let _ = send_migration.kill(); 8684 let output = send_migration.wait_with_output().unwrap(); 8685 eprintln!( 8686 "\n\n==== Start 'send_migration' output ==== \ 8687 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8688 \n\n==== End 'send_migration' output ====\n\n", 8689 String::from_utf8_lossy(&output.stdout), 8690 String::from_utf8_lossy(&output.stderr) 8691 ); 8692 } 8693 8694 // The 'receive-migration' command should be executed successfully within the given timeout 8695 let receive_success = if let Some(status) = receive_migration 8696 .wait_timeout(std::time::Duration::from_secs(30)) 8697 .unwrap() 8698 { 8699 status.success() 8700 } else { 8701 false 8702 }; 8703 8704 if !receive_success { 8705 let _ = receive_migration.kill(); 8706 let output = receive_migration.wait_with_output().unwrap(); 8707 eprintln!( 8708 "\n\n==== Start 'receive_migration' output ==== \ 8709 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8710 \n\n==== End 'receive_migration' output ====\n\n", 8711 String::from_utf8_lossy(&output.stdout), 8712 String::from_utf8_lossy(&output.stderr) 8713 ); 8714 } 8715 8716 send_success && receive_success 8717 } 8718 8719 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 8720 let mut src_vm = src_vm; 8721 let mut dest_vm = dest_vm; 8722 8723 let _ = src_vm.kill(); 8724 let src_output = src_vm.wait_with_output().unwrap(); 8725 eprintln!( 8726 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8727 String::from_utf8_lossy(&src_output.stdout) 8728 ); 8729 eprintln!( 8730 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8731 String::from_utf8_lossy(&src_output.stderr) 8732 ); 8733 let _ = dest_vm.kill(); 8734 let dest_output = dest_vm.wait_with_output().unwrap(); 8735 eprintln!( 8736 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8737 String::from_utf8_lossy(&dest_output.stdout) 8738 ); 8739 eprintln!( 8740 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8741 String::from_utf8_lossy(&dest_output.stderr) 8742 ); 8743 8744 if let Some(ovs_vm) = ovs_vm { 8745 let mut ovs_vm = ovs_vm; 8746 let _ = ovs_vm.kill(); 8747 let ovs_output = ovs_vm.wait_with_output().unwrap(); 8748 eprintln!( 8749 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 8750 String::from_utf8_lossy(&ovs_output.stdout) 8751 ); 8752 eprintln!( 8753 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 8754 String::from_utf8_lossy(&ovs_output.stderr) 8755 ); 8756 8757 cleanup_ovs_dpdk(); 8758 } 8759 8760 panic!("Test failed: {message}") 8761 } 8762 8763 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 8764 // same host. It ensures the following behaviors: 8765 // 1. The source VM is up and functional (including various virtio-devices are working properly); 8766 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 8767 // 3. The source VM terminated gracefully after live migration; 8768 // 4. The destination VM is functional (including various virtio-devices are working properly) after 8769 // live migration; 8770 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 8771 fn _test_live_migration(upgrade_test: bool, local: bool) { 8772 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8773 let guest = Guest::new(Box::new(focal)); 8774 let kernel_path = direct_kernel_boot_path(); 8775 let console_text = String::from("On a branch floating down river a cricket, singing."); 8776 let net_id = "net123"; 8777 let net_params = format!( 8778 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8779 net_id, guest.network.guest_mac, guest.network.host_ip 8780 ); 8781 8782 let memory_param: &[&str] = if local { 8783 &["--memory", "size=4G,shared=on"] 8784 } else { 8785 &["--memory", "size=4G"] 8786 }; 8787 8788 let boot_vcpus = 2; 8789 let max_vcpus = 4; 8790 8791 let pmem_temp_file = TempFile::new().unwrap(); 8792 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8793 std::process::Command::new("mkfs.ext4") 8794 .arg(pmem_temp_file.as_path()) 8795 .output() 8796 .expect("Expect creating disk image to succeed"); 8797 let pmem_path = String::from("/dev/pmem0"); 8798 8799 // Start the source VM 8800 let src_vm_path = if !upgrade_test { 8801 clh_command("cloud-hypervisor") 8802 } else { 8803 cloud_hypervisor_release_path() 8804 }; 8805 let src_api_socket = temp_api_path(&guest.tmp_dir); 8806 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8807 src_vm_cmd 8808 .args([ 8809 "--cpus", 8810 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8811 ]) 8812 .args(memory_param) 8813 .args(["--kernel", kernel_path.to_str().unwrap()]) 8814 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8815 .default_disks() 8816 .args(["--net", net_params.as_str()]) 8817 .args(["--api-socket", &src_api_socket]) 8818 .args([ 8819 "--pmem", 8820 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8821 ]); 8822 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8823 8824 // Start the destination VM 8825 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8826 dest_api_socket.push_str(".dest"); 8827 let mut dest_child = GuestCommand::new(&guest) 8828 .args(["--api-socket", &dest_api_socket]) 8829 .capture_output() 8830 .spawn() 8831 .unwrap(); 8832 8833 let r = std::panic::catch_unwind(|| { 8834 guest.wait_vm_boot(None).unwrap(); 8835 8836 // Make sure the source VM is functaionl 8837 // Check the number of vCPUs 8838 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8839 8840 // Check the guest RAM 8841 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8842 8843 // Check the guest virtio-devices, e.g. block, rng, console, and net 8844 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8845 8846 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8847 // to make sure that removing and adding back the virtio-net device does 8848 // not break the live-migration support for virtio-pci. 8849 #[cfg(target_arch = "x86_64")] 8850 { 8851 assert!(remote_command( 8852 &src_api_socket, 8853 "remove-device", 8854 Some(net_id), 8855 )); 8856 thread::sleep(std::time::Duration::new(10, 0)); 8857 8858 // Plug the virtio-net device again 8859 assert!(remote_command( 8860 &src_api_socket, 8861 "add-net", 8862 Some(net_params.as_str()), 8863 )); 8864 thread::sleep(std::time::Duration::new(10, 0)); 8865 } 8866 8867 // Start the live-migration 8868 let migration_socket = String::from( 8869 guest 8870 .tmp_dir 8871 .as_path() 8872 .join("live-migration.sock") 8873 .to_str() 8874 .unwrap(), 8875 ); 8876 8877 assert!( 8878 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8879 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8880 ); 8881 }); 8882 8883 // Check and report any errors occurred during the live-migration 8884 if r.is_err() { 8885 print_and_panic( 8886 src_child, 8887 dest_child, 8888 None, 8889 "Error occurred during live-migration", 8890 ); 8891 } 8892 8893 // Check the source vm has been terminated successful (give it '3s' to settle) 8894 thread::sleep(std::time::Duration::new(3, 0)); 8895 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8896 print_and_panic( 8897 src_child, 8898 dest_child, 8899 None, 8900 "source VM was not terminated successfully.", 8901 ); 8902 }; 8903 8904 // Post live-migration check to make sure the destination VM is funcational 8905 let r = std::panic::catch_unwind(|| { 8906 // Perform same checks to validate VM has been properly migrated 8907 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8908 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8909 8910 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8911 }); 8912 8913 // Clean-up the destination VM and make sure it terminated correctly 8914 let _ = dest_child.kill(); 8915 let dest_output = dest_child.wait_with_output().unwrap(); 8916 handle_child_output(r, &dest_output); 8917 8918 // Check the destination VM has the expected 'concole_text' from its output 8919 let r = std::panic::catch_unwind(|| { 8920 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8921 }); 8922 handle_child_output(r, &dest_output); 8923 } 8924 8925 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 8926 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8927 let guest = Guest::new(Box::new(focal)); 8928 let kernel_path = direct_kernel_boot_path(); 8929 let console_text = String::from("On a branch floating down river a cricket, singing."); 8930 let net_id = "net123"; 8931 let net_params = format!( 8932 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8933 net_id, guest.network.guest_mac, guest.network.host_ip 8934 ); 8935 8936 let memory_param: &[&str] = if local { 8937 &[ 8938 "--memory", 8939 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 8940 "--balloon", 8941 "size=0", 8942 ] 8943 } else { 8944 &[ 8945 "--memory", 8946 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 8947 "--balloon", 8948 "size=0", 8949 ] 8950 }; 8951 8952 let boot_vcpus = 2; 8953 let max_vcpus = 4; 8954 8955 let pmem_temp_file = TempFile::new().unwrap(); 8956 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8957 std::process::Command::new("mkfs.ext4") 8958 .arg(pmem_temp_file.as_path()) 8959 .output() 8960 .expect("Expect creating disk image to succeed"); 8961 let pmem_path = String::from("/dev/pmem0"); 8962 8963 // Start the source VM 8964 let src_vm_path = if !upgrade_test { 8965 clh_command("cloud-hypervisor") 8966 } else { 8967 cloud_hypervisor_release_path() 8968 }; 8969 let src_api_socket = temp_api_path(&guest.tmp_dir); 8970 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8971 src_vm_cmd 8972 .args([ 8973 "--cpus", 8974 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8975 ]) 8976 .args(memory_param) 8977 .args(["--kernel", kernel_path.to_str().unwrap()]) 8978 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8979 .default_disks() 8980 .args(["--net", net_params.as_str()]) 8981 .args(["--api-socket", &src_api_socket]) 8982 .args([ 8983 "--pmem", 8984 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8985 ]); 8986 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8987 8988 // Start the destination VM 8989 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8990 dest_api_socket.push_str(".dest"); 8991 let mut dest_child = GuestCommand::new(&guest) 8992 .args(["--api-socket", &dest_api_socket]) 8993 .capture_output() 8994 .spawn() 8995 .unwrap(); 8996 8997 let r = std::panic::catch_unwind(|| { 8998 guest.wait_vm_boot(None).unwrap(); 8999 9000 // Make sure the source VM is functaionl 9001 // Check the number of vCPUs 9002 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9003 9004 // Check the guest RAM 9005 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9006 // Increase the guest RAM 9007 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 9008 thread::sleep(std::time::Duration::new(5, 0)); 9009 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9010 // Use balloon to remove RAM from the VM 9011 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 9012 thread::sleep(std::time::Duration::new(5, 0)); 9013 let total_memory = guest.get_total_memory().unwrap_or_default(); 9014 assert!(total_memory > 4_800_000); 9015 assert!(total_memory < 5_760_000); 9016 9017 // Check the guest virtio-devices, e.g. block, rng, console, and net 9018 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9019 9020 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9021 // to make sure that removing and adding back the virtio-net device does 9022 // not break the live-migration support for virtio-pci. 9023 #[cfg(target_arch = "x86_64")] 9024 { 9025 assert!(remote_command( 9026 &src_api_socket, 9027 "remove-device", 9028 Some(net_id), 9029 )); 9030 thread::sleep(std::time::Duration::new(10, 0)); 9031 9032 // Plug the virtio-net device again 9033 assert!(remote_command( 9034 &src_api_socket, 9035 "add-net", 9036 Some(net_params.as_str()), 9037 )); 9038 thread::sleep(std::time::Duration::new(10, 0)); 9039 } 9040 9041 // Start the live-migration 9042 let migration_socket = String::from( 9043 guest 9044 .tmp_dir 9045 .as_path() 9046 .join("live-migration.sock") 9047 .to_str() 9048 .unwrap(), 9049 ); 9050 9051 assert!( 9052 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9053 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9054 ); 9055 }); 9056 9057 // Check and report any errors occurred during the live-migration 9058 if r.is_err() { 9059 print_and_panic( 9060 src_child, 9061 dest_child, 9062 None, 9063 "Error occurred during live-migration", 9064 ); 9065 } 9066 9067 // Check the source vm has been terminated successful (give it '3s' to settle) 9068 thread::sleep(std::time::Duration::new(3, 0)); 9069 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9070 print_and_panic( 9071 src_child, 9072 dest_child, 9073 None, 9074 "source VM was not terminated successfully.", 9075 ); 9076 }; 9077 9078 // Post live-migration check to make sure the destination VM is funcational 9079 let r = std::panic::catch_unwind(|| { 9080 // Perform same checks to validate VM has been properly migrated 9081 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9082 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9083 9084 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9085 9086 // Perform checks on guest RAM using balloon 9087 let total_memory = guest.get_total_memory().unwrap_or_default(); 9088 assert!(total_memory > 4_800_000); 9089 assert!(total_memory < 5_760_000); 9090 // Deflate balloon to restore entire RAM to the VM 9091 resize_command(&dest_api_socket, None, None, Some(0), None); 9092 thread::sleep(std::time::Duration::new(5, 0)); 9093 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9094 // Decrease guest RAM with virtio-mem 9095 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 9096 thread::sleep(std::time::Duration::new(5, 0)); 9097 let total_memory = guest.get_total_memory().unwrap_or_default(); 9098 assert!(total_memory > 4_800_000); 9099 assert!(total_memory < 5_760_000); 9100 }); 9101 9102 // Clean-up the destination VM and make sure it terminated correctly 9103 let _ = dest_child.kill(); 9104 let dest_output = dest_child.wait_with_output().unwrap(); 9105 handle_child_output(r, &dest_output); 9106 9107 // Check the destination VM has the expected 'concole_text' from its output 9108 let r = std::panic::catch_unwind(|| { 9109 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9110 }); 9111 handle_child_output(r, &dest_output); 9112 } 9113 9114 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 9115 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9116 let guest = Guest::new(Box::new(focal)); 9117 let kernel_path = direct_kernel_boot_path(); 9118 let console_text = String::from("On a branch floating down river a cricket, singing."); 9119 let net_id = "net123"; 9120 let net_params = format!( 9121 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9122 net_id, guest.network.guest_mac, guest.network.host_ip 9123 ); 9124 9125 let memory_param: &[&str] = if local { 9126 &[ 9127 "--memory", 9128 "size=0,hotplug_method=virtio-mem,shared=on", 9129 "--memory-zone", 9130 "id=mem0,size=1G,hotplug_size=4G,shared=on", 9131 "id=mem1,size=1G,hotplug_size=4G,shared=on", 9132 "id=mem2,size=2G,hotplug_size=4G,shared=on", 9133 "--numa", 9134 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9135 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9136 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9137 ] 9138 } else { 9139 &[ 9140 "--memory", 9141 "size=0,hotplug_method=virtio-mem", 9142 "--memory-zone", 9143 "id=mem0,size=1G,hotplug_size=4G", 9144 "id=mem1,size=1G,hotplug_size=4G", 9145 "id=mem2,size=2G,hotplug_size=4G", 9146 "--numa", 9147 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9148 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9149 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9150 ] 9151 }; 9152 9153 let boot_vcpus = 6; 9154 let max_vcpus = 12; 9155 9156 let pmem_temp_file = TempFile::new().unwrap(); 9157 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9158 std::process::Command::new("mkfs.ext4") 9159 .arg(pmem_temp_file.as_path()) 9160 .output() 9161 .expect("Expect creating disk image to succeed"); 9162 let pmem_path = String::from("/dev/pmem0"); 9163 9164 // Start the source VM 9165 let src_vm_path = if !upgrade_test { 9166 clh_command("cloud-hypervisor") 9167 } else { 9168 cloud_hypervisor_release_path() 9169 }; 9170 let src_api_socket = temp_api_path(&guest.tmp_dir); 9171 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9172 src_vm_cmd 9173 .args([ 9174 "--cpus", 9175 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9176 ]) 9177 .args(memory_param) 9178 .args(["--kernel", kernel_path.to_str().unwrap()]) 9179 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9180 .default_disks() 9181 .args(["--net", net_params.as_str()]) 9182 .args(["--api-socket", &src_api_socket]) 9183 .args([ 9184 "--pmem", 9185 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9186 ]); 9187 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9188 9189 // Start the destination VM 9190 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9191 dest_api_socket.push_str(".dest"); 9192 let mut dest_child = GuestCommand::new(&guest) 9193 .args(["--api-socket", &dest_api_socket]) 9194 .capture_output() 9195 .spawn() 9196 .unwrap(); 9197 9198 let r = std::panic::catch_unwind(|| { 9199 guest.wait_vm_boot(None).unwrap(); 9200 9201 // Make sure the source VM is functaionl 9202 // Check the number of vCPUs 9203 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9204 9205 // Check the guest RAM 9206 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 9207 9208 // Check the guest virtio-devices, e.g. block, rng, console, and net 9209 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9210 9211 // Check the NUMA parameters are applied correctly and resize 9212 // each zone to test the case where we migrate a VM with the 9213 // virtio-mem regions being used. 9214 { 9215 guest.check_numa_common( 9216 Some(&[960_000, 960_000, 1_920_000]), 9217 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9218 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9219 ); 9220 9221 // AArch64 currently does not support hotplug, and therefore we only 9222 // test hotplug-related function on x86_64 here. 9223 #[cfg(target_arch = "x86_64")] 9224 { 9225 guest.enable_memory_hotplug(); 9226 9227 // Resize every memory zone and check each associated NUMA node 9228 // has been assigned the right amount of memory. 9229 resize_zone_command(&src_api_socket, "mem0", "2G"); 9230 resize_zone_command(&src_api_socket, "mem1", "2G"); 9231 resize_zone_command(&src_api_socket, "mem2", "3G"); 9232 thread::sleep(std::time::Duration::new(5, 0)); 9233 9234 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 9235 } 9236 } 9237 9238 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9239 // to make sure that removing and adding back the virtio-net device does 9240 // not break the live-migration support for virtio-pci. 9241 #[cfg(target_arch = "x86_64")] 9242 { 9243 assert!(remote_command( 9244 &src_api_socket, 9245 "remove-device", 9246 Some(net_id), 9247 )); 9248 thread::sleep(std::time::Duration::new(10, 0)); 9249 9250 // Plug the virtio-net device again 9251 assert!(remote_command( 9252 &src_api_socket, 9253 "add-net", 9254 Some(net_params.as_str()), 9255 )); 9256 thread::sleep(std::time::Duration::new(10, 0)); 9257 } 9258 9259 // Start the live-migration 9260 let migration_socket = String::from( 9261 guest 9262 .tmp_dir 9263 .as_path() 9264 .join("live-migration.sock") 9265 .to_str() 9266 .unwrap(), 9267 ); 9268 9269 assert!( 9270 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9271 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9272 ); 9273 }); 9274 9275 // Check and report any errors occurred during the live-migration 9276 if r.is_err() { 9277 print_and_panic( 9278 src_child, 9279 dest_child, 9280 None, 9281 "Error occurred during live-migration", 9282 ); 9283 } 9284 9285 // Check the source vm has been terminated successful (give it '3s' to settle) 9286 thread::sleep(std::time::Duration::new(3, 0)); 9287 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9288 print_and_panic( 9289 src_child, 9290 dest_child, 9291 None, 9292 "source VM was not terminated successfully.", 9293 ); 9294 }; 9295 9296 // Post live-migration check to make sure the destination VM is funcational 9297 let r = std::panic::catch_unwind(|| { 9298 // Perform same checks to validate VM has been properly migrated 9299 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9300 #[cfg(target_arch = "x86_64")] 9301 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 9302 #[cfg(target_arch = "aarch64")] 9303 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9304 9305 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9306 9307 // Perform NUMA related checks 9308 { 9309 #[cfg(target_arch = "aarch64")] 9310 { 9311 guest.check_numa_common( 9312 Some(&[960_000, 960_000, 1_920_000]), 9313 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9314 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9315 ); 9316 } 9317 9318 // AArch64 currently does not support hotplug, and therefore we only 9319 // test hotplug-related function on x86_64 here. 9320 #[cfg(target_arch = "x86_64")] 9321 { 9322 guest.check_numa_common( 9323 Some(&[1_920_000, 1_920_000, 2_880_000]), 9324 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9325 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9326 ); 9327 9328 guest.enable_memory_hotplug(); 9329 9330 // Resize every memory zone and check each associated NUMA node 9331 // has been assigned the right amount of memory. 9332 resize_zone_command(&dest_api_socket, "mem0", "4G"); 9333 resize_zone_command(&dest_api_socket, "mem1", "4G"); 9334 resize_zone_command(&dest_api_socket, "mem2", "4G"); 9335 // Resize to the maximum amount of CPUs and check each NUMA 9336 // node has been assigned the right CPUs set. 9337 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 9338 thread::sleep(std::time::Duration::new(5, 0)); 9339 9340 guest.check_numa_common( 9341 Some(&[3_840_000, 3_840_000, 3_840_000]), 9342 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 9343 None, 9344 ); 9345 } 9346 } 9347 }); 9348 9349 // Clean-up the destination VM and make sure it terminated correctly 9350 let _ = dest_child.kill(); 9351 let dest_output = dest_child.wait_with_output().unwrap(); 9352 handle_child_output(r, &dest_output); 9353 9354 // Check the destination VM has the expected 'concole_text' from its output 9355 let r = std::panic::catch_unwind(|| { 9356 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9357 }); 9358 handle_child_output(r, &dest_output); 9359 } 9360 9361 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 9362 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9363 let guest = Guest::new(Box::new(focal)); 9364 let kernel_path = direct_kernel_boot_path(); 9365 let console_text = String::from("On a branch floating down river a cricket, singing."); 9366 let net_id = "net123"; 9367 let net_params = format!( 9368 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9369 net_id, guest.network.guest_mac, guest.network.host_ip 9370 ); 9371 9372 let memory_param: &[&str] = if local { 9373 &["--memory", "size=4G,shared=on"] 9374 } else { 9375 &["--memory", "size=4G"] 9376 }; 9377 9378 let boot_vcpus = 2; 9379 let max_vcpus = 4; 9380 9381 let pmem_temp_file = TempFile::new().unwrap(); 9382 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9383 std::process::Command::new("mkfs.ext4") 9384 .arg(pmem_temp_file.as_path()) 9385 .output() 9386 .expect("Expect creating disk image to succeed"); 9387 let pmem_path = String::from("/dev/pmem0"); 9388 9389 // Start the source VM 9390 let src_vm_path = if !upgrade_test { 9391 clh_command("cloud-hypervisor") 9392 } else { 9393 cloud_hypervisor_release_path() 9394 }; 9395 let src_api_socket = temp_api_path(&guest.tmp_dir); 9396 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9397 src_vm_cmd 9398 .args([ 9399 "--cpus", 9400 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9401 ]) 9402 .args(memory_param) 9403 .args(["--kernel", kernel_path.to_str().unwrap()]) 9404 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9405 .default_disks() 9406 .args(["--net", net_params.as_str()]) 9407 .args(["--api-socket", &src_api_socket]) 9408 .args([ 9409 "--pmem", 9410 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9411 ]) 9412 .args(["--watchdog"]); 9413 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9414 9415 // Start the destination VM 9416 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9417 dest_api_socket.push_str(".dest"); 9418 let mut dest_child = GuestCommand::new(&guest) 9419 .args(["--api-socket", &dest_api_socket]) 9420 .capture_output() 9421 .spawn() 9422 .unwrap(); 9423 9424 let r = std::panic::catch_unwind(|| { 9425 guest.wait_vm_boot(None).unwrap(); 9426 9427 // Make sure the source VM is functaionl 9428 // Check the number of vCPUs 9429 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9430 // Check the guest RAM 9431 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9432 // Check the guest virtio-devices, e.g. block, rng, console, and net 9433 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9434 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9435 // to make sure that removing and adding back the virtio-net device does 9436 // not break the live-migration support for virtio-pci. 9437 #[cfg(target_arch = "x86_64")] 9438 { 9439 assert!(remote_command( 9440 &src_api_socket, 9441 "remove-device", 9442 Some(net_id), 9443 )); 9444 thread::sleep(std::time::Duration::new(10, 0)); 9445 9446 // Plug the virtio-net device again 9447 assert!(remote_command( 9448 &src_api_socket, 9449 "add-net", 9450 Some(net_params.as_str()), 9451 )); 9452 thread::sleep(std::time::Duration::new(10, 0)); 9453 } 9454 9455 // Enable watchdog and ensure its functional 9456 let mut expected_reboot_count = 1; 9457 // Enable the watchdog with a 15s timeout 9458 enable_guest_watchdog(&guest, 15); 9459 // Reboot and check that systemd has activated the watchdog 9460 guest.ssh_command("sudo reboot").unwrap(); 9461 guest.wait_vm_boot(None).unwrap(); 9462 expected_reboot_count += 1; 9463 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9464 assert_eq!( 9465 guest 9466 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 9467 .unwrap() 9468 .trim() 9469 .parse::<u32>() 9470 .unwrap_or_default(), 9471 2 9472 ); 9473 // Allow some normal time to elapse to check we don't get spurious reboots 9474 thread::sleep(std::time::Duration::new(40, 0)); 9475 // Check no reboot 9476 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9477 9478 // Start the live-migration 9479 let migration_socket = String::from( 9480 guest 9481 .tmp_dir 9482 .as_path() 9483 .join("live-migration.sock") 9484 .to_str() 9485 .unwrap(), 9486 ); 9487 9488 assert!( 9489 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9490 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9491 ); 9492 }); 9493 9494 // Check and report any errors occurred during the live-migration 9495 if r.is_err() { 9496 print_and_panic( 9497 src_child, 9498 dest_child, 9499 None, 9500 "Error occurred during live-migration", 9501 ); 9502 } 9503 9504 // Check the source vm has been terminated successful (give it '3s' to settle) 9505 thread::sleep(std::time::Duration::new(3, 0)); 9506 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9507 print_and_panic( 9508 src_child, 9509 dest_child, 9510 None, 9511 "source VM was not terminated successfully.", 9512 ); 9513 }; 9514 9515 // Post live-migration check to make sure the destination VM is funcational 9516 let r = std::panic::catch_unwind(|| { 9517 // Perform same checks to validate VM has been properly migrated 9518 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9519 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9520 9521 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9522 9523 // Perform checks on watchdog 9524 let mut expected_reboot_count = 2; 9525 9526 // Allow some normal time to elapse to check we don't get spurious reboots 9527 thread::sleep(std::time::Duration::new(40, 0)); 9528 // Check no reboot 9529 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9530 9531 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 9532 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 9533 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 9534 guest.wait_vm_boot(Some(50)).unwrap(); 9535 // Check a reboot is triggered by the watchdog 9536 expected_reboot_count += 1; 9537 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9538 9539 #[cfg(target_arch = "x86_64")] 9540 { 9541 // Now pause the VM and remain offline for 30s 9542 assert!(remote_command(&dest_api_socket, "pause", None)); 9543 thread::sleep(std::time::Duration::new(30, 0)); 9544 assert!(remote_command(&dest_api_socket, "resume", None)); 9545 9546 // Check no reboot 9547 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9548 } 9549 }); 9550 9551 // Clean-up the destination VM and make sure it terminated correctly 9552 let _ = dest_child.kill(); 9553 let dest_output = dest_child.wait_with_output().unwrap(); 9554 handle_child_output(r, &dest_output); 9555 9556 // Check the destination VM has the expected 'concole_text' from its output 9557 let r = std::panic::catch_unwind(|| { 9558 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9559 }); 9560 handle_child_output(r, &dest_output); 9561 } 9562 9563 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 9564 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9565 let ovs_guest = Guest::new(Box::new(ovs_focal)); 9566 9567 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9568 let migration_guest = Guest::new(Box::new(migration_focal)); 9569 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 9570 9571 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 9572 let (mut ovs_child, mut src_child) = 9573 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 9574 9575 // Start the destination VM 9576 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 9577 dest_api_socket.push_str(".dest"); 9578 let mut dest_child = GuestCommand::new(&migration_guest) 9579 .args(["--api-socket", &dest_api_socket]) 9580 .capture_output() 9581 .spawn() 9582 .unwrap(); 9583 9584 let r = std::panic::catch_unwind(|| { 9585 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 9586 thread::sleep(std::time::Duration::new(1, 0)); 9587 9588 // Start the live-migration 9589 let migration_socket = String::from( 9590 migration_guest 9591 .tmp_dir 9592 .as_path() 9593 .join("live-migration.sock") 9594 .to_str() 9595 .unwrap(), 9596 ); 9597 9598 assert!( 9599 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9600 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9601 ); 9602 }); 9603 9604 // Check and report any errors occurred during the live-migration 9605 if r.is_err() { 9606 print_and_panic( 9607 src_child, 9608 dest_child, 9609 Some(ovs_child), 9610 "Error occurred during live-migration", 9611 ); 9612 } 9613 9614 // Check the source vm has been terminated successful (give it '3s' to settle) 9615 thread::sleep(std::time::Duration::new(3, 0)); 9616 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9617 print_and_panic( 9618 src_child, 9619 dest_child, 9620 Some(ovs_child), 9621 "source VM was not terminated successfully.", 9622 ); 9623 }; 9624 9625 // Post live-migration check to make sure the destination VM is funcational 9626 let r = std::panic::catch_unwind(|| { 9627 // Perform same checks to validate VM has been properly migrated 9628 // Spawn a new netcat listener in the OVS VM 9629 let guest_ip = ovs_guest.network.guest_ip.clone(); 9630 thread::spawn(move || { 9631 ssh_command_ip( 9632 "nc -l 12345", 9633 &guest_ip, 9634 DEFAULT_SSH_RETRIES, 9635 DEFAULT_SSH_TIMEOUT, 9636 ) 9637 .unwrap(); 9638 }); 9639 9640 // Wait for the server to be listening 9641 thread::sleep(std::time::Duration::new(5, 0)); 9642 9643 // And check the connection is still functional after live-migration 9644 migration_guest 9645 .ssh_command("nc -vz 172.100.0.1 12345") 9646 .unwrap(); 9647 }); 9648 9649 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 9650 let _ = dest_child.kill(); 9651 let _ = ovs_child.kill(); 9652 let dest_output = dest_child.wait_with_output().unwrap(); 9653 let ovs_output = ovs_child.wait_with_output().unwrap(); 9654 9655 cleanup_ovs_dpdk(); 9656 9657 handle_child_output(r, &dest_output); 9658 handle_child_output(Ok(()), &ovs_output); 9659 } 9660 9661 mod live_migration_parallel { 9662 use super::*; 9663 #[test] 9664 fn test_live_migration_basic() { 9665 _test_live_migration(false, false) 9666 } 9667 9668 #[test] 9669 fn test_live_migration_local() { 9670 _test_live_migration(false, true) 9671 } 9672 9673 #[test] 9674 #[cfg(not(feature = "mshv"))] 9675 fn test_live_migration_numa() { 9676 _test_live_migration_numa(false, false) 9677 } 9678 9679 #[test] 9680 #[cfg(not(feature = "mshv"))] 9681 fn test_live_migration_numa_local() { 9682 _test_live_migration_numa(false, true) 9683 } 9684 9685 #[test] 9686 fn test_live_migration_watchdog() { 9687 _test_live_migration_watchdog(false, false) 9688 } 9689 9690 #[test] 9691 fn test_live_migration_watchdog_local() { 9692 _test_live_migration_watchdog(false, true) 9693 } 9694 9695 #[test] 9696 fn test_live_migration_balloon() { 9697 _test_live_migration_balloon(false, false) 9698 } 9699 9700 #[test] 9701 fn test_live_migration_balloon_local() { 9702 _test_live_migration_balloon(false, true) 9703 } 9704 9705 #[test] 9706 #[ignore = "See #6134"] 9707 fn test_live_upgrade_basic() { 9708 _test_live_migration(true, false) 9709 } 9710 9711 #[test] 9712 #[ignore = "See #6134"] 9713 fn test_live_upgrade_local() { 9714 _test_live_migration(true, true) 9715 } 9716 9717 #[test] 9718 #[ignore = "See #6134"] 9719 #[cfg(not(feature = "mshv"))] 9720 fn test_live_upgrade_numa() { 9721 _test_live_migration_numa(true, false) 9722 } 9723 9724 #[test] 9725 #[ignore = "See #6134"] 9726 #[cfg(not(feature = "mshv"))] 9727 fn test_live_upgrade_numa_local() { 9728 _test_live_migration_numa(true, true) 9729 } 9730 9731 #[test] 9732 #[ignore = "See #6134"] 9733 fn test_live_upgrade_watchdog() { 9734 _test_live_migration_watchdog(true, false) 9735 } 9736 9737 #[test] 9738 #[ignore = "See #6134"] 9739 fn test_live_upgrade_watchdog_local() { 9740 _test_live_migration_watchdog(true, true) 9741 } 9742 9743 #[test] 9744 #[ignore = "See #6134"] 9745 fn test_live_upgrade_balloon() { 9746 _test_live_migration_balloon(true, false) 9747 } 9748 9749 #[test] 9750 #[ignore = "See #6134"] 9751 fn test_live_upgrade_balloon_local() { 9752 _test_live_migration_balloon(true, true) 9753 } 9754 } 9755 9756 mod live_migration_sequential { 9757 #[cfg(target_arch = "x86_64")] 9758 #[cfg(not(feature = "mshv"))] 9759 use super::*; 9760 9761 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 9762 #[test] 9763 #[ignore = "See #5532"] 9764 #[cfg(target_arch = "x86_64")] 9765 #[cfg(not(feature = "mshv"))] 9766 fn test_live_migration_ovs_dpdk() { 9767 _test_live_migration_ovs_dpdk(false, false); 9768 } 9769 9770 #[test] 9771 #[cfg(target_arch = "x86_64")] 9772 #[cfg(not(feature = "mshv"))] 9773 fn test_live_migration_ovs_dpdk_local() { 9774 _test_live_migration_ovs_dpdk(false, true); 9775 } 9776 9777 #[test] 9778 #[ignore = "See #5532"] 9779 #[cfg(target_arch = "x86_64")] 9780 #[cfg(not(feature = "mshv"))] 9781 fn test_live_upgrade_ovs_dpdk() { 9782 _test_live_migration_ovs_dpdk(true, false); 9783 } 9784 9785 #[test] 9786 #[ignore = "See #5532"] 9787 #[cfg(target_arch = "x86_64")] 9788 #[cfg(not(feature = "mshv"))] 9789 fn test_live_upgrade_ovs_dpdk_local() { 9790 _test_live_migration_ovs_dpdk(true, true); 9791 } 9792 } 9793 } 9794 9795 #[cfg(target_arch = "aarch64")] 9796 mod aarch64_acpi { 9797 use crate::*; 9798 9799 #[test] 9800 fn test_simple_launch_acpi() { 9801 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9802 9803 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 9804 let guest = Guest::new(disk_config); 9805 9806 let mut child = GuestCommand::new(&guest) 9807 .args(["--cpus", "boot=1"]) 9808 .args(["--memory", "size=512M"]) 9809 .args(["--kernel", edk2_path().to_str().unwrap()]) 9810 .default_disks() 9811 .default_net() 9812 .args(["--serial", "tty", "--console", "off"]) 9813 .capture_output() 9814 .spawn() 9815 .unwrap(); 9816 9817 let r = std::panic::catch_unwind(|| { 9818 guest.wait_vm_boot(Some(120)).unwrap(); 9819 9820 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 9821 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 9822 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 9823 }); 9824 9825 let _ = child.kill(); 9826 let output = child.wait_with_output().unwrap(); 9827 9828 handle_child_output(r, &output); 9829 }); 9830 } 9831 9832 #[test] 9833 fn test_guest_numa_nodes_acpi() { 9834 _test_guest_numa_nodes(true); 9835 } 9836 9837 #[test] 9838 fn test_cpu_topology_421_acpi() { 9839 test_cpu_topology(4, 2, 1, true); 9840 } 9841 9842 #[test] 9843 fn test_cpu_topology_142_acpi() { 9844 test_cpu_topology(1, 4, 2, true); 9845 } 9846 9847 #[test] 9848 fn test_cpu_topology_262_acpi() { 9849 test_cpu_topology(2, 6, 2, true); 9850 } 9851 9852 #[test] 9853 fn test_power_button_acpi() { 9854 _test_power_button(true); 9855 } 9856 9857 #[test] 9858 fn test_virtio_iommu() { 9859 _test_virtio_iommu(true) 9860 } 9861 } 9862 9863 mod rate_limiter { 9864 use super::*; 9865 9866 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 9867 // compared to given 'limit' rate. 9868 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 9869 let upper_limit = limit * (1_f64 + difference); 9870 let lower_limit = limit * (1_f64 - difference); 9871 9872 if measured > lower_limit && measured < upper_limit { 9873 return true; 9874 } 9875 9876 eprintln!( 9877 "\n\n==== Start 'check_rate_limit' failed ==== \ 9878 \n\nmeasured={measured}, , lower_limit={lower_limit}, upper_limit={upper_limit} \ 9879 \n\n==== End 'check_rate_limit' failed ====\n\n" 9880 ); 9881 9882 false 9883 } 9884 9885 fn _test_rate_limiter_net(rx: bool) { 9886 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9887 let guest = Guest::new(Box::new(focal)); 9888 9889 let test_timeout = 10; 9890 let num_queues = 2; 9891 let queue_size = 256; 9892 let bw_size = 10485760_u64; // bytes 9893 let bw_refill_time = 100; // ms 9894 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 9895 9896 let net_params = format!( 9897 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 9898 guest.network.guest_mac, 9899 guest.network.host_ip, 9900 num_queues, 9901 queue_size, 9902 bw_size, 9903 bw_refill_time, 9904 ); 9905 9906 let mut child = GuestCommand::new(&guest) 9907 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 9908 .args(["--memory", "size=4G"]) 9909 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9910 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9911 .default_disks() 9912 .args(["--net", net_params.as_str()]) 9913 .capture_output() 9914 .spawn() 9915 .unwrap(); 9916 9917 let r = std::panic::catch_unwind(|| { 9918 guest.wait_vm_boot(None).unwrap(); 9919 let measured_bps = 9920 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, true) 9921 .unwrap(); 9922 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 9923 }); 9924 9925 let _ = child.kill(); 9926 let output = child.wait_with_output().unwrap(); 9927 handle_child_output(r, &output); 9928 } 9929 9930 #[test] 9931 fn test_rate_limiter_net_rx() { 9932 _test_rate_limiter_net(true); 9933 } 9934 9935 #[test] 9936 fn test_rate_limiter_net_tx() { 9937 _test_rate_limiter_net(false); 9938 } 9939 9940 fn _test_rate_limiter_block(bandwidth: bool, num_queues: u32) { 9941 let test_timeout = 10; 9942 let fio_ops = FioOps::RandRW; 9943 9944 let bw_size = if bandwidth { 9945 10485760_u64 // bytes 9946 } else { 9947 100_u64 // I/O 9948 }; 9949 let bw_refill_time = 100; // ms 9950 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 9951 9952 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9953 let guest = Guest::new(Box::new(focal)); 9954 let api_socket = temp_api_path(&guest.tmp_dir); 9955 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 9956 let blk_rate_limiter_test_img = 9957 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 9958 9959 // Create the test block image 9960 assert!(exec_host_command_output(&format!( 9961 "dd if=/dev/zero of={blk_rate_limiter_test_img} bs=1M count=1024" 9962 )) 9963 .status 9964 .success()); 9965 9966 let test_blk_params = if bandwidth { 9967 format!( 9968 "path={blk_rate_limiter_test_img},num_queues={num_queues},bw_size={bw_size},bw_refill_time={bw_refill_time}" 9969 ) 9970 } else { 9971 format!( 9972 "path={blk_rate_limiter_test_img},num_queues={num_queues},ops_size={bw_size},ops_refill_time={bw_refill_time}" 9973 ) 9974 }; 9975 9976 let mut child = GuestCommand::new(&guest) 9977 .args(["--cpus", &format!("boot={num_queues}")]) 9978 .args(["--memory", "size=4G"]) 9979 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9980 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9981 .args([ 9982 "--disk", 9983 format!( 9984 "path={}", 9985 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 9986 ) 9987 .as_str(), 9988 format!( 9989 "path={}", 9990 guest.disk_config.disk(DiskType::CloudInit).unwrap() 9991 ) 9992 .as_str(), 9993 test_blk_params.as_str(), 9994 ]) 9995 .default_net() 9996 .args(["--api-socket", &api_socket]) 9997 .capture_output() 9998 .spawn() 9999 .unwrap(); 10000 10001 let r = std::panic::catch_unwind(|| { 10002 guest.wait_vm_boot(None).unwrap(); 10003 10004 let fio_command = format!( 10005 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 10006 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10007 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10008 ); 10009 let output = guest.ssh_command(&fio_command).unwrap(); 10010 10011 // Parse fio output 10012 let measured_rate = if bandwidth { 10013 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 10014 } else { 10015 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 10016 }; 10017 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10018 }); 10019 10020 let _ = child.kill(); 10021 let output = child.wait_with_output().unwrap(); 10022 handle_child_output(r, &output); 10023 } 10024 10025 fn _test_rate_limiter_group_block(bandwidth: bool, num_queues: u32, num_disks: u32) { 10026 let test_timeout = 10; 10027 let fio_ops = FioOps::RandRW; 10028 10029 let bw_size = if bandwidth { 10030 10485760_u64 // bytes 10031 } else { 10032 100_u64 // I/O 10033 }; 10034 let bw_refill_time = 100; // ms 10035 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10036 10037 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10038 let guest = Guest::new(Box::new(focal)); 10039 let api_socket = temp_api_path(&guest.tmp_dir); 10040 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10041 10042 let rate_limit_group_arg = if bandwidth { 10043 format!("id=group0,bw_size={bw_size},bw_refill_time={bw_refill_time}") 10044 } else { 10045 format!("id=group0,ops_size={bw_size},ops_refill_time={bw_refill_time}") 10046 }; 10047 10048 let mut disk_args = vec![ 10049 "--disk".to_string(), 10050 format!( 10051 "path={}", 10052 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10053 ), 10054 format!( 10055 "path={}", 10056 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10057 ), 10058 ]; 10059 10060 for i in 0..num_disks { 10061 let test_img_path = String::from( 10062 test_img_dir 10063 .as_path() 10064 .join(format!("blk{}.img", i)) 10065 .to_str() 10066 .unwrap(), 10067 ); 10068 10069 assert!(exec_host_command_output(&format!( 10070 "dd if=/dev/zero of={test_img_path} bs=1M count=1024" 10071 )) 10072 .status 10073 .success()); 10074 10075 disk_args.push(format!( 10076 "path={test_img_path},num_queues={num_queues},rate_limit_group=group0" 10077 )); 10078 } 10079 10080 let mut child = GuestCommand::new(&guest) 10081 .args(["--cpus", &format!("boot={}", num_queues * num_disks)]) 10082 .args(["--memory", "size=4G"]) 10083 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10084 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10085 .args(["--rate-limit-group", &rate_limit_group_arg]) 10086 .args(disk_args) 10087 .default_net() 10088 .args(["--api-socket", &api_socket]) 10089 .capture_output() 10090 .spawn() 10091 .unwrap(); 10092 10093 let r = std::panic::catch_unwind(|| { 10094 guest.wait_vm_boot(None).unwrap(); 10095 10096 let mut fio_command = format!( 10097 "sudo fio --name=global --output-format=json \ 10098 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10099 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10100 ); 10101 10102 // Generate additional argument for each disk: 10103 // --name=job0 --filename=/dev/vdc \ 10104 // --name=job1 --filename=/dev/vdd \ 10105 // --name=job2 --filename=/dev/vde \ 10106 // ... 10107 for i in 0..num_disks { 10108 let c: char = 'c'; 10109 let arg = format!( 10110 " --name=job{i} --filename=/dev/vd{}", 10111 char::from_u32((c as u32) + i).unwrap() 10112 ); 10113 fio_command += &arg; 10114 } 10115 let output = guest.ssh_command(&fio_command).unwrap(); 10116 10117 // Parse fio output 10118 let measured_rate = if bandwidth { 10119 parse_fio_output(&output, &fio_ops, num_queues * num_disks).unwrap() 10120 } else { 10121 parse_fio_output_iops(&output, &fio_ops, num_queues * num_disks).unwrap() 10122 }; 10123 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10124 }); 10125 10126 let _ = child.kill(); 10127 let output = child.wait_with_output().unwrap(); 10128 handle_child_output(r, &output); 10129 } 10130 10131 #[test] 10132 fn test_rate_limiter_block_bandwidth() { 10133 _test_rate_limiter_block(true, 1); 10134 _test_rate_limiter_block(true, 2) 10135 } 10136 10137 #[test] 10138 fn test_rate_limiter_group_block_bandwidth() { 10139 _test_rate_limiter_group_block(true, 1, 1); 10140 _test_rate_limiter_group_block(true, 2, 1); 10141 _test_rate_limiter_group_block(true, 1, 2); 10142 _test_rate_limiter_group_block(true, 2, 2); 10143 } 10144 10145 #[test] 10146 fn test_rate_limiter_block_iops() { 10147 _test_rate_limiter_block(false, 1); 10148 _test_rate_limiter_block(false, 2); 10149 } 10150 10151 #[test] 10152 fn test_rate_limiter_group_block_iops() { 10153 _test_rate_limiter_group_block(false, 1, 1); 10154 _test_rate_limiter_group_block(false, 2, 1); 10155 _test_rate_limiter_group_block(false, 1, 2); 10156 _test_rate_limiter_group_block(false, 2, 2); 10157 } 10158 } 10159