1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 use std::sync::Mutex; 28 use std::thread; 29 use test_infra::*; 30 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 31 use wait_timeout::ChildExt; 32 33 // Constant taken from the VMM crate. 34 const MAX_NUM_PCI_SEGMENTS: u16 = 96; 35 36 #[cfg(target_arch = "x86_64")] 37 mod x86_64 { 38 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 39 pub const JAMMY_NVIDIA_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-nvidia.raw"; 40 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 41 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 42 "focal-server-cloudimg-amd64-custom-20210609-0-backing.qcow2"; 43 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 44 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 45 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20230119-0.raw"; 46 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2022-amd64-2.raw"; 47 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 48 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 49 } 50 51 #[cfg(target_arch = "x86_64")] 52 use x86_64::*; 53 54 #[cfg(target_arch = "aarch64")] 55 mod aarch64 { 56 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 57 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 58 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 59 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 60 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 61 "focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2"; 62 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 63 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 64 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 65 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 66 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 67 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 68 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 69 } 70 71 #[cfg(target_arch = "aarch64")] 72 use aarch64::*; 73 74 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 75 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 76 77 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 78 79 // This enum exists to make it more convenient to 80 // implement test for both D-Bus and REST APIs. 81 enum TargetApi { 82 // API socket 83 HttpApi(String), 84 // well known service name, object path 85 DBusApi(String, String), 86 } 87 88 impl TargetApi { 89 fn new_http_api(tmp_dir: &TempDir) -> Self { 90 Self::HttpApi(temp_api_path(tmp_dir)) 91 } 92 93 fn new_dbus_api(tmp_dir: &TempDir) -> Self { 94 // `tmp_dir` is in the form of "/tmp/chXXXXXX" 95 // and we take the `chXXXXXX` part as a unique identifier for the guest 96 let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); 97 98 Self::DBusApi( 99 format!("org.cloudhypervisor.{id}"), 100 format!("/org/cloudhypervisor/{id}"), 101 ) 102 } 103 104 fn guest_args(&self) -> Vec<String> { 105 match self { 106 TargetApi::HttpApi(api_socket) => { 107 vec![format!("--api-socket={}", api_socket.as_str())] 108 } 109 TargetApi::DBusApi(service_name, object_path) => { 110 vec![ 111 format!("--dbus-service-name={}", service_name.as_str()), 112 format!("--dbus-object-path={}", object_path.as_str()), 113 ] 114 } 115 } 116 } 117 118 fn remote_args(&self) -> Vec<String> { 119 // `guest_args` and `remote_args` are consistent with each other 120 self.guest_args() 121 } 122 123 fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { 124 let mut cmd = Command::new(clh_command("ch-remote")); 125 cmd.args(self.remote_args()); 126 cmd.arg(command); 127 128 if let Some(arg) = arg { 129 cmd.arg(arg); 130 } 131 132 let output = cmd.output().unwrap(); 133 if output.status.success() { 134 true 135 } else { 136 eprintln!("Error running ch-remote command: {:?}", &cmd); 137 let stderr = String::from_utf8_lossy(&output.stderr); 138 eprintln!("stderr: {stderr}"); 139 false 140 } 141 } 142 } 143 144 // Start cloud-hypervisor with no VM parameters, only the API server running. 145 // From the API: Create a VM, boot it and check that it looks as expected. 146 fn _test_api_create_boot(target_api: TargetApi, guest: Guest) { 147 let mut child = GuestCommand::new(&guest) 148 .args(target_api.guest_args()) 149 .capture_output() 150 .spawn() 151 .unwrap(); 152 153 thread::sleep(std::time::Duration::new(1, 0)); 154 155 // Verify API server is running 156 assert!(target_api.remote_command("ping", None)); 157 158 // Create the VM first 159 let cpu_count: u8 = 4; 160 let request_body = guest.api_create_body( 161 cpu_count, 162 direct_kernel_boot_path().to_str().unwrap(), 163 DIRECT_KERNEL_BOOT_CMDLINE, 164 ); 165 166 let temp_config_path = guest.tmp_dir.as_path().join("config"); 167 std::fs::write(&temp_config_path, request_body).unwrap(); 168 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 169 170 assert!(target_api.remote_command("create", Some(create_config),)); 171 172 // Then boot it 173 assert!(target_api.remote_command("boot", None)); 174 thread::sleep(std::time::Duration::new(20, 0)); 175 176 let r = std::panic::catch_unwind(|| { 177 // Check that the VM booted as expected 178 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 179 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 180 }); 181 182 let _ = child.kill(); 183 let output = child.wait_with_output().unwrap(); 184 185 handle_child_output(r, &output); 186 } 187 188 // Start cloud-hypervisor with no VM parameters, only the API server running. 189 // From the API: Create a VM, boot it and check it can be shutdown and then 190 // booted again 191 fn _test_api_shutdown(target_api: TargetApi, guest: Guest) { 192 let mut child = GuestCommand::new(&guest) 193 .args(target_api.guest_args()) 194 .capture_output() 195 .spawn() 196 .unwrap(); 197 198 thread::sleep(std::time::Duration::new(1, 0)); 199 200 // Verify API server is running 201 assert!(target_api.remote_command("ping", None)); 202 203 // Create the VM first 204 let cpu_count: u8 = 4; 205 let request_body = guest.api_create_body( 206 cpu_count, 207 direct_kernel_boot_path().to_str().unwrap(), 208 DIRECT_KERNEL_BOOT_CMDLINE, 209 ); 210 211 let temp_config_path = guest.tmp_dir.as_path().join("config"); 212 std::fs::write(&temp_config_path, request_body).unwrap(); 213 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 214 215 let r = std::panic::catch_unwind(|| { 216 assert!(target_api.remote_command("create", Some(create_config))); 217 218 // Then boot it 219 assert!(target_api.remote_command("boot", None)); 220 221 guest.wait_vm_boot(None).unwrap(); 222 223 // Check that the VM booted as expected 224 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 225 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 226 227 // Sync and shutdown without powering off to prevent filesystem 228 // corruption. 229 guest.ssh_command("sync").unwrap(); 230 guest.ssh_command("sudo shutdown -H now").unwrap(); 231 232 // Wait for the guest to be fully shutdown 233 thread::sleep(std::time::Duration::new(20, 0)); 234 235 // Then shut it down 236 assert!(target_api.remote_command("shutdown", None)); 237 238 // Then boot it again 239 assert!(target_api.remote_command("boot", None)); 240 241 guest.wait_vm_boot(None).unwrap(); 242 243 // Check that the VM booted as expected 244 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 245 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 246 }); 247 248 let _ = child.kill(); 249 let output = child.wait_with_output().unwrap(); 250 251 handle_child_output(r, &output); 252 } 253 254 // Start cloud-hypervisor with no VM parameters, only the API server running. 255 // From the API: Create a VM, boot it and check it can be deleted and then recreated 256 // booted again. 257 fn _test_api_delete(target_api: TargetApi, guest: Guest) { 258 let mut child = GuestCommand::new(&guest) 259 .args(target_api.guest_args()) 260 .capture_output() 261 .spawn() 262 .unwrap(); 263 264 thread::sleep(std::time::Duration::new(1, 0)); 265 266 // Verify API server is running 267 assert!(target_api.remote_command("ping", None)); 268 269 // Create the VM first 270 let cpu_count: u8 = 4; 271 let request_body = guest.api_create_body( 272 cpu_count, 273 direct_kernel_boot_path().to_str().unwrap(), 274 DIRECT_KERNEL_BOOT_CMDLINE, 275 ); 276 let temp_config_path = guest.tmp_dir.as_path().join("config"); 277 std::fs::write(&temp_config_path, request_body).unwrap(); 278 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 279 280 let r = std::panic::catch_unwind(|| { 281 assert!(target_api.remote_command("create", Some(create_config))); 282 283 // Then boot it 284 assert!(target_api.remote_command("boot", None)); 285 286 guest.wait_vm_boot(None).unwrap(); 287 288 // Check that the VM booted as expected 289 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 290 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 291 292 // Sync and shutdown without powering off to prevent filesystem 293 // corruption. 294 guest.ssh_command("sync").unwrap(); 295 guest.ssh_command("sudo shutdown -H now").unwrap(); 296 297 // Wait for the guest to be fully shutdown 298 thread::sleep(std::time::Duration::new(20, 0)); 299 300 // Then delete it 301 assert!(target_api.remote_command("delete", None)); 302 303 assert!(target_api.remote_command("create", Some(create_config))); 304 305 // Then boot it again 306 assert!(target_api.remote_command("boot", None)); 307 308 guest.wait_vm_boot(None).unwrap(); 309 310 // Check that the VM booted as expected 311 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 312 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 313 }); 314 315 let _ = child.kill(); 316 let output = child.wait_with_output().unwrap(); 317 318 handle_child_output(r, &output); 319 } 320 321 // Start cloud-hypervisor with no VM parameters, only the API server running. 322 // From the API: Create a VM, boot it and check that it looks as expected. 323 // Then we pause the VM, check that it's no longer available. 324 // Finally we resume the VM and check that it's available. 325 fn _test_api_pause_resume(target_api: TargetApi, guest: Guest) { 326 let mut child = GuestCommand::new(&guest) 327 .args(target_api.guest_args()) 328 .capture_output() 329 .spawn() 330 .unwrap(); 331 332 thread::sleep(std::time::Duration::new(1, 0)); 333 334 // Verify API server is running 335 assert!(target_api.remote_command("ping", None)); 336 337 // Create the VM first 338 let cpu_count: u8 = 4; 339 let request_body = guest.api_create_body( 340 cpu_count, 341 direct_kernel_boot_path().to_str().unwrap(), 342 DIRECT_KERNEL_BOOT_CMDLINE, 343 ); 344 345 let temp_config_path = guest.tmp_dir.as_path().join("config"); 346 std::fs::write(&temp_config_path, request_body).unwrap(); 347 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 348 349 assert!(target_api.remote_command("create", Some(create_config))); 350 351 // Then boot it 352 assert!(target_api.remote_command("boot", None)); 353 thread::sleep(std::time::Duration::new(20, 0)); 354 355 let r = std::panic::catch_unwind(|| { 356 // Check that the VM booted as expected 357 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 358 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 359 360 // We now pause the VM 361 assert!(target_api.remote_command("pause", None)); 362 363 // Check pausing again fails 364 assert!(!target_api.remote_command("pause", None)); 365 366 thread::sleep(std::time::Duration::new(2, 0)); 367 368 // SSH into the VM should fail 369 assert!(ssh_command_ip( 370 "grep -c processor /proc/cpuinfo", 371 &guest.network.guest_ip, 372 2, 373 5 374 ) 375 .is_err()); 376 377 // Resume the VM 378 assert!(target_api.remote_command("resume", None)); 379 380 // Check resuming again fails 381 assert!(!target_api.remote_command("resume", None)); 382 383 thread::sleep(std::time::Duration::new(2, 0)); 384 385 // Now we should be able to SSH back in and get the right number of CPUs 386 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 387 }); 388 389 let _ = child.kill(); 390 let output = child.wait_with_output().unwrap(); 391 392 handle_child_output(r, &output); 393 } 394 395 fn _test_pty_interaction(pty_path: PathBuf) { 396 let mut cf = std::fs::OpenOptions::new() 397 .write(true) 398 .read(true) 399 .open(pty_path) 400 .unwrap(); 401 402 // Some dumb sleeps but we don't want to write 403 // before the console is up and we don't want 404 // to try and write the next line before the 405 // login process is ready. 406 thread::sleep(std::time::Duration::new(5, 0)); 407 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 408 thread::sleep(std::time::Duration::new(2, 0)); 409 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 410 thread::sleep(std::time::Duration::new(2, 0)); 411 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 412 thread::sleep(std::time::Duration::new(2, 0)); 413 414 // read pty and ensure they have a login shell 415 // some fairly hacky workarounds to avoid looping 416 // forever in case the channel is blocked getting output 417 let ptyc = pty_read(cf); 418 let mut empty = 0; 419 let mut prev = String::new(); 420 loop { 421 thread::sleep(std::time::Duration::new(2, 0)); 422 match ptyc.try_recv() { 423 Ok(line) => { 424 empty = 0; 425 prev = prev + &line; 426 if prev.contains("test_pty_console") { 427 break; 428 } 429 } 430 Err(mpsc::TryRecvError::Empty) => { 431 empty += 1; 432 assert!(empty <= 5, "No login on pty"); 433 } 434 _ => { 435 panic!("No login on pty") 436 } 437 } 438 } 439 } 440 441 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 442 let mut workload_path = dirs::home_dir().unwrap(); 443 workload_path.push("workloads"); 444 445 let mut virtiofsd_path = workload_path; 446 virtiofsd_path.push("virtiofsd"); 447 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 448 449 let virtiofsd_socket_path = 450 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 451 452 // Start the daemon 453 let child = Command::new(virtiofsd_path.as_str()) 454 .args(["--shared-dir", shared_dir]) 455 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 456 .args(["--cache", "never"]) 457 .spawn() 458 .unwrap(); 459 460 thread::sleep(std::time::Duration::new(10, 0)); 461 462 (child, virtiofsd_socket_path) 463 } 464 465 fn prepare_vubd( 466 tmp_dir: &TempDir, 467 blk_img: &str, 468 num_queues: usize, 469 rdonly: bool, 470 direct: bool, 471 ) -> (std::process::Child, String) { 472 let mut workload_path = dirs::home_dir().unwrap(); 473 workload_path.push("workloads"); 474 475 let mut blk_file_path = workload_path; 476 blk_file_path.push(blk_img); 477 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 478 479 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 480 481 // Start the daemon 482 let child = Command::new(clh_command("vhost_user_block")) 483 .args([ 484 "--block-backend", 485 format!( 486 "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" 487 ) 488 .as_str(), 489 ]) 490 .spawn() 491 .unwrap(); 492 493 thread::sleep(std::time::Duration::new(10, 0)); 494 495 (child, vubd_socket_path) 496 } 497 498 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 499 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 500 } 501 502 fn temp_api_path(tmp_dir: &TempDir) -> String { 503 String::from( 504 tmp_dir 505 .as_path() 506 .join("cloud-hypervisor.sock") 507 .to_str() 508 .unwrap(), 509 ) 510 } 511 512 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 513 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 514 } 515 516 // Creates the directory and returns the path. 517 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 518 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 519 std::fs::create_dir(&snapshot_dir).unwrap(); 520 snapshot_dir 521 } 522 523 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 524 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 525 vmcore_file 526 } 527 528 // Creates the path for direct kernel boot and return the path. 529 // For x86_64, this function returns the vmlinux kernel path. 530 // For AArch64, this function returns the PE kernel path. 531 fn direct_kernel_boot_path() -> PathBuf { 532 let mut workload_path = dirs::home_dir().unwrap(); 533 workload_path.push("workloads"); 534 535 let mut kernel_path = workload_path; 536 #[cfg(target_arch = "x86_64")] 537 kernel_path.push("vmlinux"); 538 #[cfg(target_arch = "aarch64")] 539 kernel_path.push("Image"); 540 541 kernel_path 542 } 543 544 fn edk2_path() -> PathBuf { 545 let mut workload_path = dirs::home_dir().unwrap(); 546 workload_path.push("workloads"); 547 let mut edk2_path = workload_path; 548 edk2_path.push(OVMF_NAME); 549 550 edk2_path 551 } 552 553 fn cloud_hypervisor_release_path() -> String { 554 let mut workload_path = dirs::home_dir().unwrap(); 555 workload_path.push("workloads"); 556 557 let mut ch_release_path = workload_path; 558 #[cfg(target_arch = "x86_64")] 559 ch_release_path.push("cloud-hypervisor-static"); 560 #[cfg(target_arch = "aarch64")] 561 ch_release_path.push("cloud-hypervisor-static-aarch64"); 562 563 ch_release_path.into_os_string().into_string().unwrap() 564 } 565 566 fn prepare_vhost_user_net_daemon( 567 tmp_dir: &TempDir, 568 ip: &str, 569 tap: Option<&str>, 570 mtu: Option<u16>, 571 num_queues: usize, 572 client_mode: bool, 573 ) -> (std::process::Command, String) { 574 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 575 576 // Start the daemon 577 let mut net_params = format!( 578 "ip={ip},mask=255.255.255.0,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" 579 ); 580 581 if let Some(tap) = tap { 582 net_params.push_str(format!(",tap={tap}").as_str()); 583 } 584 585 if let Some(mtu) = mtu { 586 net_params.push_str(format!(",mtu={mtu}").as_str()); 587 } 588 589 let mut command = Command::new(clh_command("vhost_user_net")); 590 command.args(["--net-backend", net_params.as_str()]); 591 592 (command, vunet_socket_path) 593 } 594 595 fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { 596 let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); 597 let swtpm_socket_path = String::from( 598 tmp_dir 599 .as_path() 600 .join("swtpm") 601 .join("swtpm.sock") 602 .to_str() 603 .unwrap(), 604 ); 605 std::fs::create_dir(&swtpm_tpm_dir).unwrap(); 606 607 let mut swtpm_command = Command::new("swtpm"); 608 let swtpm_args = [ 609 "socket", 610 "--tpmstate", 611 &format!("dir={swtpm_tpm_dir}"), 612 "--ctrl", 613 &format!("type=unixio,path={swtpm_socket_path}"), 614 "--flags", 615 "startup-clear", 616 "--tpm2", 617 ]; 618 swtpm_command.args(swtpm_args); 619 620 (swtpm_command, swtpm_socket_path) 621 } 622 623 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 624 let mut cmd = Command::new(clh_command("ch-remote")); 625 cmd.args([&format!("--api-socket={api_socket}"), command]); 626 627 if let Some(arg) = arg { 628 cmd.arg(arg); 629 } 630 let output = cmd.output().unwrap(); 631 if output.status.success() { 632 true 633 } else { 634 eprintln!("Error running ch-remote command: {:?}", &cmd); 635 let stderr = String::from_utf8_lossy(&output.stderr); 636 eprintln!("stderr: {stderr}"); 637 false 638 } 639 } 640 641 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 642 let mut cmd = Command::new(clh_command("ch-remote")); 643 cmd.args([&format!("--api-socket={api_socket}"), command]); 644 645 if let Some(arg) = arg { 646 cmd.arg(arg); 647 } 648 649 let output = cmd.output().expect("Failed to launch ch-remote"); 650 651 (output.status.success(), output.stdout) 652 } 653 654 fn resize_command( 655 api_socket: &str, 656 desired_vcpus: Option<u8>, 657 desired_ram: Option<usize>, 658 desired_balloon: Option<usize>, 659 event_file: Option<&str>, 660 ) -> bool { 661 let mut cmd = Command::new(clh_command("ch-remote")); 662 cmd.args([&format!("--api-socket={api_socket}"), "resize"]); 663 664 if let Some(desired_vcpus) = desired_vcpus { 665 cmd.arg(format!("--cpus={desired_vcpus}")); 666 } 667 668 if let Some(desired_ram) = desired_ram { 669 cmd.arg(format!("--memory={desired_ram}")); 670 } 671 672 if let Some(desired_balloon) = desired_balloon { 673 cmd.arg(format!("--balloon={desired_balloon}")); 674 } 675 676 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 677 678 if let Some(event_path) = event_file { 679 let latest_events = [ 680 &MetaEvent { 681 event: "resizing".to_string(), 682 device_id: None, 683 }, 684 &MetaEvent { 685 event: "resized".to_string(), 686 device_id: None, 687 }, 688 ]; 689 // See: #5938 690 thread::sleep(std::time::Duration::new(1, 0)); 691 assert!(check_latest_events_exact(&latest_events, event_path)); 692 } 693 694 ret 695 } 696 697 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 698 let mut cmd = Command::new(clh_command("ch-remote")); 699 cmd.args([ 700 &format!("--api-socket={api_socket}"), 701 "resize-zone", 702 &format!("--id={id}"), 703 &format!("--size={desired_size}"), 704 ]); 705 706 cmd.status().expect("Failed to launch ch-remote").success() 707 } 708 709 // setup OVS-DPDK bridge and ports 710 fn setup_ovs_dpdk() { 711 // setup OVS-DPDK 712 assert!(exec_host_command_status("service openvswitch-switch start").success()); 713 assert!(exec_host_command_status("ovs-vsctl init").success()); 714 assert!( 715 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 716 .success() 717 ); 718 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 719 720 // Create OVS-DPDK bridge and ports 721 assert!(exec_host_command_status( 722 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 723 ) 724 .success()); 725 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 726 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 727 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 728 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 729 } 730 fn cleanup_ovs_dpdk() { 731 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 732 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 733 } 734 // Setup two guests and ensure they are connected through ovs-dpdk 735 fn setup_ovs_dpdk_guests( 736 guest1: &Guest, 737 guest2: &Guest, 738 api_socket: &str, 739 release_binary: bool, 740 ) -> (Child, Child) { 741 setup_ovs_dpdk(); 742 743 let clh_path = if !release_binary { 744 clh_command("cloud-hypervisor") 745 } else { 746 cloud_hypervisor_release_path() 747 }; 748 749 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 750 .args(["--cpus", "boot=2"]) 751 .args(["--memory", "size=0,shared=on"]) 752 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 753 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 754 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 755 .default_disks() 756 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 757 .capture_output() 758 .spawn() 759 .unwrap(); 760 761 #[cfg(target_arch = "x86_64")] 762 let guest_net_iface = "ens5"; 763 #[cfg(target_arch = "aarch64")] 764 let guest_net_iface = "enp0s5"; 765 766 let r = std::panic::catch_unwind(|| { 767 guest1.wait_vm_boot(None).unwrap(); 768 769 guest1 770 .ssh_command(&format!( 771 "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" 772 )) 773 .unwrap(); 774 guest1 775 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 776 .unwrap(); 777 778 let guest_ip = guest1.network.guest_ip.clone(); 779 thread::spawn(move || { 780 ssh_command_ip( 781 "nc -l 12345", 782 &guest_ip, 783 DEFAULT_SSH_RETRIES, 784 DEFAULT_SSH_TIMEOUT, 785 ) 786 .unwrap(); 787 }); 788 }); 789 if r.is_err() { 790 cleanup_ovs_dpdk(); 791 792 let _ = child1.kill(); 793 let output = child1.wait_with_output().unwrap(); 794 handle_child_output(r, &output); 795 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 796 } 797 798 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 799 .args(["--api-socket", api_socket]) 800 .args(["--cpus", "boot=2"]) 801 .args(["--memory", "size=0,shared=on"]) 802 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 803 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 804 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 805 .default_disks() 806 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 807 .capture_output() 808 .spawn() 809 .unwrap(); 810 811 let r = std::panic::catch_unwind(|| { 812 guest2.wait_vm_boot(None).unwrap(); 813 814 guest2 815 .ssh_command(&format!( 816 "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" 817 )) 818 .unwrap(); 819 guest2 820 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 821 .unwrap(); 822 823 // Check the connection works properly between the two VMs 824 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 825 }); 826 if r.is_err() { 827 cleanup_ovs_dpdk(); 828 829 let _ = child1.kill(); 830 let _ = child2.kill(); 831 let output = child2.wait_with_output().unwrap(); 832 handle_child_output(r, &output); 833 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 834 } 835 836 (child1, child2) 837 } 838 839 enum FwType { 840 Ovmf, 841 RustHypervisorFirmware, 842 } 843 844 fn fw_path(_fw_type: FwType) -> String { 845 let mut workload_path = dirs::home_dir().unwrap(); 846 workload_path.push("workloads"); 847 848 let mut fw_path = workload_path; 849 #[cfg(target_arch = "aarch64")] 850 fw_path.push("CLOUDHV_EFI.fd"); 851 #[cfg(target_arch = "x86_64")] 852 { 853 match _fw_type { 854 FwType::Ovmf => fw_path.push(OVMF_NAME), 855 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 856 } 857 } 858 859 fw_path.to_str().unwrap().to_string() 860 } 861 862 #[derive(Debug)] 863 struct MetaEvent { 864 event: String, 865 device_id: Option<String>, 866 } 867 868 impl MetaEvent { 869 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 870 let mut matched = false; 871 if v["event"].as_str().unwrap() == self.event { 872 if let Some(device_id) = &self.device_id { 873 if v["properties"]["id"].as_str().unwrap() == device_id { 874 matched = true 875 } 876 } else { 877 matched = true; 878 } 879 } 880 matched 881 } 882 } 883 884 // Parse the event_monitor file based on the format that each event 885 // is followed by a double newline 886 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 887 let content = fs::read(event_file).unwrap(); 888 let mut ret = Vec::new(); 889 for entry in String::from_utf8_lossy(&content) 890 .trim() 891 .split("\n\n") 892 .collect::<Vec<&str>>() 893 { 894 ret.push(serde_json::from_str(entry).unwrap()); 895 } 896 897 ret 898 } 899 900 // Return true if all events from the input 'expected_events' are matched sequentially 901 // with events from the 'event_file' 902 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 903 let json_events = parse_event_file(event_file); 904 let len = expected_events.len(); 905 let mut idx = 0; 906 for e in &json_events { 907 if idx == len { 908 break; 909 } 910 if expected_events[idx].match_with_json_event(e) { 911 idx += 1; 912 } 913 } 914 915 let ret = idx == len; 916 917 if !ret { 918 eprintln!( 919 "\n\n==== Start 'check_sequential_events' failed ==== \ 920 \n\nexpected_events={:?}\nactual_events={:?} \ 921 \n\n==== End 'check_sequential_events' failed ====", 922 expected_events, json_events, 923 ); 924 } 925 926 ret 927 } 928 929 // Return true if all events from the input 'expected_events' are matched exactly 930 // with events from the 'event_file' 931 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 932 let json_events = parse_event_file(event_file); 933 assert!(expected_events.len() <= json_events.len()); 934 let json_events = &json_events[..expected_events.len()]; 935 936 for (idx, e) in json_events.iter().enumerate() { 937 if !expected_events[idx].match_with_json_event(e) { 938 eprintln!( 939 "\n\n==== Start 'check_sequential_events_exact' failed ==== \ 940 \n\nexpected_events={:?}\nactual_events={:?} \ 941 \n\n==== End 'check_sequential_events_exact' failed ====", 942 expected_events, json_events, 943 ); 944 945 return false; 946 } 947 } 948 949 true 950 } 951 952 // Return true if events from the input 'latest_events' are matched exactly 953 // with the most recent events from the 'event_file' 954 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 955 let json_events = parse_event_file(event_file); 956 assert!(latest_events.len() <= json_events.len()); 957 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 958 959 for (idx, e) in json_events.iter().enumerate() { 960 if !latest_events[idx].match_with_json_event(e) { 961 eprintln!( 962 "\n\n==== Start 'check_latest_events_exact' failed ==== \ 963 \n\nexpected_events={:?}\nactual_events={:?} \ 964 \n\n==== End 'check_latest_events_exact' failed ====", 965 latest_events, json_events, 966 ); 967 968 return false; 969 } 970 } 971 972 true 973 } 974 975 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 976 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 977 let guest = Guest::new(Box::new(focal)); 978 let total_vcpus = threads_per_core * cores_per_package * packages; 979 let direct_kernel_boot_path = direct_kernel_boot_path(); 980 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 981 let fw_path = fw_path(FwType::RustHypervisorFirmware); 982 if use_fw { 983 kernel_path = fw_path.as_str(); 984 } 985 986 let mut child = GuestCommand::new(&guest) 987 .args([ 988 "--cpus", 989 &format!( 990 "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" 991 ), 992 ]) 993 .args(["--memory", "size=512M"]) 994 .args(["--kernel", kernel_path]) 995 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 996 .default_disks() 997 .default_net() 998 .capture_output() 999 .spawn() 1000 .unwrap(); 1001 1002 let r = std::panic::catch_unwind(|| { 1003 guest.wait_vm_boot(None).unwrap(); 1004 assert_eq!( 1005 guest.get_cpu_count().unwrap_or_default(), 1006 u32::from(total_vcpus) 1007 ); 1008 assert_eq!( 1009 guest 1010 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1011 .unwrap() 1012 .trim() 1013 .parse::<u8>() 1014 .unwrap_or(0), 1015 threads_per_core 1016 ); 1017 1018 assert_eq!( 1019 guest 1020 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1021 .unwrap() 1022 .trim() 1023 .parse::<u8>() 1024 .unwrap_or(0), 1025 cores_per_package 1026 ); 1027 1028 assert_eq!( 1029 guest 1030 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1031 .unwrap() 1032 .trim() 1033 .parse::<u8>() 1034 .unwrap_or(0), 1035 packages 1036 ); 1037 1038 #[cfg(target_arch = "x86_64")] 1039 { 1040 let mut cpu_id = 0; 1041 for package_id in 0..packages { 1042 for core_id in 0..cores_per_package { 1043 for _ in 0..threads_per_core { 1044 assert_eq!( 1045 guest 1046 .ssh_command(&format!("cat /sys/devices/system/cpu/cpu{cpu_id}/topology/physical_package_id")) 1047 .unwrap() 1048 .trim() 1049 .parse::<u8>() 1050 .unwrap_or(0), 1051 package_id 1052 ); 1053 1054 assert_eq!( 1055 guest 1056 .ssh_command(&format!( 1057 "cat /sys/devices/system/cpu/cpu{cpu_id}/topology/core_id" 1058 )) 1059 .unwrap() 1060 .trim() 1061 .parse::<u8>() 1062 .unwrap_or(0), 1063 core_id 1064 ); 1065 1066 cpu_id += 1; 1067 } 1068 } 1069 } 1070 } 1071 }); 1072 1073 let _ = child.kill(); 1074 let output = child.wait_with_output().unwrap(); 1075 1076 handle_child_output(r, &output); 1077 } 1078 1079 #[allow(unused_variables)] 1080 fn _test_guest_numa_nodes(acpi: bool) { 1081 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1082 let guest = Guest::new(Box::new(focal)); 1083 let api_socket = temp_api_path(&guest.tmp_dir); 1084 #[cfg(target_arch = "x86_64")] 1085 let kernel_path = direct_kernel_boot_path(); 1086 #[cfg(target_arch = "aarch64")] 1087 let kernel_path = if acpi { 1088 edk2_path() 1089 } else { 1090 direct_kernel_boot_path() 1091 }; 1092 1093 let mut child = GuestCommand::new(&guest) 1094 .args(["--cpus", "boot=6,max=12"]) 1095 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 1096 .args([ 1097 "--memory-zone", 1098 "id=mem0,size=1G,hotplug_size=3G", 1099 "id=mem1,size=2G,hotplug_size=3G", 1100 "id=mem2,size=3G,hotplug_size=3G", 1101 ]) 1102 .args([ 1103 "--numa", 1104 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 1105 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 1106 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 1107 ]) 1108 .args(["--kernel", kernel_path.to_str().unwrap()]) 1109 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1110 .args(["--api-socket", &api_socket]) 1111 .capture_output() 1112 .default_disks() 1113 .default_net() 1114 .spawn() 1115 .unwrap(); 1116 1117 let r = std::panic::catch_unwind(|| { 1118 guest.wait_vm_boot(None).unwrap(); 1119 1120 guest.check_numa_common( 1121 Some(&[960_000, 1_920_000, 2_880_000]), 1122 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 1123 Some(&["10 15 20", "20 10 25", "25 30 10"]), 1124 ); 1125 1126 // AArch64 currently does not support hotplug, and therefore we only 1127 // test hotplug-related function on x86_64 here. 1128 #[cfg(target_arch = "x86_64")] 1129 { 1130 guest.enable_memory_hotplug(); 1131 1132 // Resize every memory zone and check each associated NUMA node 1133 // has been assigned the right amount of memory. 1134 resize_zone_command(&api_socket, "mem0", "4G"); 1135 resize_zone_command(&api_socket, "mem1", "4G"); 1136 resize_zone_command(&api_socket, "mem2", "4G"); 1137 // Resize to the maximum amount of CPUs and check each NUMA 1138 // node has been assigned the right CPUs set. 1139 resize_command(&api_socket, Some(12), None, None, None); 1140 thread::sleep(std::time::Duration::new(5, 0)); 1141 1142 guest.check_numa_common( 1143 Some(&[3_840_000, 3_840_000, 3_840_000]), 1144 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 1145 None, 1146 ); 1147 } 1148 }); 1149 1150 let _ = child.kill(); 1151 let output = child.wait_with_output().unwrap(); 1152 1153 handle_child_output(r, &output); 1154 } 1155 1156 #[allow(unused_variables)] 1157 fn _test_power_button(acpi: bool) { 1158 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1159 let guest = Guest::new(Box::new(focal)); 1160 let mut cmd = GuestCommand::new(&guest); 1161 let api_socket = temp_api_path(&guest.tmp_dir); 1162 1163 #[cfg(target_arch = "x86_64")] 1164 let kernel_path = direct_kernel_boot_path(); 1165 #[cfg(target_arch = "aarch64")] 1166 let kernel_path = if acpi { 1167 edk2_path() 1168 } else { 1169 direct_kernel_boot_path() 1170 }; 1171 1172 cmd.args(["--cpus", "boot=1"]) 1173 .args(["--memory", "size=512M"]) 1174 .args(["--kernel", kernel_path.to_str().unwrap()]) 1175 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1176 .capture_output() 1177 .default_disks() 1178 .default_net() 1179 .args(["--api-socket", &api_socket]); 1180 1181 let child = cmd.spawn().unwrap(); 1182 1183 let r = std::panic::catch_unwind(|| { 1184 guest.wait_vm_boot(None).unwrap(); 1185 assert!(remote_command(&api_socket, "power-button", None)); 1186 }); 1187 1188 let output = child.wait_with_output().unwrap(); 1189 assert!(output.status.success()); 1190 handle_child_output(r, &output); 1191 } 1192 1193 type PrepareNetDaemon = dyn Fn( 1194 &TempDir, 1195 &str, 1196 Option<&str>, 1197 Option<u16>, 1198 usize, 1199 bool, 1200 ) -> (std::process::Command, String); 1201 1202 fn test_vhost_user_net( 1203 tap: Option<&str>, 1204 num_queues: usize, 1205 prepare_daemon: &PrepareNetDaemon, 1206 generate_host_mac: bool, 1207 client_mode_daemon: bool, 1208 ) { 1209 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1210 let guest = Guest::new(Box::new(focal)); 1211 let api_socket = temp_api_path(&guest.tmp_dir); 1212 1213 let kernel_path = direct_kernel_boot_path(); 1214 1215 let host_mac = if generate_host_mac { 1216 Some(MacAddr::local_random()) 1217 } else { 1218 None 1219 }; 1220 1221 let mtu = Some(3000); 1222 1223 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 1224 &guest.tmp_dir, 1225 &guest.network.host_ip, 1226 tap, 1227 mtu, 1228 num_queues, 1229 client_mode_daemon, 1230 ); 1231 1232 let net_params = format!( 1233 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 1234 guest.network.guest_mac, 1235 vunet_socket_path, 1236 num_queues, 1237 if let Some(host_mac) = host_mac { 1238 format!(",host_mac={host_mac}") 1239 } else { 1240 "".to_owned() 1241 }, 1242 if client_mode_daemon { 1243 "server" 1244 } else { 1245 "client" 1246 }, 1247 ); 1248 1249 let mut ch_command = GuestCommand::new(&guest); 1250 ch_command 1251 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 1252 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1253 .args(["--kernel", kernel_path.to_str().unwrap()]) 1254 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1255 .default_disks() 1256 .args(["--net", net_params.as_str()]) 1257 .args(["--api-socket", &api_socket]) 1258 .capture_output(); 1259 1260 let mut daemon_child: std::process::Child; 1261 let mut child: std::process::Child; 1262 1263 if client_mode_daemon { 1264 child = ch_command.spawn().unwrap(); 1265 // Make sure the VMM is waiting for the backend to connect 1266 thread::sleep(std::time::Duration::new(10, 0)); 1267 daemon_child = daemon_command.spawn().unwrap(); 1268 } else { 1269 daemon_child = daemon_command.spawn().unwrap(); 1270 // Make sure the backend is waiting for the VMM to connect 1271 thread::sleep(std::time::Duration::new(10, 0)); 1272 child = ch_command.spawn().unwrap(); 1273 } 1274 1275 let r = std::panic::catch_unwind(|| { 1276 guest.wait_vm_boot(None).unwrap(); 1277 1278 if let Some(tap_name) = tap { 1279 let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); 1280 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1281 } 1282 1283 if let Some(host_mac) = tap { 1284 let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); 1285 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1286 } 1287 1288 #[cfg(target_arch = "aarch64")] 1289 let iface = "enp0s4"; 1290 #[cfg(target_arch = "x86_64")] 1291 let iface = "ens4"; 1292 1293 assert_eq!( 1294 guest 1295 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 1296 .unwrap() 1297 .trim(), 1298 "3000" 1299 ); 1300 1301 // 1 network interface + default localhost ==> 2 interfaces 1302 // It's important to note that this test is fully exercising the 1303 // vhost-user-net implementation and the associated backend since 1304 // it does not define any --net network interface. That means all 1305 // the ssh communication in that test happens through the network 1306 // interface backed by vhost-user-net. 1307 assert_eq!( 1308 guest 1309 .ssh_command("ip -o link | wc -l") 1310 .unwrap() 1311 .trim() 1312 .parse::<u32>() 1313 .unwrap_or_default(), 1314 2 1315 ); 1316 1317 // The following pci devices will appear on guest with PCI-MSI 1318 // interrupt vectors assigned. 1319 // 1 virtio-console with 3 vectors: config, Rx, Tx 1320 // 1 virtio-blk with 2 vectors: config, Request 1321 // 1 virtio-blk with 2 vectors: config, Request 1322 // 1 virtio-rng with 2 vectors: config, Request 1323 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1324 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1325 // Based on the above, the total vectors should 14. 1326 #[cfg(target_arch = "x86_64")] 1327 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1328 #[cfg(target_arch = "aarch64")] 1329 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1330 assert_eq!( 1331 guest 1332 .ssh_command(grep_cmd) 1333 .unwrap() 1334 .trim() 1335 .parse::<u32>() 1336 .unwrap_or_default(), 1337 10 + (num_queues as u32) 1338 ); 1339 1340 // ACPI feature is needed. 1341 #[cfg(target_arch = "x86_64")] 1342 { 1343 guest.enable_memory_hotplug(); 1344 1345 // Add RAM to the VM 1346 let desired_ram = 1024 << 20; 1347 resize_command(&api_socket, None, Some(desired_ram), None, None); 1348 1349 thread::sleep(std::time::Duration::new(10, 0)); 1350 1351 // Here by simply checking the size (through ssh), we validate 1352 // the connection is still working, which means vhost-user-net 1353 // keeps working after the resize. 1354 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1355 } 1356 }); 1357 1358 let _ = child.kill(); 1359 let output = child.wait_with_output().unwrap(); 1360 1361 thread::sleep(std::time::Duration::new(5, 0)); 1362 let _ = daemon_child.kill(); 1363 let _ = daemon_child.wait(); 1364 1365 handle_child_output(r, &output); 1366 } 1367 1368 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1369 1370 fn test_vhost_user_blk( 1371 num_queues: usize, 1372 readonly: bool, 1373 direct: bool, 1374 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1375 ) { 1376 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1377 let guest = Guest::new(Box::new(focal)); 1378 let api_socket = temp_api_path(&guest.tmp_dir); 1379 1380 let kernel_path = direct_kernel_boot_path(); 1381 1382 let (blk_params, daemon_child) = { 1383 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1384 // Start the daemon 1385 let (daemon_child, vubd_socket_path) = 1386 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1387 1388 ( 1389 format!( 1390 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1391 ), 1392 Some(daemon_child), 1393 ) 1394 }; 1395 1396 let mut child = GuestCommand::new(&guest) 1397 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1398 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1399 .args(["--kernel", kernel_path.to_str().unwrap()]) 1400 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1401 .args([ 1402 "--disk", 1403 format!( 1404 "path={}", 1405 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1406 ) 1407 .as_str(), 1408 format!( 1409 "path={}", 1410 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1411 ) 1412 .as_str(), 1413 blk_params.as_str(), 1414 ]) 1415 .default_net() 1416 .args(["--api-socket", &api_socket]) 1417 .capture_output() 1418 .spawn() 1419 .unwrap(); 1420 1421 let r = std::panic::catch_unwind(|| { 1422 guest.wait_vm_boot(None).unwrap(); 1423 1424 // Check both if /dev/vdc exists and if the block size is 16M. 1425 assert_eq!( 1426 guest 1427 .ssh_command("lsblk | grep vdc | grep -c 16M") 1428 .unwrap() 1429 .trim() 1430 .parse::<u32>() 1431 .unwrap_or_default(), 1432 1 1433 ); 1434 1435 // Check if this block is RO or RW. 1436 assert_eq!( 1437 guest 1438 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1439 .unwrap() 1440 .trim() 1441 .parse::<u32>() 1442 .unwrap_or_default(), 1443 readonly as u32 1444 ); 1445 1446 // Check if the number of queues in /sys/block/vdc/mq matches the 1447 // expected num_queues. 1448 assert_eq!( 1449 guest 1450 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1451 .unwrap() 1452 .trim() 1453 .parse::<u32>() 1454 .unwrap_or_default(), 1455 num_queues as u32 1456 ); 1457 1458 // Mount the device 1459 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1460 guest.ssh_command("mkdir mount_image").unwrap(); 1461 guest 1462 .ssh_command( 1463 format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), 1464 ) 1465 .unwrap(); 1466 1467 // Check the content of the block device. The file "foo" should 1468 // contain "bar". 1469 assert_eq!( 1470 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1471 "bar" 1472 ); 1473 1474 // ACPI feature is needed. 1475 #[cfg(target_arch = "x86_64")] 1476 { 1477 guest.enable_memory_hotplug(); 1478 1479 // Add RAM to the VM 1480 let desired_ram = 1024 << 20; 1481 resize_command(&api_socket, None, Some(desired_ram), None, None); 1482 1483 thread::sleep(std::time::Duration::new(10, 0)); 1484 1485 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1486 1487 // Check again the content of the block device after the resize 1488 // has been performed. 1489 assert_eq!( 1490 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1491 "bar" 1492 ); 1493 } 1494 1495 // Unmount the device 1496 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1497 guest.ssh_command("rm -r mount_image").unwrap(); 1498 }); 1499 1500 let _ = child.kill(); 1501 let output = child.wait_with_output().unwrap(); 1502 1503 if let Some(mut daemon_child) = daemon_child { 1504 thread::sleep(std::time::Duration::new(5, 0)); 1505 let _ = daemon_child.kill(); 1506 let _ = daemon_child.wait(); 1507 } 1508 1509 handle_child_output(r, &output); 1510 } 1511 1512 fn test_boot_from_vhost_user_blk( 1513 num_queues: usize, 1514 readonly: bool, 1515 direct: bool, 1516 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1517 ) { 1518 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1519 let guest = Guest::new(Box::new(focal)); 1520 1521 let kernel_path = direct_kernel_boot_path(); 1522 1523 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1524 1525 let (blk_boot_params, daemon_child) = { 1526 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1527 // Start the daemon 1528 let (daemon_child, vubd_socket_path) = prepare_daemon( 1529 &guest.tmp_dir, 1530 disk_path.as_str(), 1531 num_queues, 1532 readonly, 1533 direct, 1534 ); 1535 1536 ( 1537 format!( 1538 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1539 ), 1540 Some(daemon_child), 1541 ) 1542 }; 1543 1544 let mut child = GuestCommand::new(&guest) 1545 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1546 .args(["--memory", "size=512M,shared=on"]) 1547 .args(["--kernel", kernel_path.to_str().unwrap()]) 1548 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1549 .args([ 1550 "--disk", 1551 blk_boot_params.as_str(), 1552 format!( 1553 "path={}", 1554 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1555 ) 1556 .as_str(), 1557 ]) 1558 .default_net() 1559 .capture_output() 1560 .spawn() 1561 .unwrap(); 1562 1563 let r = std::panic::catch_unwind(|| { 1564 guest.wait_vm_boot(None).unwrap(); 1565 1566 // Just check the VM booted correctly. 1567 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1568 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1569 }); 1570 let _ = child.kill(); 1571 let output = child.wait_with_output().unwrap(); 1572 1573 if let Some(mut daemon_child) = daemon_child { 1574 thread::sleep(std::time::Duration::new(5, 0)); 1575 let _ = daemon_child.kill(); 1576 let _ = daemon_child.wait(); 1577 } 1578 1579 handle_child_output(r, &output); 1580 } 1581 1582 fn _test_virtio_fs( 1583 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1584 hotplug: bool, 1585 pci_segment: Option<u16>, 1586 ) { 1587 #[cfg(target_arch = "aarch64")] 1588 let focal_image = if hotplug { 1589 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1590 } else { 1591 FOCAL_IMAGE_NAME.to_string() 1592 }; 1593 #[cfg(target_arch = "x86_64")] 1594 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1595 let focal = UbuntuDiskConfig::new(focal_image); 1596 let guest = Guest::new(Box::new(focal)); 1597 let api_socket = temp_api_path(&guest.tmp_dir); 1598 1599 let mut workload_path = dirs::home_dir().unwrap(); 1600 workload_path.push("workloads"); 1601 1602 let mut shared_dir = workload_path; 1603 shared_dir.push("shared_dir"); 1604 1605 #[cfg(target_arch = "x86_64")] 1606 let kernel_path = direct_kernel_boot_path(); 1607 #[cfg(target_arch = "aarch64")] 1608 let kernel_path = if hotplug { 1609 edk2_path() 1610 } else { 1611 direct_kernel_boot_path() 1612 }; 1613 1614 let (mut daemon_child, virtiofsd_socket_path) = 1615 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1616 1617 let mut guest_command = GuestCommand::new(&guest); 1618 guest_command 1619 .args(["--cpus", "boot=1"]) 1620 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1621 .args(["--kernel", kernel_path.to_str().unwrap()]) 1622 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1623 .default_disks() 1624 .default_net() 1625 .args(["--api-socket", &api_socket]); 1626 if pci_segment.is_some() { 1627 guest_command.args([ 1628 "--platform", 1629 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 1630 ]); 1631 } 1632 1633 let fs_params = format!( 1634 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1635 virtiofsd_socket_path, 1636 if let Some(pci_segment) = pci_segment { 1637 format!(",pci_segment={pci_segment}") 1638 } else { 1639 "".to_owned() 1640 } 1641 ); 1642 1643 if !hotplug { 1644 guest_command.args(["--fs", fs_params.as_str()]); 1645 } 1646 1647 let mut child = guest_command.capture_output().spawn().unwrap(); 1648 1649 let r = std::panic::catch_unwind(|| { 1650 guest.wait_vm_boot(None).unwrap(); 1651 1652 if hotplug { 1653 // Add fs to the VM 1654 let (cmd_success, cmd_output) = 1655 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1656 assert!(cmd_success); 1657 1658 if let Some(pci_segment) = pci_segment { 1659 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1660 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1661 ))); 1662 } else { 1663 assert!(String::from_utf8_lossy(&cmd_output) 1664 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1665 } 1666 1667 thread::sleep(std::time::Duration::new(10, 0)); 1668 } 1669 1670 // Mount shared directory through virtio_fs filesystem 1671 guest 1672 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1673 .unwrap(); 1674 1675 // Check file1 exists and its content is "foo" 1676 assert_eq!( 1677 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1678 "foo" 1679 ); 1680 // Check file2 does not exist 1681 guest 1682 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1683 .unwrap(); 1684 1685 // Check file3 exists and its content is "bar" 1686 assert_eq!( 1687 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1688 "bar" 1689 ); 1690 1691 // ACPI feature is needed. 1692 #[cfg(target_arch = "x86_64")] 1693 { 1694 guest.enable_memory_hotplug(); 1695 1696 // Add RAM to the VM 1697 let desired_ram = 1024 << 20; 1698 resize_command(&api_socket, None, Some(desired_ram), None, None); 1699 1700 thread::sleep(std::time::Duration::new(30, 0)); 1701 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1702 1703 // After the resize, check again that file1 exists and its 1704 // content is "foo". 1705 assert_eq!( 1706 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1707 "foo" 1708 ); 1709 } 1710 1711 if hotplug { 1712 // Remove from VM 1713 guest.ssh_command("sudo umount mount_dir").unwrap(); 1714 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1715 } 1716 }); 1717 1718 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1719 thread::sleep(std::time::Duration::new(10, 0)); 1720 let (daemon_child, virtiofsd_socket_path) = 1721 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1722 1723 let r = std::panic::catch_unwind(|| { 1724 thread::sleep(std::time::Duration::new(10, 0)); 1725 let fs_params = format!( 1726 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1727 virtiofsd_socket_path, 1728 if let Some(pci_segment) = pci_segment { 1729 format!(",pci_segment={pci_segment}") 1730 } else { 1731 "".to_owned() 1732 } 1733 ); 1734 1735 // Add back and check it works 1736 let (cmd_success, cmd_output) = 1737 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1738 assert!(cmd_success); 1739 if let Some(pci_segment) = pci_segment { 1740 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1741 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1742 ))); 1743 } else { 1744 assert!(String::from_utf8_lossy(&cmd_output) 1745 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1746 } 1747 1748 thread::sleep(std::time::Duration::new(10, 0)); 1749 // Mount shared directory through virtio_fs filesystem 1750 guest 1751 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1752 .unwrap(); 1753 1754 // Check file1 exists and its content is "foo" 1755 assert_eq!( 1756 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1757 "foo" 1758 ); 1759 }); 1760 1761 (r, Some(daemon_child)) 1762 } else { 1763 (r, None) 1764 }; 1765 1766 let _ = child.kill(); 1767 let output = child.wait_with_output().unwrap(); 1768 1769 let _ = daemon_child.kill(); 1770 let _ = daemon_child.wait(); 1771 1772 if let Some(mut daemon_child) = hotplug_daemon_child { 1773 let _ = daemon_child.kill(); 1774 let _ = daemon_child.wait(); 1775 } 1776 1777 handle_child_output(r, &output); 1778 } 1779 1780 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1781 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1782 let guest = Guest::new(Box::new(focal)); 1783 1784 let kernel_path = direct_kernel_boot_path(); 1785 1786 let pmem_temp_file = TempFile::new().unwrap(); 1787 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1788 1789 std::process::Command::new("mkfs.ext4") 1790 .arg(pmem_temp_file.as_path()) 1791 .output() 1792 .expect("Expect creating disk image to succeed"); 1793 1794 let mut child = GuestCommand::new(&guest) 1795 .args(["--cpus", "boot=1"]) 1796 .args(["--memory", "size=512M"]) 1797 .args(["--kernel", kernel_path.to_str().unwrap()]) 1798 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1799 .default_disks() 1800 .default_net() 1801 .args([ 1802 "--pmem", 1803 format!( 1804 "file={}{}{}", 1805 pmem_temp_file.as_path().to_str().unwrap(), 1806 if specify_size { ",size=128M" } else { "" }, 1807 if discard_writes { 1808 ",discard_writes=on" 1809 } else { 1810 "" 1811 } 1812 ) 1813 .as_str(), 1814 ]) 1815 .capture_output() 1816 .spawn() 1817 .unwrap(); 1818 1819 let r = std::panic::catch_unwind(|| { 1820 guest.wait_vm_boot(None).unwrap(); 1821 1822 // Check for the presence of /dev/pmem0 1823 assert_eq!( 1824 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1825 "/dev/pmem0" 1826 ); 1827 1828 // Check changes persist after reboot 1829 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1830 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1831 guest 1832 .ssh_command("echo test123 | sudo tee /mnt/test") 1833 .unwrap(); 1834 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1835 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1836 1837 guest.reboot_linux(0, None); 1838 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1839 assert_eq!( 1840 guest 1841 .ssh_command("sudo cat /mnt/test || true") 1842 .unwrap() 1843 .trim(), 1844 if discard_writes { "" } else { "test123" } 1845 ); 1846 }); 1847 1848 let _ = child.kill(); 1849 let output = child.wait_with_output().unwrap(); 1850 1851 handle_child_output(r, &output); 1852 } 1853 1854 fn get_fd_count(pid: u32) -> usize { 1855 fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() 1856 } 1857 1858 fn _test_virtio_vsock(hotplug: bool) { 1859 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1860 let guest = Guest::new(Box::new(focal)); 1861 1862 #[cfg(target_arch = "x86_64")] 1863 let kernel_path = direct_kernel_boot_path(); 1864 #[cfg(target_arch = "aarch64")] 1865 let kernel_path = if hotplug { 1866 edk2_path() 1867 } else { 1868 direct_kernel_boot_path() 1869 }; 1870 1871 let socket = temp_vsock_path(&guest.tmp_dir); 1872 let api_socket = temp_api_path(&guest.tmp_dir); 1873 1874 let mut cmd = GuestCommand::new(&guest); 1875 cmd.args(["--api-socket", &api_socket]); 1876 cmd.args(["--cpus", "boot=1"]); 1877 cmd.args(["--memory", "size=512M"]); 1878 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1879 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1880 cmd.default_disks(); 1881 cmd.default_net(); 1882 1883 if !hotplug { 1884 cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); 1885 } 1886 1887 let mut child = cmd.capture_output().spawn().unwrap(); 1888 1889 let r = std::panic::catch_unwind(|| { 1890 guest.wait_vm_boot(None).unwrap(); 1891 1892 if hotplug { 1893 let (cmd_success, cmd_output) = remote_command_w_output( 1894 &api_socket, 1895 "add-vsock", 1896 Some(format!("cid=3,socket={socket},id=test0").as_str()), 1897 ); 1898 assert!(cmd_success); 1899 assert!(String::from_utf8_lossy(&cmd_output) 1900 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1901 thread::sleep(std::time::Duration::new(10, 0)); 1902 // Check adding a second one fails 1903 assert!(!remote_command( 1904 &api_socket, 1905 "add-vsock", 1906 Some("cid=1234,socket=/tmp/fail") 1907 )); 1908 } 1909 1910 // Validate vsock works as expected. 1911 guest.check_vsock(socket.as_str()); 1912 guest.reboot_linux(0, None); 1913 // Validate vsock still works after a reboot. 1914 guest.check_vsock(socket.as_str()); 1915 1916 if hotplug { 1917 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1918 } 1919 }); 1920 1921 let _ = child.kill(); 1922 let output = child.wait_with_output().unwrap(); 1923 1924 handle_child_output(r, &output); 1925 } 1926 1927 fn get_ksm_pages_shared() -> u32 { 1928 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1929 .unwrap() 1930 .trim() 1931 .parse::<u32>() 1932 .unwrap() 1933 } 1934 1935 fn test_memory_mergeable(mergeable: bool) { 1936 let memory_param = if mergeable { 1937 "mergeable=on" 1938 } else { 1939 "mergeable=off" 1940 }; 1941 1942 // We are assuming the rest of the system in our CI is not using mergeable memory 1943 let ksm_ps_init = get_ksm_pages_shared(); 1944 assert!(ksm_ps_init == 0); 1945 1946 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1947 let guest1 = Guest::new(Box::new(focal1)); 1948 let mut child1 = GuestCommand::new(&guest1) 1949 .args(["--cpus", "boot=1"]) 1950 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1951 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1952 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1953 .default_disks() 1954 .args(["--net", guest1.default_net_string().as_str()]) 1955 .args(["--serial", "tty", "--console", "off"]) 1956 .capture_output() 1957 .spawn() 1958 .unwrap(); 1959 1960 let r = std::panic::catch_unwind(|| { 1961 guest1.wait_vm_boot(None).unwrap(); 1962 }); 1963 if r.is_err() { 1964 let _ = child1.kill(); 1965 let output = child1.wait_with_output().unwrap(); 1966 handle_child_output(r, &output); 1967 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1968 } 1969 1970 let ksm_ps_guest1 = get_ksm_pages_shared(); 1971 1972 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1973 let guest2 = Guest::new(Box::new(focal2)); 1974 let mut child2 = GuestCommand::new(&guest2) 1975 .args(["--cpus", "boot=1"]) 1976 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1977 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1978 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1979 .default_disks() 1980 .args(["--net", guest2.default_net_string().as_str()]) 1981 .args(["--serial", "tty", "--console", "off"]) 1982 .capture_output() 1983 .spawn() 1984 .unwrap(); 1985 1986 let r = std::panic::catch_unwind(|| { 1987 guest2.wait_vm_boot(None).unwrap(); 1988 let ksm_ps_guest2 = get_ksm_pages_shared(); 1989 1990 if mergeable { 1991 println!( 1992 "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" 1993 ); 1994 // We are expecting the number of shared pages to increase as the number of VM increases 1995 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1996 } else { 1997 assert!(ksm_ps_guest1 == 0); 1998 assert!(ksm_ps_guest2 == 0); 1999 } 2000 }); 2001 2002 let _ = child1.kill(); 2003 let _ = child2.kill(); 2004 2005 let output = child1.wait_with_output().unwrap(); 2006 child2.wait().unwrap(); 2007 2008 handle_child_output(r, &output); 2009 } 2010 2011 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 2012 let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); 2013 let reader = io::BufReader::new(smaps); 2014 2015 let mut skip_map: bool = false; 2016 let mut region_name: String = "".to_string(); 2017 let mut region_maps = HashMap::new(); 2018 for line in reader.lines() { 2019 let l = line.unwrap(); 2020 2021 if l.contains('-') { 2022 let values: Vec<&str> = l.split_whitespace().collect(); 2023 region_name = values.last().unwrap().trim().to_string(); 2024 if region_name == "0" { 2025 region_name = "anonymous".to_string() 2026 } 2027 } 2028 2029 // Each section begins with something that looks like: 2030 // Size: 2184 kB 2031 if l.starts_with("Size:") { 2032 let values: Vec<&str> = l.split_whitespace().collect(); 2033 let map_size = values[1].parse::<u32>().unwrap(); 2034 // We skip the assigned guest RAM map, its RSS is only 2035 // dependent on the guest actual memory usage. 2036 // Everything else can be added to the VMM overhead. 2037 skip_map = map_size >= guest_memory_size; 2038 continue; 2039 } 2040 2041 // If this is a map we're taking into account, then we only 2042 // count the RSS. The sum of all counted RSS is the VMM overhead. 2043 if !skip_map && l.starts_with("Rss:") { 2044 let values: Vec<&str> = l.split_whitespace().collect(); 2045 let value = values[1].trim().parse::<u32>().unwrap(); 2046 *region_maps.entry(region_name.clone()).or_insert(0) += value; 2047 } 2048 } 2049 2050 region_maps 2051 } 2052 2053 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 2054 let mut total = 0; 2055 2056 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 2057 eprintln!("{region_name}: {value}"); 2058 total += value; 2059 } 2060 2061 total 2062 } 2063 2064 fn process_rss_kib(pid: u32) -> usize { 2065 let command = format!("ps -q {pid} -o rss="); 2066 let rss = exec_host_command_output(&command); 2067 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 2068 } 2069 2070 // 10MB is our maximum accepted overhead. 2071 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 2072 2073 #[derive(PartialEq, Eq, PartialOrd)] 2074 struct Counters { 2075 rx_bytes: u64, 2076 rx_frames: u64, 2077 tx_bytes: u64, 2078 tx_frames: u64, 2079 read_bytes: u64, 2080 write_bytes: u64, 2081 read_ops: u64, 2082 write_ops: u64, 2083 } 2084 2085 fn get_counters(api_socket: &str) -> Counters { 2086 // Get counters 2087 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 2088 assert!(cmd_success); 2089 2090 let counters: HashMap<&str, HashMap<&str, u64>> = 2091 serde_json::from_slice(&cmd_output).unwrap_or_default(); 2092 2093 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 2094 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 2095 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 2096 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 2097 2098 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 2099 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 2100 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 2101 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 2102 2103 Counters { 2104 rx_bytes, 2105 rx_frames, 2106 tx_bytes, 2107 tx_frames, 2108 read_bytes, 2109 write_bytes, 2110 read_ops, 2111 write_ops, 2112 } 2113 } 2114 2115 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 2116 let (tx, rx) = mpsc::channel::<String>(); 2117 thread::spawn(move || loop { 2118 thread::sleep(std::time::Duration::new(1, 0)); 2119 let mut buf = [0; 512]; 2120 match pty.read(&mut buf) { 2121 Ok(_bytes) => { 2122 let output = std::str::from_utf8(&buf).unwrap().to_string(); 2123 match tx.send(output) { 2124 Ok(_) => (), 2125 Err(_) => break, 2126 } 2127 } 2128 Err(_) => break, 2129 } 2130 }); 2131 rx 2132 } 2133 2134 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 2135 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2136 assert!(cmd_success); 2137 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2138 assert_eq!("Pty", info["config"][pty_type]["mode"]); 2139 PathBuf::from( 2140 info["config"][pty_type]["file"] 2141 .as_str() 2142 .expect("Missing pty path"), 2143 ) 2144 } 2145 2146 // VFIO test network setup. 2147 // We reserve a different IP class for it: 172.18.0.0/24. 2148 #[cfg(target_arch = "x86_64")] 2149 fn setup_vfio_network_interfaces() { 2150 // 'vfio-br0' 2151 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 2152 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 2153 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 2154 // 'vfio-tap0' 2155 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 2156 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 2157 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 2158 // 'vfio-tap1' 2159 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 2160 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 2161 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 2162 // 'vfio-tap2' 2163 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 2164 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 2165 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 2166 // 'vfio-tap3' 2167 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 2168 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 2169 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 2170 } 2171 2172 // Tear VFIO test network down 2173 #[cfg(target_arch = "x86_64")] 2174 fn cleanup_vfio_network_interfaces() { 2175 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 2176 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 2177 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 2178 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 2179 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 2180 } 2181 2182 fn balloon_size(api_socket: &str) -> u64 { 2183 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2184 assert!(cmd_success); 2185 2186 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2187 let total_mem = &info["config"]["memory"]["size"] 2188 .to_string() 2189 .parse::<u64>() 2190 .unwrap(); 2191 let actual_mem = &info["memory_actual_size"] 2192 .to_string() 2193 .parse::<u64>() 2194 .unwrap(); 2195 total_mem - actual_mem 2196 } 2197 2198 fn vm_state(api_socket: &str) -> String { 2199 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2200 assert!(cmd_success); 2201 2202 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2203 let state = &info["state"].as_str().unwrap(); 2204 2205 state.to_string() 2206 } 2207 2208 // This test validates that it can find the virtio-iommu device at first. 2209 // It also verifies that both disks and the network card are attached to 2210 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 2211 // The last interesting part of this test is that it exercises the network 2212 // interface attached to the virtual IOMMU since this is the one used to 2213 // send all commands through SSH. 2214 fn _test_virtio_iommu(acpi: bool) { 2215 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 2216 // Focal image is still old. 2217 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 2218 // the kernel binary has been updated. 2219 #[cfg(target_arch = "aarch64")] 2220 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 2221 #[cfg(target_arch = "x86_64")] 2222 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2223 let focal = UbuntuDiskConfig::new(focal_image); 2224 let guest = Guest::new(Box::new(focal)); 2225 2226 #[cfg(target_arch = "x86_64")] 2227 let kernel_path = direct_kernel_boot_path(); 2228 #[cfg(target_arch = "aarch64")] 2229 let kernel_path = if acpi { 2230 edk2_path() 2231 } else { 2232 direct_kernel_boot_path() 2233 }; 2234 2235 let mut child = GuestCommand::new(&guest) 2236 .args(["--cpus", "boot=1"]) 2237 .args(["--memory", "size=512M"]) 2238 .args(["--kernel", kernel_path.to_str().unwrap()]) 2239 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2240 .args([ 2241 "--disk", 2242 format!( 2243 "path={},iommu=on", 2244 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2245 ) 2246 .as_str(), 2247 format!( 2248 "path={},iommu=on", 2249 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2250 ) 2251 .as_str(), 2252 ]) 2253 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 2254 .capture_output() 2255 .spawn() 2256 .unwrap(); 2257 2258 let r = std::panic::catch_unwind(|| { 2259 guest.wait_vm_boot(None).unwrap(); 2260 2261 // Verify the virtio-iommu device is present. 2262 assert!(guest 2263 .does_device_vendor_pair_match("0x1057", "0x1af4") 2264 .unwrap_or_default()); 2265 2266 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 2267 // different with ACPI. 2268 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 2269 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 2270 // and appear under folder '/sys/kernel/iommu_groups/'. 2271 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 2272 // which is the console. The first disk "0000:00:02.0" is in group '1'. 2273 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 2274 // contains "0000:00:02.0" which is the first disk. 2275 // 2276 // Verify the iommu group of the first disk. 2277 let iommu_group = !acpi as i32; 2278 assert_eq!( 2279 guest 2280 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2281 .unwrap() 2282 .trim(), 2283 "0000:00:02.0" 2284 ); 2285 2286 // Verify the iommu group of the second disk. 2287 let iommu_group = if acpi { 1 } else { 2 }; 2288 assert_eq!( 2289 guest 2290 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2291 .unwrap() 2292 .trim(), 2293 "0000:00:03.0" 2294 ); 2295 2296 // Verify the iommu group of the network card. 2297 let iommu_group = if acpi { 2 } else { 3 }; 2298 assert_eq!( 2299 guest 2300 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2301 .unwrap() 2302 .trim(), 2303 "0000:00:04.0" 2304 ); 2305 }); 2306 2307 let _ = child.kill(); 2308 let output = child.wait_with_output().unwrap(); 2309 2310 handle_child_output(r, &output); 2311 } 2312 2313 fn get_reboot_count(guest: &Guest) -> u32 { 2314 guest 2315 .ssh_command("sudo last | grep -c reboot") 2316 .unwrap() 2317 .trim() 2318 .parse::<u32>() 2319 .unwrap_or_default() 2320 } 2321 2322 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 2323 // Check for PCI device 2324 assert!(guest 2325 .does_device_vendor_pair_match("0x1063", "0x1af4") 2326 .unwrap_or_default()); 2327 2328 // Enable systemd watchdog 2329 guest 2330 .ssh_command(&format!( 2331 "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" 2332 )) 2333 .unwrap(); 2334 } 2335 2336 fn make_guest_panic(guest: &Guest) { 2337 // Check for pvpanic device 2338 assert!(guest 2339 .does_device_vendor_pair_match("0x0011", "0x1b36") 2340 .unwrap_or_default()); 2341 2342 // Trigger guest a panic 2343 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 2344 } 2345 2346 mod common_parallel { 2347 use std::{fs::OpenOptions, io::SeekFrom}; 2348 2349 use crate::*; 2350 2351 #[test] 2352 #[cfg(target_arch = "x86_64")] 2353 fn test_focal_hypervisor_fw() { 2354 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 2355 } 2356 2357 #[test] 2358 #[cfg(target_arch = "x86_64")] 2359 fn test_focal_ovmf() { 2360 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 2361 } 2362 2363 #[cfg(target_arch = "x86_64")] 2364 fn test_simple_launch(fw_path: String, disk_path: &str) { 2365 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 2366 let guest = Guest::new(disk_config); 2367 let event_path = temp_event_monitor_path(&guest.tmp_dir); 2368 2369 let mut child = GuestCommand::new(&guest) 2370 .args(["--cpus", "boot=1"]) 2371 .args(["--memory", "size=512M"]) 2372 .args(["--kernel", fw_path.as_str()]) 2373 .default_disks() 2374 .default_net() 2375 .args(["--serial", "tty", "--console", "off"]) 2376 .args(["--event-monitor", format!("path={event_path}").as_str()]) 2377 .capture_output() 2378 .spawn() 2379 .unwrap(); 2380 2381 let r = std::panic::catch_unwind(|| { 2382 guest.wait_vm_boot(Some(120)).unwrap(); 2383 2384 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2385 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2386 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2387 2388 let expected_sequential_events = [ 2389 &MetaEvent { 2390 event: "starting".to_string(), 2391 device_id: None, 2392 }, 2393 &MetaEvent { 2394 event: "booting".to_string(), 2395 device_id: None, 2396 }, 2397 &MetaEvent { 2398 event: "booted".to_string(), 2399 device_id: None, 2400 }, 2401 &MetaEvent { 2402 event: "activated".to_string(), 2403 device_id: Some("_disk0".to_string()), 2404 }, 2405 &MetaEvent { 2406 event: "reset".to_string(), 2407 device_id: Some("_disk0".to_string()), 2408 }, 2409 ]; 2410 assert!(check_sequential_events( 2411 &expected_sequential_events, 2412 &event_path 2413 )); 2414 2415 // It's been observed on the Bionic image that udev and snapd 2416 // services can cause some delay in the VM's shutdown. Disabling 2417 // them improves the reliability of this test. 2418 let _ = guest.ssh_command("sudo systemctl disable udev"); 2419 let _ = guest.ssh_command("sudo systemctl stop udev"); 2420 let _ = guest.ssh_command("sudo systemctl disable snapd"); 2421 let _ = guest.ssh_command("sudo systemctl stop snapd"); 2422 2423 guest.ssh_command("sudo poweroff").unwrap(); 2424 thread::sleep(std::time::Duration::new(20, 0)); 2425 let latest_events = [ 2426 &MetaEvent { 2427 event: "shutdown".to_string(), 2428 device_id: None, 2429 }, 2430 &MetaEvent { 2431 event: "deleted".to_string(), 2432 device_id: None, 2433 }, 2434 &MetaEvent { 2435 event: "shutdown".to_string(), 2436 device_id: None, 2437 }, 2438 ]; 2439 assert!(check_latest_events_exact(&latest_events, &event_path)); 2440 }); 2441 2442 let _ = child.kill(); 2443 let output = child.wait_with_output().unwrap(); 2444 2445 handle_child_output(r, &output); 2446 } 2447 2448 #[test] 2449 fn test_multi_cpu() { 2450 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2451 let jammy = UbuntuDiskConfig::new(jammy_image); 2452 let guest = Guest::new(Box::new(jammy)); 2453 2454 let mut cmd = GuestCommand::new(&guest); 2455 cmd.args(["--cpus", "boot=2,max=4"]) 2456 .args(["--memory", "size=512M"]) 2457 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2458 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2459 .capture_output() 2460 .default_disks() 2461 .default_net(); 2462 2463 let mut child = cmd.spawn().unwrap(); 2464 2465 let r = std::panic::catch_unwind(|| { 2466 guest.wait_vm_boot(Some(120)).unwrap(); 2467 2468 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2469 2470 assert_eq!( 2471 guest 2472 .ssh_command( 2473 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2474 ) 2475 .unwrap() 2476 .trim(), 2477 "smp: Brought up 1 node, 2 CPUs" 2478 ); 2479 }); 2480 2481 let _ = child.kill(); 2482 let output = child.wait_with_output().unwrap(); 2483 2484 handle_child_output(r, &output); 2485 } 2486 2487 #[test] 2488 fn test_cpu_topology_421() { 2489 test_cpu_topology(4, 2, 1, false); 2490 } 2491 2492 #[test] 2493 fn test_cpu_topology_142() { 2494 test_cpu_topology(1, 4, 2, false); 2495 } 2496 2497 #[test] 2498 fn test_cpu_topology_262() { 2499 test_cpu_topology(2, 6, 2, false); 2500 } 2501 2502 #[test] 2503 #[cfg(target_arch = "x86_64")] 2504 #[cfg(not(feature = "mshv"))] 2505 fn test_cpu_physical_bits() { 2506 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2507 let guest = Guest::new(Box::new(focal)); 2508 let max_phys_bits: u8 = 36; 2509 let mut child = GuestCommand::new(&guest) 2510 .args(["--cpus", &format!("max_phys_bits={max_phys_bits}")]) 2511 .args(["--memory", "size=512M"]) 2512 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2513 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2514 .default_disks() 2515 .default_net() 2516 .capture_output() 2517 .spawn() 2518 .unwrap(); 2519 2520 let r = std::panic::catch_unwind(|| { 2521 guest.wait_vm_boot(None).unwrap(); 2522 2523 assert!( 2524 guest 2525 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2526 .unwrap() 2527 .trim() 2528 .parse::<u8>() 2529 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2530 ); 2531 }); 2532 2533 let _ = child.kill(); 2534 let output = child.wait_with_output().unwrap(); 2535 2536 handle_child_output(r, &output); 2537 } 2538 2539 #[test] 2540 fn test_cpu_affinity() { 2541 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2542 let guest = Guest::new(Box::new(focal)); 2543 2544 // We need the host to have at least 4 CPUs if we want to be able 2545 // to run this test. 2546 let host_cpus_count = exec_host_command_output("nproc"); 2547 assert!( 2548 String::from_utf8_lossy(&host_cpus_count.stdout) 2549 .trim() 2550 .parse::<u16>() 2551 .unwrap_or(0) 2552 >= 4 2553 ); 2554 2555 let mut child = GuestCommand::new(&guest) 2556 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2557 .args(["--memory", "size=512M"]) 2558 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2559 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2560 .default_disks() 2561 .default_net() 2562 .capture_output() 2563 .spawn() 2564 .unwrap(); 2565 2566 let r = std::panic::catch_unwind(|| { 2567 guest.wait_vm_boot(None).unwrap(); 2568 let pid = child.id(); 2569 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2570 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2571 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2572 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2573 }); 2574 2575 let _ = child.kill(); 2576 let output = child.wait_with_output().unwrap(); 2577 handle_child_output(r, &output); 2578 } 2579 2580 #[test] 2581 fn test_virtio_queue_affinity() { 2582 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2583 let guest = Guest::new(Box::new(focal)); 2584 2585 // We need the host to have at least 4 CPUs if we want to be able 2586 // to run this test. 2587 let host_cpus_count = exec_host_command_output("nproc"); 2588 assert!( 2589 String::from_utf8_lossy(&host_cpus_count.stdout) 2590 .trim() 2591 .parse::<u16>() 2592 .unwrap_or(0) 2593 >= 4 2594 ); 2595 2596 let mut child = GuestCommand::new(&guest) 2597 .args(["--cpus", "boot=4"]) 2598 .args(["--memory", "size=512M"]) 2599 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2600 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2601 .args([ 2602 "--disk", 2603 format!( 2604 "path={}", 2605 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2606 ) 2607 .as_str(), 2608 format!( 2609 "path={},num_queues=4,queue_affinity=[0@[0,2],1@[1,3],2@[1],3@[3]]", 2610 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2611 ) 2612 .as_str(), 2613 ]) 2614 .default_net() 2615 .capture_output() 2616 .spawn() 2617 .unwrap(); 2618 2619 let r = std::panic::catch_unwind(|| { 2620 guest.wait_vm_boot(None).unwrap(); 2621 let pid = child.id(); 2622 let taskset_q0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2623 assert_eq!(String::from_utf8_lossy(&taskset_q0.stdout).trim(), "0,2"); 2624 let taskset_q1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2625 assert_eq!(String::from_utf8_lossy(&taskset_q1.stdout).trim(), "1,3"); 2626 let taskset_q2 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q2 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2627 assert_eq!(String::from_utf8_lossy(&taskset_q2.stdout).trim(), "1"); 2628 let taskset_q3 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q3 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2629 assert_eq!(String::from_utf8_lossy(&taskset_q3.stdout).trim(), "3"); 2630 }); 2631 2632 let _ = child.kill(); 2633 let output = child.wait_with_output().unwrap(); 2634 handle_child_output(r, &output); 2635 } 2636 2637 #[test] 2638 #[cfg(not(feature = "mshv"))] 2639 fn test_large_vm() { 2640 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2641 let guest = Guest::new(Box::new(focal)); 2642 let mut cmd = GuestCommand::new(&guest); 2643 cmd.args(["--cpus", "boot=48"]) 2644 .args(["--memory", "size=5120M"]) 2645 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2646 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2647 .args(["--serial", "tty"]) 2648 .args(["--console", "off"]) 2649 .capture_output() 2650 .default_disks() 2651 .default_net(); 2652 2653 let mut child = cmd.spawn().unwrap(); 2654 2655 guest.wait_vm_boot(None).unwrap(); 2656 2657 let r = std::panic::catch_unwind(|| { 2658 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2659 assert_eq!( 2660 guest 2661 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2662 .unwrap() 2663 .trim(), 2664 "0-47" 2665 ); 2666 2667 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2668 }); 2669 2670 let _ = child.kill(); 2671 let output = child.wait_with_output().unwrap(); 2672 2673 handle_child_output(r, &output); 2674 } 2675 2676 #[test] 2677 #[cfg(not(feature = "mshv"))] 2678 fn test_huge_memory() { 2679 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2680 let guest = Guest::new(Box::new(focal)); 2681 let mut cmd = GuestCommand::new(&guest); 2682 cmd.args(["--cpus", "boot=1"]) 2683 .args(["--memory", "size=128G"]) 2684 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2685 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2686 .capture_output() 2687 .default_disks() 2688 .default_net(); 2689 2690 let mut child = cmd.spawn().unwrap(); 2691 2692 guest.wait_vm_boot(Some(120)).unwrap(); 2693 2694 let r = std::panic::catch_unwind(|| { 2695 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2696 }); 2697 2698 let _ = child.kill(); 2699 let output = child.wait_with_output().unwrap(); 2700 2701 handle_child_output(r, &output); 2702 } 2703 2704 #[test] 2705 fn test_power_button() { 2706 _test_power_button(false); 2707 } 2708 2709 #[test] 2710 #[cfg(not(feature = "mshv"))] 2711 fn test_user_defined_memory_regions() { 2712 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2713 let guest = Guest::new(Box::new(focal)); 2714 let api_socket = temp_api_path(&guest.tmp_dir); 2715 2716 let kernel_path = direct_kernel_boot_path(); 2717 2718 let mut child = GuestCommand::new(&guest) 2719 .args(["--cpus", "boot=1"]) 2720 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2721 .args([ 2722 "--memory-zone", 2723 "id=mem0,size=1G,hotplug_size=2G", 2724 "id=mem1,size=1G,shared=on", 2725 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2726 ]) 2727 .args(["--kernel", kernel_path.to_str().unwrap()]) 2728 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2729 .args(["--api-socket", &api_socket]) 2730 .capture_output() 2731 .default_disks() 2732 .default_net() 2733 .spawn() 2734 .unwrap(); 2735 2736 let r = std::panic::catch_unwind(|| { 2737 guest.wait_vm_boot(None).unwrap(); 2738 2739 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2740 2741 guest.enable_memory_hotplug(); 2742 2743 resize_zone_command(&api_socket, "mem0", "3G"); 2744 thread::sleep(std::time::Duration::new(5, 0)); 2745 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2746 resize_zone_command(&api_socket, "mem2", "3G"); 2747 thread::sleep(std::time::Duration::new(5, 0)); 2748 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2749 resize_zone_command(&api_socket, "mem0", "2G"); 2750 thread::sleep(std::time::Duration::new(5, 0)); 2751 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2752 resize_zone_command(&api_socket, "mem2", "2G"); 2753 thread::sleep(std::time::Duration::new(5, 0)); 2754 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2755 2756 guest.reboot_linux(0, None); 2757 2758 // Check the amount of RAM after reboot 2759 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2760 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2761 2762 // Check if we can still resize down to the initial 'boot'size 2763 resize_zone_command(&api_socket, "mem0", "1G"); 2764 thread::sleep(std::time::Duration::new(5, 0)); 2765 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2766 resize_zone_command(&api_socket, "mem2", "1G"); 2767 thread::sleep(std::time::Duration::new(5, 0)); 2768 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2769 }); 2770 2771 let _ = child.kill(); 2772 let output = child.wait_with_output().unwrap(); 2773 2774 handle_child_output(r, &output); 2775 } 2776 2777 #[test] 2778 #[cfg(not(feature = "mshv"))] 2779 fn test_guest_numa_nodes() { 2780 _test_guest_numa_nodes(false); 2781 } 2782 2783 #[test] 2784 #[cfg(target_arch = "x86_64")] 2785 fn test_iommu_segments() { 2786 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2787 let focal = UbuntuDiskConfig::new(focal_image); 2788 let guest = Guest::new(Box::new(focal)); 2789 2790 // Prepare another disk file for the virtio-disk device 2791 let test_disk_path = String::from( 2792 guest 2793 .tmp_dir 2794 .as_path() 2795 .join("test-disk.raw") 2796 .to_str() 2797 .unwrap(), 2798 ); 2799 assert!( 2800 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2801 ); 2802 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2803 2804 let api_socket = temp_api_path(&guest.tmp_dir); 2805 let mut cmd = GuestCommand::new(&guest); 2806 2807 cmd.args(["--cpus", "boot=1"]) 2808 .args(["--api-socket", &api_socket]) 2809 .args(["--memory", "size=512M"]) 2810 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2811 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2812 .args([ 2813 "--platform", 2814 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS},iommu_segments=[1]"), 2815 ]) 2816 .default_disks() 2817 .capture_output() 2818 .default_net(); 2819 2820 let mut child = cmd.spawn().unwrap(); 2821 2822 guest.wait_vm_boot(None).unwrap(); 2823 2824 let r = std::panic::catch_unwind(|| { 2825 let (cmd_success, cmd_output) = remote_command_w_output( 2826 &api_socket, 2827 "add-disk", 2828 Some( 2829 format!( 2830 "path={},id=test0,pci_segment=1,iommu=on", 2831 test_disk_path.as_str() 2832 ) 2833 .as_str(), 2834 ), 2835 ); 2836 assert!(cmd_success); 2837 assert!(String::from_utf8_lossy(&cmd_output) 2838 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2839 2840 // Check IOMMU setup 2841 assert!(guest 2842 .does_device_vendor_pair_match("0x1057", "0x1af4") 2843 .unwrap_or_default()); 2844 assert_eq!( 2845 guest 2846 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2847 .unwrap() 2848 .trim(), 2849 "0001:00:01.0" 2850 ); 2851 }); 2852 2853 let _ = child.kill(); 2854 let output = child.wait_with_output().unwrap(); 2855 2856 handle_child_output(r, &output); 2857 } 2858 2859 #[test] 2860 fn test_pci_msi() { 2861 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2862 let guest = Guest::new(Box::new(focal)); 2863 let mut cmd = GuestCommand::new(&guest); 2864 cmd.args(["--cpus", "boot=1"]) 2865 .args(["--memory", "size=512M"]) 2866 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2867 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2868 .capture_output() 2869 .default_disks() 2870 .default_net(); 2871 2872 let mut child = cmd.spawn().unwrap(); 2873 2874 guest.wait_vm_boot(None).unwrap(); 2875 2876 #[cfg(target_arch = "x86_64")] 2877 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2878 #[cfg(target_arch = "aarch64")] 2879 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2880 2881 let r = std::panic::catch_unwind(|| { 2882 assert_eq!( 2883 guest 2884 .ssh_command(grep_cmd) 2885 .unwrap() 2886 .trim() 2887 .parse::<u32>() 2888 .unwrap_or_default(), 2889 12 2890 ); 2891 }); 2892 2893 let _ = child.kill(); 2894 let output = child.wait_with_output().unwrap(); 2895 2896 handle_child_output(r, &output); 2897 } 2898 2899 #[test] 2900 fn test_virtio_net_ctrl_queue() { 2901 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2902 let guest = Guest::new(Box::new(focal)); 2903 let mut cmd = GuestCommand::new(&guest); 2904 cmd.args(["--cpus", "boot=1"]) 2905 .args(["--memory", "size=512M"]) 2906 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2907 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2908 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2909 .capture_output() 2910 .default_disks(); 2911 2912 let mut child = cmd.spawn().unwrap(); 2913 2914 guest.wait_vm_boot(None).unwrap(); 2915 2916 #[cfg(target_arch = "aarch64")] 2917 let iface = "enp0s4"; 2918 #[cfg(target_arch = "x86_64")] 2919 let iface = "ens4"; 2920 2921 let r = std::panic::catch_unwind(|| { 2922 assert_eq!( 2923 guest 2924 .ssh_command( 2925 format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() 2926 ) 2927 .unwrap() 2928 .trim(), 2929 "success" 2930 ); 2931 assert_eq!( 2932 guest 2933 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 2934 .unwrap() 2935 .trim(), 2936 "3000" 2937 ); 2938 }); 2939 2940 let _ = child.kill(); 2941 let output = child.wait_with_output().unwrap(); 2942 2943 handle_child_output(r, &output); 2944 } 2945 2946 #[test] 2947 fn test_pci_multiple_segments() { 2948 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2949 let guest = Guest::new(Box::new(focal)); 2950 2951 // Prepare another disk file for the virtio-disk device 2952 let test_disk_path = String::from( 2953 guest 2954 .tmp_dir 2955 .as_path() 2956 .join("test-disk.raw") 2957 .to_str() 2958 .unwrap(), 2959 ); 2960 assert!( 2961 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2962 ); 2963 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2964 2965 let mut cmd = GuestCommand::new(&guest); 2966 cmd.args(["--cpus", "boot=1"]) 2967 .args(["--memory", "size=512M"]) 2968 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2969 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2970 .args([ 2971 "--platform", 2972 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 2973 ]) 2974 .args([ 2975 "--disk", 2976 format!( 2977 "path={}", 2978 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2979 ) 2980 .as_str(), 2981 format!( 2982 "path={}", 2983 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2984 ) 2985 .as_str(), 2986 format!("path={test_disk_path},pci_segment=15").as_str(), 2987 ]) 2988 .capture_output() 2989 .default_net(); 2990 2991 let mut child = cmd.spawn().unwrap(); 2992 2993 guest.wait_vm_boot(None).unwrap(); 2994 2995 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2996 2997 let r = std::panic::catch_unwind(|| { 2998 // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. 2999 assert_eq!( 3000 guest 3001 .ssh_command(grep_cmd) 3002 .unwrap() 3003 .trim() 3004 .parse::<u16>() 3005 .unwrap_or_default(), 3006 MAX_NUM_PCI_SEGMENTS 3007 ); 3008 3009 // Check both if /dev/vdc exists and if the block size is 4M. 3010 assert_eq!( 3011 guest 3012 .ssh_command("lsblk | grep vdc | grep -c 4M") 3013 .unwrap() 3014 .trim() 3015 .parse::<u32>() 3016 .unwrap_or_default(), 3017 1 3018 ); 3019 3020 // Mount the device. 3021 guest.ssh_command("mkdir mount_image").unwrap(); 3022 guest 3023 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 3024 .unwrap(); 3025 // Grant all users with write permission. 3026 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 3027 3028 // Write something to the device. 3029 guest 3030 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 3031 .unwrap(); 3032 3033 // Check the content of the block device. The file "foo" should 3034 // contain "bar". 3035 assert_eq!( 3036 guest 3037 .ssh_command("sudo cat mount_image/foo") 3038 .unwrap() 3039 .trim(), 3040 "bar" 3041 ); 3042 }); 3043 3044 let _ = child.kill(); 3045 let output = child.wait_with_output().unwrap(); 3046 3047 handle_child_output(r, &output); 3048 } 3049 3050 #[test] 3051 fn test_pci_multiple_segments_numa_node() { 3052 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3053 let guest = Guest::new(Box::new(focal)); 3054 let api_socket = temp_api_path(&guest.tmp_dir); 3055 #[cfg(target_arch = "x86_64")] 3056 let kernel_path = direct_kernel_boot_path(); 3057 #[cfg(target_arch = "aarch64")] 3058 let kernel_path = edk2_path(); 3059 3060 // Prepare another disk file for the virtio-disk device 3061 let test_disk_path = String::from( 3062 guest 3063 .tmp_dir 3064 .as_path() 3065 .join("test-disk.raw") 3066 .to_str() 3067 .unwrap(), 3068 ); 3069 assert!( 3070 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 3071 ); 3072 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 3073 const TEST_DISK_NODE: u16 = 1; 3074 3075 let mut child = GuestCommand::new(&guest) 3076 .args(["--platform", "num_pci_segments=2"]) 3077 .args(["--cpus", "boot=2"]) 3078 .args(["--memory", "size=0"]) 3079 .args(["--memory-zone", "id=mem0,size=256M", "id=mem1,size=256M"]) 3080 .args([ 3081 "--numa", 3082 "guest_numa_id=0,cpus=[0],distances=[1@20],memory_zones=mem0,pci_segments=[0]", 3083 "guest_numa_id=1,cpus=[1],distances=[0@20],memory_zones=mem1,pci_segments=[1]", 3084 ]) 3085 .args(["--kernel", kernel_path.to_str().unwrap()]) 3086 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3087 .args(["--api-socket", &api_socket]) 3088 .capture_output() 3089 .args([ 3090 "--disk", 3091 format!( 3092 "path={}", 3093 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3094 ) 3095 .as_str(), 3096 format!( 3097 "path={}", 3098 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3099 ) 3100 .as_str(), 3101 format!("path={test_disk_path},pci_segment={TEST_DISK_NODE}").as_str(), 3102 ]) 3103 .default_net() 3104 .spawn() 3105 .unwrap(); 3106 3107 let cmd = "cat /sys/block/vdc/device/../numa_node"; 3108 3109 let r = std::panic::catch_unwind(|| { 3110 guest.wait_vm_boot(None).unwrap(); 3111 3112 assert_eq!( 3113 guest 3114 .ssh_command(cmd) 3115 .unwrap() 3116 .trim() 3117 .parse::<u16>() 3118 .unwrap_or_default(), 3119 TEST_DISK_NODE 3120 ); 3121 }); 3122 3123 let _ = child.kill(); 3124 let output = child.wait_with_output().unwrap(); 3125 3126 handle_child_output(r, &output); 3127 } 3128 3129 #[test] 3130 fn test_direct_kernel_boot() { 3131 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3132 let guest = Guest::new(Box::new(focal)); 3133 3134 let kernel_path = direct_kernel_boot_path(); 3135 3136 let mut child = GuestCommand::new(&guest) 3137 .args(["--cpus", "boot=1"]) 3138 .args(["--memory", "size=512M"]) 3139 .args(["--kernel", kernel_path.to_str().unwrap()]) 3140 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3141 .default_disks() 3142 .default_net() 3143 .capture_output() 3144 .spawn() 3145 .unwrap(); 3146 3147 let r = std::panic::catch_unwind(|| { 3148 guest.wait_vm_boot(None).unwrap(); 3149 3150 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3151 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3152 3153 let grep_cmd = if cfg!(target_arch = "x86_64") { 3154 "grep -c PCI-MSI /proc/interrupts" 3155 } else { 3156 "grep -c ITS-MSI /proc/interrupts" 3157 }; 3158 assert_eq!( 3159 guest 3160 .ssh_command(grep_cmd) 3161 .unwrap() 3162 .trim() 3163 .parse::<u32>() 3164 .unwrap_or_default(), 3165 12 3166 ); 3167 }); 3168 3169 let _ = child.kill(); 3170 let output = child.wait_with_output().unwrap(); 3171 3172 handle_child_output(r, &output); 3173 } 3174 3175 #[test] 3176 #[cfg(target_arch = "x86_64")] 3177 fn test_direct_kernel_boot_bzimage() { 3178 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3179 let guest = Guest::new(Box::new(focal)); 3180 3181 let mut kernel_path = direct_kernel_boot_path(); 3182 // Replace the default kernel with the bzImage. 3183 kernel_path.pop(); 3184 kernel_path.push("bzImage"); 3185 3186 let mut child = GuestCommand::new(&guest) 3187 .args(["--cpus", "boot=1"]) 3188 .args(["--memory", "size=512M"]) 3189 .args(["--kernel", kernel_path.to_str().unwrap()]) 3190 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3191 .default_disks() 3192 .default_net() 3193 .capture_output() 3194 .spawn() 3195 .unwrap(); 3196 3197 let r = std::panic::catch_unwind(|| { 3198 guest.wait_vm_boot(None).unwrap(); 3199 3200 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3201 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3202 3203 let grep_cmd = if cfg!(target_arch = "x86_64") { 3204 "grep -c PCI-MSI /proc/interrupts" 3205 } else { 3206 "grep -c ITS-MSI /proc/interrupts" 3207 }; 3208 assert_eq!( 3209 guest 3210 .ssh_command(grep_cmd) 3211 .unwrap() 3212 .trim() 3213 .parse::<u32>() 3214 .unwrap_or_default(), 3215 12 3216 ); 3217 }); 3218 3219 let _ = child.kill(); 3220 let output = child.wait_with_output().unwrap(); 3221 3222 handle_child_output(r, &output); 3223 } 3224 3225 fn _test_virtio_block(image_name: &str, disable_io_uring: bool, disable_aio: bool) { 3226 let focal = UbuntuDiskConfig::new(image_name.to_string()); 3227 let guest = Guest::new(Box::new(focal)); 3228 3229 let mut workload_path = dirs::home_dir().unwrap(); 3230 workload_path.push("workloads"); 3231 3232 let mut blk_file_path = workload_path; 3233 blk_file_path.push("blk.img"); 3234 3235 let kernel_path = direct_kernel_boot_path(); 3236 3237 let mut cloud_child = GuestCommand::new(&guest) 3238 .args(["--cpus", "boot=4"]) 3239 .args(["--memory", "size=512M,shared=on"]) 3240 .args(["--kernel", kernel_path.to_str().unwrap()]) 3241 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3242 .args([ 3243 "--disk", 3244 format!( 3245 "path={}", 3246 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3247 ) 3248 .as_str(), 3249 format!( 3250 "path={}", 3251 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3252 ) 3253 .as_str(), 3254 format!( 3255 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", 3256 blk_file_path.to_str().unwrap(), 3257 disable_io_uring, 3258 disable_aio, 3259 ) 3260 .as_str(), 3261 ]) 3262 .default_net() 3263 .capture_output() 3264 .spawn() 3265 .unwrap(); 3266 3267 let r = std::panic::catch_unwind(|| { 3268 guest.wait_vm_boot(None).unwrap(); 3269 3270 // Check both if /dev/vdc exists and if the block size is 16M. 3271 assert_eq!( 3272 guest 3273 .ssh_command("lsblk | grep vdc | grep -c 16M") 3274 .unwrap() 3275 .trim() 3276 .parse::<u32>() 3277 .unwrap_or_default(), 3278 1 3279 ); 3280 3281 // Check both if /dev/vdc exists and if this block is RO. 3282 assert_eq!( 3283 guest 3284 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 3285 .unwrap() 3286 .trim() 3287 .parse::<u32>() 3288 .unwrap_or_default(), 3289 1 3290 ); 3291 3292 // Check if the number of queues is 4. 3293 assert_eq!( 3294 guest 3295 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 3296 .unwrap() 3297 .trim() 3298 .parse::<u32>() 3299 .unwrap_or_default(), 3300 4 3301 ); 3302 }); 3303 3304 let _ = cloud_child.kill(); 3305 let output = cloud_child.wait_with_output().unwrap(); 3306 3307 handle_child_output(r, &output); 3308 } 3309 3310 #[test] 3311 fn test_virtio_block_io_uring() { 3312 _test_virtio_block(FOCAL_IMAGE_NAME, false, true) 3313 } 3314 3315 #[test] 3316 fn test_virtio_block_aio() { 3317 _test_virtio_block(FOCAL_IMAGE_NAME, true, false) 3318 } 3319 3320 #[test] 3321 fn test_virtio_block_sync() { 3322 _test_virtio_block(FOCAL_IMAGE_NAME, true, true) 3323 } 3324 3325 #[test] 3326 fn test_virtio_block_qcow2() { 3327 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false, false) 3328 } 3329 3330 #[test] 3331 fn test_virtio_block_qcow2_backing_file() { 3332 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE, false, false) 3333 } 3334 3335 #[test] 3336 fn test_virtio_block_vhd() { 3337 let mut workload_path = dirs::home_dir().unwrap(); 3338 workload_path.push("workloads"); 3339 3340 let mut raw_file_path = workload_path.clone(); 3341 let mut vhd_file_path = workload_path; 3342 raw_file_path.push(FOCAL_IMAGE_NAME); 3343 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 3344 3345 // Generate VHD file from RAW file 3346 std::process::Command::new("qemu-img") 3347 .arg("convert") 3348 .arg("-p") 3349 .args(["-f", "raw"]) 3350 .args(["-O", "vpc"]) 3351 .args(["-o", "subformat=fixed"]) 3352 .arg(raw_file_path.to_str().unwrap()) 3353 .arg(vhd_file_path.to_str().unwrap()) 3354 .output() 3355 .expect("Expect generating VHD image from RAW image"); 3356 3357 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false, false) 3358 } 3359 3360 #[test] 3361 fn test_virtio_block_vhdx() { 3362 let mut workload_path = dirs::home_dir().unwrap(); 3363 workload_path.push("workloads"); 3364 3365 let mut raw_file_path = workload_path.clone(); 3366 let mut vhdx_file_path = workload_path; 3367 raw_file_path.push(FOCAL_IMAGE_NAME); 3368 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 3369 3370 // Generate dynamic VHDX file from RAW file 3371 std::process::Command::new("qemu-img") 3372 .arg("convert") 3373 .arg("-p") 3374 .args(["-f", "raw"]) 3375 .args(["-O", "vhdx"]) 3376 .arg(raw_file_path.to_str().unwrap()) 3377 .arg(vhdx_file_path.to_str().unwrap()) 3378 .output() 3379 .expect("Expect generating dynamic VHDx image from RAW image"); 3380 3381 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false, false) 3382 } 3383 3384 #[test] 3385 fn test_virtio_block_dynamic_vhdx_expand() { 3386 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 3387 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 3388 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 3389 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 3390 3391 let mut workload_path = dirs::home_dir().unwrap(); 3392 workload_path.push("workloads"); 3393 3394 let mut vhdx_file_path = workload_path; 3395 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 3396 let vhdx_path = vhdx_file_path.to_str().unwrap(); 3397 3398 // Generate a 100 MiB dynamic VHDX file 3399 std::process::Command::new("qemu-img") 3400 .arg("create") 3401 .args(["-f", "vhdx"]) 3402 .arg(vhdx_path) 3403 .arg(VIRTUAL_DISK_SIZE.to_string()) 3404 .output() 3405 .expect("Expect generating dynamic VHDx image from RAW image"); 3406 3407 // Check if the size matches with empty VHDx file size 3408 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 3409 3410 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3411 let guest = Guest::new(Box::new(focal)); 3412 let kernel_path = direct_kernel_boot_path(); 3413 3414 let mut cloud_child = GuestCommand::new(&guest) 3415 .args(["--cpus", "boot=1"]) 3416 .args(["--memory", "size=512M"]) 3417 .args(["--kernel", kernel_path.to_str().unwrap()]) 3418 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3419 .args([ 3420 "--disk", 3421 format!( 3422 "path={}", 3423 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3424 ) 3425 .as_str(), 3426 format!( 3427 "path={}", 3428 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3429 ) 3430 .as_str(), 3431 format!("path={vhdx_path}").as_str(), 3432 ]) 3433 .default_net() 3434 .capture_output() 3435 .spawn() 3436 .unwrap(); 3437 3438 let r = std::panic::catch_unwind(|| { 3439 guest.wait_vm_boot(None).unwrap(); 3440 3441 // Check both if /dev/vdc exists and if the block size is 100 MiB. 3442 assert_eq!( 3443 guest 3444 .ssh_command("lsblk | grep vdc | grep -c 100M") 3445 .unwrap() 3446 .trim() 3447 .parse::<u32>() 3448 .unwrap_or_default(), 3449 1 3450 ); 3451 3452 // Write 100 MB of data to the VHDx disk 3453 guest 3454 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 3455 .unwrap(); 3456 }); 3457 3458 // Check if the size matches with expected expanded VHDx file size 3459 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 3460 3461 let _ = cloud_child.kill(); 3462 let output = cloud_child.wait_with_output().unwrap(); 3463 3464 handle_child_output(r, &output); 3465 } 3466 3467 fn vhdx_image_size(disk_name: &str) -> u64 { 3468 std::fs::File::open(disk_name) 3469 .unwrap() 3470 .seek(SeekFrom::End(0)) 3471 .unwrap() 3472 } 3473 3474 #[test] 3475 fn test_virtio_block_direct_and_firmware() { 3476 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3477 let guest = Guest::new(Box::new(focal)); 3478 3479 // The OS disk must be copied to a location that is not backed by 3480 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 3481 // with EINVAL because tmpfs doesn't support this flag. 3482 let mut workloads_path = dirs::home_dir().unwrap(); 3483 workloads_path.push("workloads"); 3484 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 3485 let mut os_path = os_dir.as_path().to_path_buf(); 3486 os_path.push("osdisk.img"); 3487 rate_limited_copy( 3488 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3489 os_path.as_path(), 3490 ) 3491 .expect("copying of OS disk failed"); 3492 3493 let mut child = GuestCommand::new(&guest) 3494 .args(["--cpus", "boot=1"]) 3495 .args(["--memory", "size=512M"]) 3496 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 3497 .args([ 3498 "--disk", 3499 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 3500 format!( 3501 "path={}", 3502 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3503 ) 3504 .as_str(), 3505 ]) 3506 .default_net() 3507 .capture_output() 3508 .spawn() 3509 .unwrap(); 3510 3511 let r = std::panic::catch_unwind(|| { 3512 guest.wait_vm_boot(Some(120)).unwrap(); 3513 }); 3514 3515 let _ = child.kill(); 3516 let output = child.wait_with_output().unwrap(); 3517 3518 handle_child_output(r, &output); 3519 } 3520 3521 #[test] 3522 fn test_vhost_user_net_default() { 3523 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 3524 } 3525 3526 #[test] 3527 fn test_vhost_user_net_named_tap() { 3528 test_vhost_user_net( 3529 Some("mytap0"), 3530 2, 3531 &prepare_vhost_user_net_daemon, 3532 false, 3533 false, 3534 ) 3535 } 3536 3537 #[test] 3538 fn test_vhost_user_net_existing_tap() { 3539 test_vhost_user_net( 3540 Some("vunet-tap0"), 3541 2, 3542 &prepare_vhost_user_net_daemon, 3543 false, 3544 false, 3545 ) 3546 } 3547 3548 #[test] 3549 fn test_vhost_user_net_multiple_queues() { 3550 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 3551 } 3552 3553 #[test] 3554 fn test_vhost_user_net_tap_multiple_queues() { 3555 test_vhost_user_net( 3556 Some("vunet-tap1"), 3557 4, 3558 &prepare_vhost_user_net_daemon, 3559 false, 3560 false, 3561 ) 3562 } 3563 3564 #[test] 3565 fn test_vhost_user_net_host_mac() { 3566 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 3567 } 3568 3569 #[test] 3570 fn test_vhost_user_net_client_mode() { 3571 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 3572 } 3573 3574 #[test] 3575 #[cfg(not(target_arch = "aarch64"))] 3576 fn test_vhost_user_blk_default() { 3577 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 3578 } 3579 3580 #[test] 3581 #[cfg(not(target_arch = "aarch64"))] 3582 fn test_vhost_user_blk_readonly() { 3583 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3584 } 3585 3586 #[test] 3587 #[cfg(not(target_arch = "aarch64"))] 3588 fn test_vhost_user_blk_direct() { 3589 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3590 } 3591 3592 #[test] 3593 fn test_boot_from_vhost_user_blk_default() { 3594 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3595 } 3596 3597 #[test] 3598 #[cfg(target_arch = "x86_64")] 3599 fn test_split_irqchip() { 3600 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3601 let guest = Guest::new(Box::new(focal)); 3602 3603 let mut child = GuestCommand::new(&guest) 3604 .args(["--cpus", "boot=1"]) 3605 .args(["--memory", "size=512M"]) 3606 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3607 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3608 .default_disks() 3609 .default_net() 3610 .capture_output() 3611 .spawn() 3612 .unwrap(); 3613 3614 let r = std::panic::catch_unwind(|| { 3615 guest.wait_vm_boot(None).unwrap(); 3616 3617 assert_eq!( 3618 guest 3619 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3620 .unwrap() 3621 .trim() 3622 .parse::<u32>() 3623 .unwrap_or(1), 3624 0 3625 ); 3626 assert_eq!( 3627 guest 3628 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3629 .unwrap() 3630 .trim() 3631 .parse::<u32>() 3632 .unwrap_or(1), 3633 0 3634 ); 3635 }); 3636 3637 let _ = child.kill(); 3638 let output = child.wait_with_output().unwrap(); 3639 3640 handle_child_output(r, &output); 3641 } 3642 3643 #[test] 3644 #[cfg(target_arch = "x86_64")] 3645 fn test_dmi_serial_number() { 3646 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3647 let guest = Guest::new(Box::new(focal)); 3648 3649 let mut child = GuestCommand::new(&guest) 3650 .args(["--cpus", "boot=1"]) 3651 .args(["--memory", "size=512M"]) 3652 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3653 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3654 .args(["--platform", "serial_number=a=b;c=d"]) 3655 .default_disks() 3656 .default_net() 3657 .capture_output() 3658 .spawn() 3659 .unwrap(); 3660 3661 let r = std::panic::catch_unwind(|| { 3662 guest.wait_vm_boot(None).unwrap(); 3663 3664 assert_eq!( 3665 guest 3666 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3667 .unwrap() 3668 .trim(), 3669 "a=b;c=d" 3670 ); 3671 }); 3672 3673 let _ = child.kill(); 3674 let output = child.wait_with_output().unwrap(); 3675 3676 handle_child_output(r, &output); 3677 } 3678 3679 #[test] 3680 #[cfg(target_arch = "x86_64")] 3681 fn test_dmi_uuid() { 3682 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3683 let guest = Guest::new(Box::new(focal)); 3684 3685 let mut child = GuestCommand::new(&guest) 3686 .args(["--cpus", "boot=1"]) 3687 .args(["--memory", "size=512M"]) 3688 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3689 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3690 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3691 .default_disks() 3692 .default_net() 3693 .capture_output() 3694 .spawn() 3695 .unwrap(); 3696 3697 let r = std::panic::catch_unwind(|| { 3698 guest.wait_vm_boot(None).unwrap(); 3699 3700 assert_eq!( 3701 guest 3702 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3703 .unwrap() 3704 .trim(), 3705 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3706 ); 3707 }); 3708 3709 let _ = child.kill(); 3710 let output = child.wait_with_output().unwrap(); 3711 3712 handle_child_output(r, &output); 3713 } 3714 3715 #[test] 3716 #[cfg(target_arch = "x86_64")] 3717 fn test_dmi_oem_strings() { 3718 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3719 let guest = Guest::new(Box::new(focal)); 3720 3721 let s1 = "io.systemd.credential:xx=yy"; 3722 let s2 = "This is a test string"; 3723 3724 let oem_strings = format!("oem_strings=[{s1},{s2}]"); 3725 3726 let mut child = GuestCommand::new(&guest) 3727 .args(["--cpus", "boot=1"]) 3728 .args(["--memory", "size=512M"]) 3729 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3730 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3731 .args(["--platform", &oem_strings]) 3732 .default_disks() 3733 .default_net() 3734 .capture_output() 3735 .spawn() 3736 .unwrap(); 3737 3738 let r = std::panic::catch_unwind(|| { 3739 guest.wait_vm_boot(None).unwrap(); 3740 3741 assert_eq!( 3742 guest 3743 .ssh_command("sudo dmidecode --oem-string count") 3744 .unwrap() 3745 .trim(), 3746 "2" 3747 ); 3748 3749 assert_eq!( 3750 guest 3751 .ssh_command("sudo dmidecode --oem-string 1") 3752 .unwrap() 3753 .trim(), 3754 s1 3755 ); 3756 3757 assert_eq!( 3758 guest 3759 .ssh_command("sudo dmidecode --oem-string 2") 3760 .unwrap() 3761 .trim(), 3762 s2 3763 ); 3764 }); 3765 3766 let _ = child.kill(); 3767 let output = child.wait_with_output().unwrap(); 3768 3769 handle_child_output(r, &output); 3770 } 3771 3772 #[test] 3773 fn test_virtio_fs() { 3774 _test_virtio_fs(&prepare_virtiofsd, false, None) 3775 } 3776 3777 #[test] 3778 fn test_virtio_fs_hotplug() { 3779 _test_virtio_fs(&prepare_virtiofsd, true, None) 3780 } 3781 3782 #[test] 3783 #[cfg(not(feature = "mshv"))] 3784 fn test_virtio_fs_multi_segment_hotplug() { 3785 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3786 } 3787 3788 #[test] 3789 #[cfg(not(feature = "mshv"))] 3790 fn test_virtio_fs_multi_segment() { 3791 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3792 } 3793 3794 #[test] 3795 fn test_virtio_pmem_persist_writes() { 3796 test_virtio_pmem(false, false) 3797 } 3798 3799 #[test] 3800 fn test_virtio_pmem_discard_writes() { 3801 test_virtio_pmem(true, false) 3802 } 3803 3804 #[test] 3805 fn test_virtio_pmem_with_size() { 3806 test_virtio_pmem(true, true) 3807 } 3808 3809 #[test] 3810 fn test_boot_from_virtio_pmem() { 3811 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3812 let guest = Guest::new(Box::new(focal)); 3813 3814 let kernel_path = direct_kernel_boot_path(); 3815 3816 let mut child = GuestCommand::new(&guest) 3817 .args(["--cpus", "boot=1"]) 3818 .args(["--memory", "size=512M"]) 3819 .args(["--kernel", kernel_path.to_str().unwrap()]) 3820 .args([ 3821 "--disk", 3822 format!( 3823 "path={}", 3824 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3825 ) 3826 .as_str(), 3827 ]) 3828 .default_net() 3829 .args([ 3830 "--pmem", 3831 format!( 3832 "file={},size={}", 3833 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3834 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3835 .unwrap() 3836 .len() 3837 ) 3838 .as_str(), 3839 ]) 3840 .args([ 3841 "--cmdline", 3842 DIRECT_KERNEL_BOOT_CMDLINE 3843 .replace("vda1", "pmem0p1") 3844 .as_str(), 3845 ]) 3846 .capture_output() 3847 .spawn() 3848 .unwrap(); 3849 3850 let r = std::panic::catch_unwind(|| { 3851 guest.wait_vm_boot(None).unwrap(); 3852 3853 // Simple checks to validate the VM booted properly 3854 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3855 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3856 }); 3857 3858 let _ = child.kill(); 3859 let output = child.wait_with_output().unwrap(); 3860 3861 handle_child_output(r, &output); 3862 } 3863 3864 #[test] 3865 fn test_multiple_network_interfaces() { 3866 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3867 let guest = Guest::new(Box::new(focal)); 3868 3869 let kernel_path = direct_kernel_boot_path(); 3870 3871 let mut child = GuestCommand::new(&guest) 3872 .args(["--cpus", "boot=1"]) 3873 .args(["--memory", "size=512M"]) 3874 .args(["--kernel", kernel_path.to_str().unwrap()]) 3875 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3876 .default_disks() 3877 .args([ 3878 "--net", 3879 guest.default_net_string().as_str(), 3880 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3881 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3882 ]) 3883 .capture_output() 3884 .spawn() 3885 .unwrap(); 3886 3887 let r = std::panic::catch_unwind(|| { 3888 guest.wait_vm_boot(None).unwrap(); 3889 3890 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3891 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3892 3893 // 3 network interfaces + default localhost ==> 4 interfaces 3894 assert_eq!( 3895 guest 3896 .ssh_command("ip -o link | wc -l") 3897 .unwrap() 3898 .trim() 3899 .parse::<u32>() 3900 .unwrap_or_default(), 3901 4 3902 ); 3903 }); 3904 3905 let _ = child.kill(); 3906 let output = child.wait_with_output().unwrap(); 3907 3908 handle_child_output(r, &output); 3909 } 3910 3911 #[test] 3912 #[cfg(target_arch = "aarch64")] 3913 fn test_pmu_on() { 3914 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3915 let guest = Guest::new(Box::new(focal)); 3916 let mut child = GuestCommand::new(&guest) 3917 .args(["--cpus", "boot=1"]) 3918 .args(["--memory", "size=512M"]) 3919 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3920 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3921 .default_disks() 3922 .default_net() 3923 .capture_output() 3924 .spawn() 3925 .unwrap(); 3926 3927 let r = std::panic::catch_unwind(|| { 3928 guest.wait_vm_boot(None).unwrap(); 3929 3930 // Test that PMU exists. 3931 assert_eq!( 3932 guest 3933 .ssh_command(GREP_PMU_IRQ_CMD) 3934 .unwrap() 3935 .trim() 3936 .parse::<u32>() 3937 .unwrap_or_default(), 3938 1 3939 ); 3940 }); 3941 3942 let _ = child.kill(); 3943 let output = child.wait_with_output().unwrap(); 3944 3945 handle_child_output(r, &output); 3946 } 3947 3948 #[test] 3949 fn test_serial_off() { 3950 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3951 let guest = Guest::new(Box::new(focal)); 3952 let mut child = GuestCommand::new(&guest) 3953 .args(["--cpus", "boot=1"]) 3954 .args(["--memory", "size=512M"]) 3955 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3956 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3957 .default_disks() 3958 .default_net() 3959 .args(["--serial", "off"]) 3960 .capture_output() 3961 .spawn() 3962 .unwrap(); 3963 3964 let r = std::panic::catch_unwind(|| { 3965 guest.wait_vm_boot(None).unwrap(); 3966 3967 // Test that there is no ttyS0 3968 assert_eq!( 3969 guest 3970 .ssh_command(GREP_SERIAL_IRQ_CMD) 3971 .unwrap() 3972 .trim() 3973 .parse::<u32>() 3974 .unwrap_or(1), 3975 0 3976 ); 3977 }); 3978 3979 let _ = child.kill(); 3980 let output = child.wait_with_output().unwrap(); 3981 3982 handle_child_output(r, &output); 3983 } 3984 3985 #[test] 3986 fn test_serial_null() { 3987 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3988 let guest = Guest::new(Box::new(focal)); 3989 let mut cmd = GuestCommand::new(&guest); 3990 #[cfg(target_arch = "x86_64")] 3991 let console_str: &str = "console=ttyS0"; 3992 #[cfg(target_arch = "aarch64")] 3993 let console_str: &str = "console=ttyAMA0"; 3994 3995 cmd.args(["--cpus", "boot=1"]) 3996 .args(["--memory", "size=512M"]) 3997 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3998 .args([ 3999 "--cmdline", 4000 DIRECT_KERNEL_BOOT_CMDLINE 4001 .replace("console=hvc0 ", console_str) 4002 .as_str(), 4003 ]) 4004 .default_disks() 4005 .default_net() 4006 .args(["--serial", "null"]) 4007 .args(["--console", "off"]) 4008 .capture_output(); 4009 4010 let mut child = cmd.spawn().unwrap(); 4011 4012 let r = std::panic::catch_unwind(|| { 4013 guest.wait_vm_boot(None).unwrap(); 4014 4015 // Test that there is a ttyS0 4016 assert_eq!( 4017 guest 4018 .ssh_command(GREP_SERIAL_IRQ_CMD) 4019 .unwrap() 4020 .trim() 4021 .parse::<u32>() 4022 .unwrap_or_default(), 4023 1 4024 ); 4025 }); 4026 4027 let _ = child.kill(); 4028 let output = child.wait_with_output().unwrap(); 4029 handle_child_output(r, &output); 4030 4031 let r = std::panic::catch_unwind(|| { 4032 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4033 }); 4034 4035 handle_child_output(r, &output); 4036 } 4037 4038 #[test] 4039 fn test_serial_tty() { 4040 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4041 let guest = Guest::new(Box::new(focal)); 4042 4043 let kernel_path = direct_kernel_boot_path(); 4044 4045 #[cfg(target_arch = "x86_64")] 4046 let console_str: &str = "console=ttyS0"; 4047 #[cfg(target_arch = "aarch64")] 4048 let console_str: &str = "console=ttyAMA0"; 4049 4050 let mut child = GuestCommand::new(&guest) 4051 .args(["--cpus", "boot=1"]) 4052 .args(["--memory", "size=512M"]) 4053 .args(["--kernel", kernel_path.to_str().unwrap()]) 4054 .args([ 4055 "--cmdline", 4056 DIRECT_KERNEL_BOOT_CMDLINE 4057 .replace("console=hvc0 ", console_str) 4058 .as_str(), 4059 ]) 4060 .default_disks() 4061 .default_net() 4062 .args(["--serial", "tty"]) 4063 .args(["--console", "off"]) 4064 .capture_output() 4065 .spawn() 4066 .unwrap(); 4067 4068 let r = std::panic::catch_unwind(|| { 4069 guest.wait_vm_boot(None).unwrap(); 4070 4071 // Test that there is a ttyS0 4072 assert_eq!( 4073 guest 4074 .ssh_command(GREP_SERIAL_IRQ_CMD) 4075 .unwrap() 4076 .trim() 4077 .parse::<u32>() 4078 .unwrap_or_default(), 4079 1 4080 ); 4081 }); 4082 4083 // This sleep is needed to wait for the login prompt 4084 thread::sleep(std::time::Duration::new(2, 0)); 4085 4086 let _ = child.kill(); 4087 let output = child.wait_with_output().unwrap(); 4088 handle_child_output(r, &output); 4089 4090 let r = std::panic::catch_unwind(|| { 4091 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4092 }); 4093 4094 handle_child_output(r, &output); 4095 } 4096 4097 #[test] 4098 fn test_serial_file() { 4099 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4100 let guest = Guest::new(Box::new(focal)); 4101 4102 let serial_path = guest.tmp_dir.as_path().join("serial-output"); 4103 #[cfg(target_arch = "x86_64")] 4104 let console_str: &str = "console=ttyS0"; 4105 #[cfg(target_arch = "aarch64")] 4106 let console_str: &str = "console=ttyAMA0"; 4107 4108 let mut child = GuestCommand::new(&guest) 4109 .args(["--cpus", "boot=1"]) 4110 .args(["--memory", "size=512M"]) 4111 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4112 .args([ 4113 "--cmdline", 4114 DIRECT_KERNEL_BOOT_CMDLINE 4115 .replace("console=hvc0 ", console_str) 4116 .as_str(), 4117 ]) 4118 .default_disks() 4119 .default_net() 4120 .args([ 4121 "--serial", 4122 format!("file={}", serial_path.to_str().unwrap()).as_str(), 4123 ]) 4124 .capture_output() 4125 .spawn() 4126 .unwrap(); 4127 4128 let r = std::panic::catch_unwind(|| { 4129 guest.wait_vm_boot(None).unwrap(); 4130 4131 // Test that there is a ttyS0 4132 assert_eq!( 4133 guest 4134 .ssh_command(GREP_SERIAL_IRQ_CMD) 4135 .unwrap() 4136 .trim() 4137 .parse::<u32>() 4138 .unwrap_or_default(), 4139 1 4140 ); 4141 4142 guest.ssh_command("sudo shutdown -h now").unwrap(); 4143 }); 4144 4145 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4146 let _ = child.kill(); 4147 let output = child.wait_with_output().unwrap(); 4148 handle_child_output(r, &output); 4149 4150 let r = std::panic::catch_unwind(|| { 4151 // Check that the cloud-hypervisor binary actually terminated 4152 assert!(output.status.success()); 4153 4154 // Do this check after shutdown of the VM as an easy way to ensure 4155 // all writes are flushed to disk 4156 let mut f = std::fs::File::open(serial_path).unwrap(); 4157 let mut buf = String::new(); 4158 f.read_to_string(&mut buf).unwrap(); 4159 assert!(buf.contains(CONSOLE_TEST_STRING)); 4160 }); 4161 4162 handle_child_output(r, &output); 4163 } 4164 4165 #[test] 4166 fn test_pty_interaction() { 4167 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4168 let guest = Guest::new(Box::new(focal)); 4169 let api_socket = temp_api_path(&guest.tmp_dir); 4170 let serial_option = if cfg!(target_arch = "x86_64") { 4171 " console=ttyS0" 4172 } else { 4173 " console=ttyAMA0" 4174 }; 4175 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4176 4177 let mut child = GuestCommand::new(&guest) 4178 .args(["--cpus", "boot=1"]) 4179 .args(["--memory", "size=512M"]) 4180 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4181 .args(["--cmdline", &cmdline]) 4182 .default_disks() 4183 .default_net() 4184 .args(["--serial", "null"]) 4185 .args(["--console", "pty"]) 4186 .args(["--api-socket", &api_socket]) 4187 .spawn() 4188 .unwrap(); 4189 4190 let r = std::panic::catch_unwind(|| { 4191 guest.wait_vm_boot(None).unwrap(); 4192 // Get pty fd for console 4193 let console_path = get_pty_path(&api_socket, "console"); 4194 _test_pty_interaction(console_path); 4195 4196 guest.ssh_command("sudo shutdown -h now").unwrap(); 4197 }); 4198 4199 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4200 let _ = child.kill(); 4201 let output = child.wait_with_output().unwrap(); 4202 handle_child_output(r, &output); 4203 4204 let r = std::panic::catch_unwind(|| { 4205 // Check that the cloud-hypervisor binary actually terminated 4206 assert!(output.status.success()) 4207 }); 4208 handle_child_output(r, &output); 4209 } 4210 4211 #[test] 4212 fn test_serial_socket_interaction() { 4213 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4214 let guest = Guest::new(Box::new(focal)); 4215 let serial_socket = guest.tmp_dir.as_path().join("serial.socket"); 4216 let serial_socket_pty = guest.tmp_dir.as_path().join("serial.pty"); 4217 let serial_option = if cfg!(target_arch = "x86_64") { 4218 " console=ttyS0" 4219 } else { 4220 " console=ttyAMA0" 4221 }; 4222 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4223 4224 let mut child = GuestCommand::new(&guest) 4225 .args(["--cpus", "boot=1"]) 4226 .args(["--memory", "size=512M"]) 4227 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4228 .args(["--cmdline", &cmdline]) 4229 .default_disks() 4230 .default_net() 4231 .args(["--console", "null"]) 4232 .args([ 4233 "--serial", 4234 format!("socket={}", serial_socket.to_str().unwrap()).as_str(), 4235 ]) 4236 .spawn() 4237 .unwrap(); 4238 4239 let _ = std::panic::catch_unwind(|| { 4240 guest.wait_vm_boot(None).unwrap(); 4241 }); 4242 4243 let mut socat_command = Command::new("socat"); 4244 let socat_args = [ 4245 &format!("pty,link={},raw", serial_socket_pty.display()), 4246 &format!("UNIX-CONNECT:{}", serial_socket.display()), 4247 ]; 4248 socat_command.args(socat_args); 4249 4250 let mut socat_child = socat_command.spawn().unwrap(); 4251 thread::sleep(std::time::Duration::new(1, 0)); 4252 4253 let _ = std::panic::catch_unwind(|| { 4254 _test_pty_interaction(serial_socket_pty); 4255 }); 4256 4257 let _ = socat_child.kill(); 4258 4259 let r = std::panic::catch_unwind(|| { 4260 guest.ssh_command("sudo shutdown -h now").unwrap(); 4261 }); 4262 4263 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4264 let _ = child.kill(); 4265 let output = child.wait_with_output().unwrap(); 4266 handle_child_output(r, &output); 4267 4268 let r = std::panic::catch_unwind(|| { 4269 // Check that the cloud-hypervisor binary actually terminated 4270 if !output.status.success() { 4271 panic!( 4272 "Cloud Hypervisor process failed to terminate gracefully: {:?}", 4273 output.status 4274 ); 4275 } 4276 }); 4277 handle_child_output(r, &output); 4278 } 4279 4280 #[test] 4281 fn test_virtio_console() { 4282 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4283 let guest = Guest::new(Box::new(focal)); 4284 4285 let kernel_path = direct_kernel_boot_path(); 4286 4287 let mut child = GuestCommand::new(&guest) 4288 .args(["--cpus", "boot=1"]) 4289 .args(["--memory", "size=512M"]) 4290 .args(["--kernel", kernel_path.to_str().unwrap()]) 4291 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4292 .default_disks() 4293 .default_net() 4294 .args(["--console", "tty"]) 4295 .args(["--serial", "null"]) 4296 .capture_output() 4297 .spawn() 4298 .unwrap(); 4299 4300 let text = String::from("On a branch floating down river a cricket, singing."); 4301 let cmd = format!("echo {text} | sudo tee /dev/hvc0"); 4302 4303 let r = std::panic::catch_unwind(|| { 4304 guest.wait_vm_boot(None).unwrap(); 4305 4306 assert!(guest 4307 .does_device_vendor_pair_match("0x1043", "0x1af4") 4308 .unwrap_or_default()); 4309 4310 guest.ssh_command(&cmd).unwrap(); 4311 }); 4312 4313 let _ = child.kill(); 4314 let output = child.wait_with_output().unwrap(); 4315 handle_child_output(r, &output); 4316 4317 let r = std::panic::catch_unwind(|| { 4318 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 4319 }); 4320 4321 handle_child_output(r, &output); 4322 } 4323 4324 #[test] 4325 fn test_console_file() { 4326 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4327 let guest = Guest::new(Box::new(focal)); 4328 4329 let console_path = guest.tmp_dir.as_path().join("console-output"); 4330 let mut child = GuestCommand::new(&guest) 4331 .args(["--cpus", "boot=1"]) 4332 .args(["--memory", "size=512M"]) 4333 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4334 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4335 .default_disks() 4336 .default_net() 4337 .args([ 4338 "--console", 4339 format!("file={}", console_path.to_str().unwrap()).as_str(), 4340 ]) 4341 .capture_output() 4342 .spawn() 4343 .unwrap(); 4344 4345 guest.wait_vm_boot(None).unwrap(); 4346 4347 guest.ssh_command("sudo shutdown -h now").unwrap(); 4348 4349 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4350 let _ = child.kill(); 4351 let output = child.wait_with_output().unwrap(); 4352 4353 let r = std::panic::catch_unwind(|| { 4354 // Check that the cloud-hypervisor binary actually terminated 4355 assert!(output.status.success()); 4356 4357 // Do this check after shutdown of the VM as an easy way to ensure 4358 // all writes are flushed to disk 4359 let mut f = std::fs::File::open(console_path).unwrap(); 4360 let mut buf = String::new(); 4361 f.read_to_string(&mut buf).unwrap(); 4362 4363 if !buf.contains(CONSOLE_TEST_STRING) { 4364 eprintln!( 4365 "\n\n==== Console file output ====\n\n{buf}\n\n==== End console file output ====" 4366 ); 4367 } 4368 assert!(buf.contains(CONSOLE_TEST_STRING)); 4369 }); 4370 4371 handle_child_output(r, &output); 4372 } 4373 4374 #[test] 4375 #[cfg(target_arch = "x86_64")] 4376 #[cfg(not(feature = "mshv"))] 4377 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 4378 // backed networking interfaces, bound through a simple bridge on the host. 4379 // So if the nested cloud-hypervisor succeeds in getting a directly 4380 // assigned interface from its cloud-hypervisor host, we should be able to 4381 // ssh into it, and verify that it's running with the right kernel command 4382 // line (We tag the command line from cloud-hypervisor for that purpose). 4383 // The third device is added to validate that hotplug works correctly since 4384 // it is being added to the L2 VM through hotplugging mechanism. 4385 // Also, we pass-through a virtio-blk device to the L2 VM to test the 32-bit 4386 // vfio device support 4387 fn test_vfio() { 4388 setup_vfio_network_interfaces(); 4389 4390 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 4391 let guest = Guest::new_from_ip_range(Box::new(jammy), "172.18", 0); 4392 4393 let mut workload_path = dirs::home_dir().unwrap(); 4394 workload_path.push("workloads"); 4395 4396 let kernel_path = direct_kernel_boot_path(); 4397 4398 let mut vfio_path = workload_path.clone(); 4399 vfio_path.push("vfio"); 4400 4401 let mut cloud_init_vfio_base_path = vfio_path.clone(); 4402 cloud_init_vfio_base_path.push("cloudinit.img"); 4403 4404 // We copy our cloudinit into the vfio mount point, for the nested 4405 // cloud-hypervisor guest to use. 4406 rate_limited_copy( 4407 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 4408 &cloud_init_vfio_base_path, 4409 ) 4410 .expect("copying of cloud-init disk failed"); 4411 4412 let mut vfio_disk_path = workload_path.clone(); 4413 vfio_disk_path.push("vfio.img"); 4414 4415 // Create the vfio disk image 4416 let output = Command::new("mkfs.ext4") 4417 .arg("-d") 4418 .arg(vfio_path.to_str().unwrap()) 4419 .arg(vfio_disk_path.to_str().unwrap()) 4420 .arg("2g") 4421 .output() 4422 .unwrap(); 4423 if !output.status.success() { 4424 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 4425 panic!("mkfs.ext4 command generated an error"); 4426 } 4427 4428 let mut blk_file_path = workload_path; 4429 blk_file_path.push("blk.img"); 4430 4431 let vfio_tap0 = "vfio-tap0"; 4432 let vfio_tap1 = "vfio-tap1"; 4433 let vfio_tap2 = "vfio-tap2"; 4434 let vfio_tap3 = "vfio-tap3"; 4435 4436 let mut child = GuestCommand::new(&guest) 4437 .args(["--cpus", "boot=4"]) 4438 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 4439 .args(["--kernel", kernel_path.to_str().unwrap()]) 4440 .args([ 4441 "--disk", 4442 format!( 4443 "path={}", 4444 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4445 ) 4446 .as_str(), 4447 format!( 4448 "path={}", 4449 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4450 ) 4451 .as_str(), 4452 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 4453 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 4454 ]) 4455 .args([ 4456 "--cmdline", 4457 format!( 4458 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 4459 ) 4460 .as_str(), 4461 ]) 4462 .args([ 4463 "--net", 4464 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 4465 format!( 4466 "tap={},mac={},iommu=on", 4467 vfio_tap1, guest.network.l2_guest_mac1 4468 ) 4469 .as_str(), 4470 format!( 4471 "tap={},mac={},iommu=on", 4472 vfio_tap2, guest.network.l2_guest_mac2 4473 ) 4474 .as_str(), 4475 format!( 4476 "tap={},mac={},iommu=on", 4477 vfio_tap3, guest.network.l2_guest_mac3 4478 ) 4479 .as_str(), 4480 ]) 4481 .capture_output() 4482 .spawn() 4483 .unwrap(); 4484 4485 thread::sleep(std::time::Duration::new(30, 0)); 4486 4487 let r = std::panic::catch_unwind(|| { 4488 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 4489 thread::sleep(std::time::Duration::new(120, 0)); 4490 4491 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 4492 // added to its kernel command line. 4493 // Let's ssh into it and verify that it's there. If it is it means 4494 // we're in the right guest (The L2 one) because the QEMU L1 guest 4495 // does not have this command line tag. 4496 assert!(check_matched_lines_count( 4497 guest.ssh_command_l2_1("cat /proc/cmdline").unwrap().trim(), 4498 vec!["VFIOTAG"], 4499 1 4500 )); 4501 4502 // Let's also verify from the second virtio-net device passed to 4503 // the L2 VM. 4504 assert!(check_matched_lines_count( 4505 guest.ssh_command_l2_2("cat /proc/cmdline").unwrap().trim(), 4506 vec!["VFIOTAG"], 4507 1 4508 )); 4509 4510 // Check the amount of PCI devices appearing in L2 VM. 4511 assert!(check_lines_count( 4512 guest 4513 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4514 .unwrap() 4515 .trim(), 4516 8 4517 )); 4518 4519 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 4520 assert!(check_matched_lines_count( 4521 guest.ssh_command_l2_1("lsblk").unwrap().trim(), 4522 vec!["vdc", "16M"], 4523 1 4524 )); 4525 4526 // Hotplug an extra virtio-net device through L2 VM. 4527 guest 4528 .ssh_command_l1( 4529 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 4530 ) 4531 .unwrap(); 4532 guest 4533 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 4534 .unwrap(); 4535 let vfio_hotplug_output = guest 4536 .ssh_command_l1( 4537 "sudo /mnt/ch-remote \ 4538 --api-socket=/tmp/ch_api.sock \ 4539 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 4540 ) 4541 .unwrap(); 4542 assert!(check_matched_lines_count( 4543 vfio_hotplug_output.trim(), 4544 vec!["{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}"], 4545 1 4546 )); 4547 4548 thread::sleep(std::time::Duration::new(10, 0)); 4549 4550 // Let's also verify from the third virtio-net device passed to 4551 // the L2 VM. This third device has been hotplugged through the L2 4552 // VM, so this is our way to validate hotplug works for VFIO PCI. 4553 assert!(check_matched_lines_count( 4554 guest.ssh_command_l2_3("cat /proc/cmdline").unwrap().trim(), 4555 vec!["VFIOTAG"], 4556 1 4557 )); 4558 4559 // Check the amount of PCI devices appearing in L2 VM. 4560 // There should be one more device than before, raising the count 4561 // up to 9 PCI devices. 4562 assert!(check_lines_count( 4563 guest 4564 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4565 .unwrap() 4566 .trim(), 4567 9 4568 )); 4569 4570 // Let's now verify that we can correctly remove the virtio-net 4571 // device through the "remove-device" command responsible for 4572 // unplugging VFIO devices. 4573 guest 4574 .ssh_command_l1( 4575 "sudo /mnt/ch-remote \ 4576 --api-socket=/tmp/ch_api.sock \ 4577 remove-device vfio123", 4578 ) 4579 .unwrap(); 4580 thread::sleep(std::time::Duration::new(10, 0)); 4581 4582 // Check the amount of PCI devices appearing in L2 VM is back down 4583 // to 8 devices. 4584 assert!(check_lines_count( 4585 guest 4586 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4587 .unwrap() 4588 .trim(), 4589 8 4590 )); 4591 4592 // Perform memory hotplug in L2 and validate the memory is showing 4593 // up as expected. In order to check, we will use the virtio-net 4594 // device already passed through L2 as a VFIO device, this will 4595 // verify that VFIO devices are functional with memory hotplug. 4596 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 4597 guest 4598 .ssh_command_l2_1( 4599 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 4600 ) 4601 .unwrap(); 4602 guest 4603 .ssh_command_l1( 4604 "sudo /mnt/ch-remote \ 4605 --api-socket=/tmp/ch_api.sock \ 4606 resize --memory=1073741824", 4607 ) 4608 .unwrap(); 4609 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 4610 }); 4611 4612 let _ = child.kill(); 4613 let output = child.wait_with_output().unwrap(); 4614 4615 cleanup_vfio_network_interfaces(); 4616 4617 handle_child_output(r, &output); 4618 } 4619 4620 #[test] 4621 fn test_direct_kernel_boot_noacpi() { 4622 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4623 let guest = Guest::new(Box::new(focal)); 4624 4625 let kernel_path = direct_kernel_boot_path(); 4626 4627 let mut child = GuestCommand::new(&guest) 4628 .args(["--cpus", "boot=1"]) 4629 .args(["--memory", "size=512M"]) 4630 .args(["--kernel", kernel_path.to_str().unwrap()]) 4631 .args([ 4632 "--cmdline", 4633 format!("{DIRECT_KERNEL_BOOT_CMDLINE} acpi=off").as_str(), 4634 ]) 4635 .default_disks() 4636 .default_net() 4637 .capture_output() 4638 .spawn() 4639 .unwrap(); 4640 4641 let r = std::panic::catch_unwind(|| { 4642 guest.wait_vm_boot(None).unwrap(); 4643 4644 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4645 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4646 }); 4647 4648 let _ = child.kill(); 4649 let output = child.wait_with_output().unwrap(); 4650 4651 handle_child_output(r, &output); 4652 } 4653 4654 #[test] 4655 fn test_virtio_vsock() { 4656 _test_virtio_vsock(false) 4657 } 4658 4659 #[test] 4660 fn test_virtio_vsock_hotplug() { 4661 _test_virtio_vsock(true); 4662 } 4663 4664 #[test] 4665 fn test_api_http_shutdown() { 4666 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4667 let guest = Guest::new(Box::new(focal)); 4668 4669 _test_api_shutdown(TargetApi::new_http_api(&guest.tmp_dir), guest) 4670 } 4671 4672 #[test] 4673 fn test_api_http_delete() { 4674 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4675 let guest = Guest::new(Box::new(focal)); 4676 4677 _test_api_delete(TargetApi::new_http_api(&guest.tmp_dir), guest); 4678 } 4679 4680 #[test] 4681 fn test_api_http_pause_resume() { 4682 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4683 let guest = Guest::new(Box::new(focal)); 4684 4685 _test_api_pause_resume(TargetApi::new_http_api(&guest.tmp_dir), guest) 4686 } 4687 4688 #[test] 4689 fn test_api_http_create_boot() { 4690 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4691 let guest = Guest::new(Box::new(focal)); 4692 4693 _test_api_create_boot(TargetApi::new_http_api(&guest.tmp_dir), guest) 4694 } 4695 4696 #[test] 4697 fn test_virtio_iommu() { 4698 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4699 } 4700 4701 #[test] 4702 // We cannot force the software running in the guest to reprogram the BAR 4703 // with some different addresses, but we have a reliable way of testing it 4704 // with a standard Linux kernel. 4705 // By removing a device from the PCI tree, and then rescanning the tree, 4706 // Linux consistently chooses to reorganize the PCI device BARs to other 4707 // locations in the guest address space. 4708 // This test creates a dedicated PCI network device to be checked as being 4709 // properly probed first, then removing it, and adding it again by doing a 4710 // rescan. 4711 fn test_pci_bar_reprogramming() { 4712 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4713 let guest = Guest::new(Box::new(focal)); 4714 4715 #[cfg(target_arch = "x86_64")] 4716 let kernel_path = direct_kernel_boot_path(); 4717 #[cfg(target_arch = "aarch64")] 4718 let kernel_path = edk2_path(); 4719 4720 let mut child = GuestCommand::new(&guest) 4721 .args(["--cpus", "boot=1"]) 4722 .args(["--memory", "size=512M"]) 4723 .args(["--kernel", kernel_path.to_str().unwrap()]) 4724 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4725 .default_disks() 4726 .args([ 4727 "--net", 4728 guest.default_net_string().as_str(), 4729 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4730 ]) 4731 .capture_output() 4732 .spawn() 4733 .unwrap(); 4734 4735 let r = std::panic::catch_unwind(|| { 4736 guest.wait_vm_boot(None).unwrap(); 4737 4738 // 2 network interfaces + default localhost ==> 3 interfaces 4739 assert_eq!( 4740 guest 4741 .ssh_command("ip -o link | wc -l") 4742 .unwrap() 4743 .trim() 4744 .parse::<u32>() 4745 .unwrap_or_default(), 4746 3 4747 ); 4748 4749 let init_bar_addr = guest 4750 .ssh_command( 4751 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4752 ) 4753 .unwrap(); 4754 4755 // Remove the PCI device 4756 guest 4757 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4758 .unwrap(); 4759 4760 // Only 1 network interface left + default localhost ==> 2 interfaces 4761 assert_eq!( 4762 guest 4763 .ssh_command("ip -o link | wc -l") 4764 .unwrap() 4765 .trim() 4766 .parse::<u32>() 4767 .unwrap_or_default(), 4768 2 4769 ); 4770 4771 // Remove the PCI device 4772 guest 4773 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4774 .unwrap(); 4775 4776 // Back to 2 network interface + default localhost ==> 3 interfaces 4777 assert_eq!( 4778 guest 4779 .ssh_command("ip -o link | wc -l") 4780 .unwrap() 4781 .trim() 4782 .parse::<u32>() 4783 .unwrap_or_default(), 4784 3 4785 ); 4786 4787 let new_bar_addr = guest 4788 .ssh_command( 4789 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4790 ) 4791 .unwrap(); 4792 4793 // Let's compare the BAR addresses for our virtio-net device. 4794 // They should be different as we expect the BAR reprogramming 4795 // to have happened. 4796 assert_ne!(init_bar_addr, new_bar_addr); 4797 }); 4798 4799 let _ = child.kill(); 4800 let output = child.wait_with_output().unwrap(); 4801 4802 handle_child_output(r, &output); 4803 } 4804 4805 #[test] 4806 fn test_memory_mergeable_off() { 4807 test_memory_mergeable(false) 4808 } 4809 4810 #[test] 4811 #[cfg(target_arch = "x86_64")] 4812 fn test_cpu_hotplug() { 4813 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4814 let guest = Guest::new(Box::new(focal)); 4815 let api_socket = temp_api_path(&guest.tmp_dir); 4816 4817 let kernel_path = direct_kernel_boot_path(); 4818 4819 let mut child = GuestCommand::new(&guest) 4820 .args(["--cpus", "boot=2,max=4"]) 4821 .args(["--memory", "size=512M"]) 4822 .args(["--kernel", kernel_path.to_str().unwrap()]) 4823 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4824 .default_disks() 4825 .default_net() 4826 .args(["--api-socket", &api_socket]) 4827 .capture_output() 4828 .spawn() 4829 .unwrap(); 4830 4831 let r = std::panic::catch_unwind(|| { 4832 guest.wait_vm_boot(None).unwrap(); 4833 4834 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4835 4836 // Resize the VM 4837 let desired_vcpus = 4; 4838 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4839 4840 guest 4841 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4842 .unwrap(); 4843 guest 4844 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4845 .unwrap(); 4846 thread::sleep(std::time::Duration::new(10, 0)); 4847 assert_eq!( 4848 guest.get_cpu_count().unwrap_or_default(), 4849 u32::from(desired_vcpus) 4850 ); 4851 4852 guest.reboot_linux(0, None); 4853 4854 assert_eq!( 4855 guest.get_cpu_count().unwrap_or_default(), 4856 u32::from(desired_vcpus) 4857 ); 4858 4859 // Resize the VM 4860 let desired_vcpus = 2; 4861 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4862 4863 thread::sleep(std::time::Duration::new(10, 0)); 4864 assert_eq!( 4865 guest.get_cpu_count().unwrap_or_default(), 4866 u32::from(desired_vcpus) 4867 ); 4868 4869 // Resize the VM back up to 4 4870 let desired_vcpus = 4; 4871 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4872 4873 guest 4874 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4875 .unwrap(); 4876 guest 4877 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4878 .unwrap(); 4879 thread::sleep(std::time::Duration::new(10, 0)); 4880 assert_eq!( 4881 guest.get_cpu_count().unwrap_or_default(), 4882 u32::from(desired_vcpus) 4883 ); 4884 }); 4885 4886 let _ = child.kill(); 4887 let output = child.wait_with_output().unwrap(); 4888 4889 handle_child_output(r, &output); 4890 } 4891 4892 #[test] 4893 fn test_memory_hotplug() { 4894 #[cfg(target_arch = "aarch64")] 4895 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4896 #[cfg(target_arch = "x86_64")] 4897 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4898 let focal = UbuntuDiskConfig::new(focal_image); 4899 let guest = Guest::new(Box::new(focal)); 4900 let api_socket = temp_api_path(&guest.tmp_dir); 4901 4902 #[cfg(target_arch = "aarch64")] 4903 let kernel_path = edk2_path(); 4904 #[cfg(target_arch = "x86_64")] 4905 let kernel_path = direct_kernel_boot_path(); 4906 4907 let mut child = GuestCommand::new(&guest) 4908 .args(["--cpus", "boot=2,max=4"]) 4909 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4910 .args(["--kernel", kernel_path.to_str().unwrap()]) 4911 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4912 .default_disks() 4913 .default_net() 4914 .args(["--balloon", "size=0"]) 4915 .args(["--api-socket", &api_socket]) 4916 .capture_output() 4917 .spawn() 4918 .unwrap(); 4919 4920 let r = std::panic::catch_unwind(|| { 4921 guest.wait_vm_boot(None).unwrap(); 4922 4923 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4924 4925 guest.enable_memory_hotplug(); 4926 4927 // Add RAM to the VM 4928 let desired_ram = 1024 << 20; 4929 resize_command(&api_socket, None, Some(desired_ram), None, None); 4930 4931 thread::sleep(std::time::Duration::new(10, 0)); 4932 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4933 4934 // Use balloon to remove RAM from the VM 4935 let desired_balloon = 512 << 20; 4936 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4937 4938 thread::sleep(std::time::Duration::new(10, 0)); 4939 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4940 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4941 4942 guest.reboot_linux(0, None); 4943 4944 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4945 4946 // Use balloon add RAM to the VM 4947 let desired_balloon = 0; 4948 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4949 4950 thread::sleep(std::time::Duration::new(10, 0)); 4951 4952 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4953 4954 guest.enable_memory_hotplug(); 4955 4956 // Add RAM to the VM 4957 let desired_ram = 2048 << 20; 4958 resize_command(&api_socket, None, Some(desired_ram), None, None); 4959 4960 thread::sleep(std::time::Duration::new(10, 0)); 4961 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4962 4963 // Remove RAM to the VM (only applies after reboot) 4964 let desired_ram = 1024 << 20; 4965 resize_command(&api_socket, None, Some(desired_ram), None, None); 4966 4967 guest.reboot_linux(1, None); 4968 4969 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4970 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4971 }); 4972 4973 let _ = child.kill(); 4974 let output = child.wait_with_output().unwrap(); 4975 4976 handle_child_output(r, &output); 4977 } 4978 4979 #[test] 4980 #[cfg(not(feature = "mshv"))] 4981 fn test_virtio_mem() { 4982 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4983 let guest = Guest::new(Box::new(focal)); 4984 let api_socket = temp_api_path(&guest.tmp_dir); 4985 4986 let kernel_path = direct_kernel_boot_path(); 4987 4988 let mut child = GuestCommand::new(&guest) 4989 .args(["--cpus", "boot=2,max=4"]) 4990 .args([ 4991 "--memory", 4992 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4993 ]) 4994 .args(["--kernel", kernel_path.to_str().unwrap()]) 4995 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4996 .default_disks() 4997 .default_net() 4998 .args(["--api-socket", &api_socket]) 4999 .capture_output() 5000 .spawn() 5001 .unwrap(); 5002 5003 let r = std::panic::catch_unwind(|| { 5004 guest.wait_vm_boot(None).unwrap(); 5005 5006 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5007 5008 guest.enable_memory_hotplug(); 5009 5010 // Add RAM to the VM 5011 let desired_ram = 1024 << 20; 5012 resize_command(&api_socket, None, Some(desired_ram), None, None); 5013 5014 thread::sleep(std::time::Duration::new(10, 0)); 5015 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5016 5017 // Add RAM to the VM 5018 let desired_ram = 2048 << 20; 5019 resize_command(&api_socket, None, Some(desired_ram), None, None); 5020 5021 thread::sleep(std::time::Duration::new(10, 0)); 5022 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 5023 5024 // Remove RAM from the VM 5025 let desired_ram = 1024 << 20; 5026 resize_command(&api_socket, None, Some(desired_ram), None, None); 5027 5028 thread::sleep(std::time::Duration::new(10, 0)); 5029 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5030 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5031 5032 guest.reboot_linux(0, None); 5033 5034 // Check the amount of memory after reboot is 1GiB 5035 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5036 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5037 5038 // Check we can still resize to 512MiB 5039 let desired_ram = 512 << 20; 5040 resize_command(&api_socket, None, Some(desired_ram), None, None); 5041 thread::sleep(std::time::Duration::new(10, 0)); 5042 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5043 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 5044 }); 5045 5046 let _ = child.kill(); 5047 let output = child.wait_with_output().unwrap(); 5048 5049 handle_child_output(r, &output); 5050 } 5051 5052 #[test] 5053 #[cfg(target_arch = "x86_64")] 5054 #[cfg(not(feature = "mshv"))] 5055 // Test both vCPU and memory resizing together 5056 fn test_resize() { 5057 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5058 let guest = Guest::new(Box::new(focal)); 5059 let api_socket = temp_api_path(&guest.tmp_dir); 5060 5061 let kernel_path = direct_kernel_boot_path(); 5062 5063 let mut child = GuestCommand::new(&guest) 5064 .args(["--cpus", "boot=2,max=4"]) 5065 .args(["--memory", "size=512M,hotplug_size=8192M"]) 5066 .args(["--kernel", kernel_path.to_str().unwrap()]) 5067 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5068 .default_disks() 5069 .default_net() 5070 .args(["--api-socket", &api_socket]) 5071 .capture_output() 5072 .spawn() 5073 .unwrap(); 5074 5075 let r = std::panic::catch_unwind(|| { 5076 guest.wait_vm_boot(None).unwrap(); 5077 5078 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 5079 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5080 5081 guest.enable_memory_hotplug(); 5082 5083 // Resize the VM 5084 let desired_vcpus = 4; 5085 let desired_ram = 1024 << 20; 5086 resize_command( 5087 &api_socket, 5088 Some(desired_vcpus), 5089 Some(desired_ram), 5090 None, 5091 None, 5092 ); 5093 5094 guest 5095 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 5096 .unwrap(); 5097 guest 5098 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 5099 .unwrap(); 5100 thread::sleep(std::time::Duration::new(10, 0)); 5101 assert_eq!( 5102 guest.get_cpu_count().unwrap_or_default(), 5103 u32::from(desired_vcpus) 5104 ); 5105 5106 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5107 }); 5108 5109 let _ = child.kill(); 5110 let output = child.wait_with_output().unwrap(); 5111 5112 handle_child_output(r, &output); 5113 } 5114 5115 #[test] 5116 fn test_memory_overhead() { 5117 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5118 let guest = Guest::new(Box::new(focal)); 5119 5120 let kernel_path = direct_kernel_boot_path(); 5121 5122 let guest_memory_size_kb = 512 * 1024; 5123 5124 let mut child = GuestCommand::new(&guest) 5125 .args(["--cpus", "boot=1"]) 5126 .args(["--memory", format!("size={guest_memory_size_kb}K").as_str()]) 5127 .args(["--kernel", kernel_path.to_str().unwrap()]) 5128 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5129 .default_net() 5130 .default_disks() 5131 .capture_output() 5132 .spawn() 5133 .unwrap(); 5134 5135 guest.wait_vm_boot(None).unwrap(); 5136 5137 let r = std::panic::catch_unwind(|| { 5138 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 5139 eprintln!("Guest memory overhead: {overhead} vs {MAXIMUM_VMM_OVERHEAD_KB}"); 5140 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 5141 }); 5142 5143 let _ = child.kill(); 5144 let output = child.wait_with_output().unwrap(); 5145 5146 handle_child_output(r, &output); 5147 } 5148 5149 #[test] 5150 fn test_disk_hotplug() { 5151 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5152 let guest = Guest::new(Box::new(focal)); 5153 5154 #[cfg(target_arch = "x86_64")] 5155 let kernel_path = direct_kernel_boot_path(); 5156 #[cfg(target_arch = "aarch64")] 5157 let kernel_path = edk2_path(); 5158 5159 let api_socket = temp_api_path(&guest.tmp_dir); 5160 5161 let mut child = GuestCommand::new(&guest) 5162 .args(["--api-socket", &api_socket]) 5163 .args(["--cpus", "boot=1"]) 5164 .args(["--memory", "size=512M"]) 5165 .args(["--kernel", kernel_path.to_str().unwrap()]) 5166 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5167 .default_disks() 5168 .default_net() 5169 .capture_output() 5170 .spawn() 5171 .unwrap(); 5172 5173 let r = std::panic::catch_unwind(|| { 5174 guest.wait_vm_boot(None).unwrap(); 5175 5176 // Check /dev/vdc is not there 5177 assert_eq!( 5178 guest 5179 .ssh_command("lsblk | grep -c vdc.*16M || true") 5180 .unwrap() 5181 .trim() 5182 .parse::<u32>() 5183 .unwrap_or(1), 5184 0 5185 ); 5186 5187 // Now let's add the extra disk. 5188 let mut blk_file_path = dirs::home_dir().unwrap(); 5189 blk_file_path.push("workloads"); 5190 blk_file_path.push("blk.img"); 5191 let (cmd_success, cmd_output) = remote_command_w_output( 5192 &api_socket, 5193 "add-disk", 5194 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5195 ); 5196 assert!(cmd_success); 5197 assert!(String::from_utf8_lossy(&cmd_output) 5198 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5199 5200 thread::sleep(std::time::Duration::new(10, 0)); 5201 5202 // Check that /dev/vdc exists and the block size is 16M. 5203 assert_eq!( 5204 guest 5205 .ssh_command("lsblk | grep vdc | grep -c 16M") 5206 .unwrap() 5207 .trim() 5208 .parse::<u32>() 5209 .unwrap_or_default(), 5210 1 5211 ); 5212 // And check the block device can be read. 5213 guest 5214 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5215 .unwrap(); 5216 5217 // Let's remove it the extra disk. 5218 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5219 thread::sleep(std::time::Duration::new(5, 0)); 5220 // And check /dev/vdc is not there 5221 assert_eq!( 5222 guest 5223 .ssh_command("lsblk | grep -c vdc.*16M || true") 5224 .unwrap() 5225 .trim() 5226 .parse::<u32>() 5227 .unwrap_or(1), 5228 0 5229 ); 5230 5231 // And add it back to validate unplug did work correctly. 5232 let (cmd_success, cmd_output) = remote_command_w_output( 5233 &api_socket, 5234 "add-disk", 5235 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5236 ); 5237 assert!(cmd_success); 5238 assert!(String::from_utf8_lossy(&cmd_output) 5239 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5240 5241 thread::sleep(std::time::Duration::new(10, 0)); 5242 5243 // Check that /dev/vdc exists and the block size is 16M. 5244 assert_eq!( 5245 guest 5246 .ssh_command("lsblk | grep vdc | grep -c 16M") 5247 .unwrap() 5248 .trim() 5249 .parse::<u32>() 5250 .unwrap_or_default(), 5251 1 5252 ); 5253 // And check the block device can be read. 5254 guest 5255 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5256 .unwrap(); 5257 5258 // Reboot the VM. 5259 guest.reboot_linux(0, None); 5260 5261 // Check still there after reboot 5262 assert_eq!( 5263 guest 5264 .ssh_command("lsblk | grep vdc | grep -c 16M") 5265 .unwrap() 5266 .trim() 5267 .parse::<u32>() 5268 .unwrap_or_default(), 5269 1 5270 ); 5271 5272 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5273 5274 thread::sleep(std::time::Duration::new(20, 0)); 5275 5276 // Check device has gone away 5277 assert_eq!( 5278 guest 5279 .ssh_command("lsblk | grep -c vdc.*16M || true") 5280 .unwrap() 5281 .trim() 5282 .parse::<u32>() 5283 .unwrap_or(1), 5284 0 5285 ); 5286 5287 guest.reboot_linux(1, None); 5288 5289 // Check device still absent 5290 assert_eq!( 5291 guest 5292 .ssh_command("lsblk | grep -c vdc.*16M || true") 5293 .unwrap() 5294 .trim() 5295 .parse::<u32>() 5296 .unwrap_or(1), 5297 0 5298 ); 5299 }); 5300 5301 let _ = child.kill(); 5302 let output = child.wait_with_output().unwrap(); 5303 5304 handle_child_output(r, &output); 5305 } 5306 5307 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 5308 const LOOP_CONFIGURE: u64 = 0x4c0a; 5309 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 5310 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 5311 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 5312 5313 #[repr(C)] 5314 struct LoopInfo64 { 5315 lo_device: u64, 5316 lo_inode: u64, 5317 lo_rdevice: u64, 5318 lo_offset: u64, 5319 lo_sizelimit: u64, 5320 lo_number: u32, 5321 lo_encrypt_type: u32, 5322 lo_encrypt_key_size: u32, 5323 lo_flags: u32, 5324 lo_file_name: [u8; 64], 5325 lo_crypt_name: [u8; 64], 5326 lo_encrypt_key: [u8; 32], 5327 lo_init: [u64; 2], 5328 } 5329 5330 impl Default for LoopInfo64 { 5331 fn default() -> Self { 5332 LoopInfo64 { 5333 lo_device: 0, 5334 lo_inode: 0, 5335 lo_rdevice: 0, 5336 lo_offset: 0, 5337 lo_sizelimit: 0, 5338 lo_number: 0, 5339 lo_encrypt_type: 0, 5340 lo_encrypt_key_size: 0, 5341 lo_flags: 0, 5342 lo_file_name: [0; 64], 5343 lo_crypt_name: [0; 64], 5344 lo_encrypt_key: [0; 32], 5345 lo_init: [0; 2], 5346 } 5347 } 5348 } 5349 5350 #[derive(Default)] 5351 #[repr(C)] 5352 struct LoopConfig { 5353 fd: u32, 5354 block_size: u32, 5355 info: LoopInfo64, 5356 _reserved: [u64; 8], 5357 } 5358 5359 // Open loop-control device 5360 let loop_ctl_file = OpenOptions::new() 5361 .read(true) 5362 .write(true) 5363 .open(LOOP_CTL_PATH) 5364 .unwrap(); 5365 5366 // Request a free loop device 5367 let loop_device_number = 5368 unsafe { libc::ioctl(loop_ctl_file.as_raw_fd(), LOOP_CTL_GET_FREE as _) }; 5369 5370 if loop_device_number < 0 { 5371 panic!("Couldn't find a free loop device"); 5372 } 5373 5374 // Create loop device path 5375 let loop_device_path = format!("{LOOP_DEVICE_PREFIX}{loop_device_number}"); 5376 5377 // Open loop device 5378 let loop_device_file = OpenOptions::new() 5379 .read(true) 5380 .write(true) 5381 .open(&loop_device_path) 5382 .unwrap(); 5383 5384 // Open backing file 5385 let backing_file = OpenOptions::new() 5386 .read(true) 5387 .write(true) 5388 .open(backing_file_path) 5389 .unwrap(); 5390 5391 let loop_config = LoopConfig { 5392 fd: backing_file.as_raw_fd() as u32, 5393 block_size, 5394 ..Default::default() 5395 }; 5396 5397 for i in 0..num_retries { 5398 let ret = unsafe { 5399 libc::ioctl( 5400 loop_device_file.as_raw_fd(), 5401 LOOP_CONFIGURE as _, 5402 &loop_config, 5403 ) 5404 }; 5405 if ret != 0 { 5406 if i < num_retries - 1 { 5407 println!( 5408 "Iteration {}: Failed to configure the loop device {}: {}", 5409 i, 5410 loop_device_path, 5411 std::io::Error::last_os_error() 5412 ); 5413 } else { 5414 panic!( 5415 "Failed {} times trying to configure the loop device {}: {}", 5416 num_retries, 5417 loop_device_path, 5418 std::io::Error::last_os_error() 5419 ); 5420 } 5421 } else { 5422 break; 5423 } 5424 5425 // Wait for a bit before retrying 5426 thread::sleep(std::time::Duration::new(5, 0)); 5427 } 5428 5429 loop_device_path 5430 } 5431 5432 #[test] 5433 fn test_virtio_block_topology() { 5434 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5435 let guest = Guest::new(Box::new(focal)); 5436 5437 let kernel_path = direct_kernel_boot_path(); 5438 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5439 5440 let output = exec_host_command_output( 5441 format!( 5442 "qemu-img create -f raw {} 16M", 5443 test_disk_path.to_str().unwrap() 5444 ) 5445 .as_str(), 5446 ); 5447 if !output.status.success() { 5448 let stdout = String::from_utf8_lossy(&output.stdout); 5449 let stderr = String::from_utf8_lossy(&output.stderr); 5450 panic!("qemu-img command failed\nstdout\n{stdout}\nstderr\n{stderr}"); 5451 } 5452 5453 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5454 5455 let mut child = GuestCommand::new(&guest) 5456 .args(["--cpus", "boot=1"]) 5457 .args(["--memory", "size=512M"]) 5458 .args(["--kernel", kernel_path.to_str().unwrap()]) 5459 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5460 .args([ 5461 "--disk", 5462 format!( 5463 "path={}", 5464 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5465 ) 5466 .as_str(), 5467 format!( 5468 "path={}", 5469 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5470 ) 5471 .as_str(), 5472 format!("path={}", &loop_dev).as_str(), 5473 ]) 5474 .default_net() 5475 .capture_output() 5476 .spawn() 5477 .unwrap(); 5478 5479 let r = std::panic::catch_unwind(|| { 5480 guest.wait_vm_boot(None).unwrap(); 5481 5482 // MIN-IO column 5483 assert_eq!( 5484 guest 5485 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5486 .unwrap() 5487 .trim() 5488 .parse::<u32>() 5489 .unwrap_or_default(), 5490 4096 5491 ); 5492 // PHY-SEC column 5493 assert_eq!( 5494 guest 5495 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5496 .unwrap() 5497 .trim() 5498 .parse::<u32>() 5499 .unwrap_or_default(), 5500 4096 5501 ); 5502 // LOG-SEC column 5503 assert_eq!( 5504 guest 5505 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5506 .unwrap() 5507 .trim() 5508 .parse::<u32>() 5509 .unwrap_or_default(), 5510 4096 5511 ); 5512 }); 5513 5514 let _ = child.kill(); 5515 let output = child.wait_with_output().unwrap(); 5516 5517 handle_child_output(r, &output); 5518 5519 Command::new("losetup") 5520 .args(["-d", &loop_dev]) 5521 .output() 5522 .expect("loop device not found"); 5523 } 5524 5525 #[test] 5526 fn test_virtio_balloon_deflate_on_oom() { 5527 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5528 let guest = Guest::new(Box::new(focal)); 5529 5530 let kernel_path = direct_kernel_boot_path(); 5531 5532 let api_socket = temp_api_path(&guest.tmp_dir); 5533 5534 //Let's start a 4G guest with balloon occupied 2G memory 5535 let mut child = GuestCommand::new(&guest) 5536 .args(["--api-socket", &api_socket]) 5537 .args(["--cpus", "boot=1"]) 5538 .args(["--memory", "size=4G"]) 5539 .args(["--kernel", kernel_path.to_str().unwrap()]) 5540 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5541 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5542 .default_disks() 5543 .default_net() 5544 .capture_output() 5545 .spawn() 5546 .unwrap(); 5547 5548 let r = std::panic::catch_unwind(|| { 5549 guest.wait_vm_boot(None).unwrap(); 5550 5551 // Wait for balloon memory's initialization and check its size. 5552 // The virtio-balloon driver might take a few seconds to report the 5553 // balloon effective size back to the VMM. 5554 thread::sleep(std::time::Duration::new(20, 0)); 5555 5556 let orig_balloon = balloon_size(&api_socket); 5557 println!("The original balloon memory size is {orig_balloon} bytes"); 5558 assert!(orig_balloon == 2147483648); 5559 5560 // Two steps to verify if the 'deflate_on_oom' parameter works. 5561 // 1st: run a command to trigger an OOM in the guest. 5562 guest 5563 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5564 .unwrap(); 5565 5566 // Give some time for the OOM to happen in the guest and be reported 5567 // back to the host. 5568 thread::sleep(std::time::Duration::new(20, 0)); 5569 5570 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5571 let deflated_balloon = balloon_size(&api_socket); 5572 println!("After deflating, balloon memory size is {deflated_balloon} bytes"); 5573 // Verify the balloon size deflated 5574 assert!(deflated_balloon < 2147483648); 5575 }); 5576 5577 let _ = child.kill(); 5578 let output = child.wait_with_output().unwrap(); 5579 5580 handle_child_output(r, &output); 5581 } 5582 5583 #[test] 5584 #[cfg(not(feature = "mshv"))] 5585 fn test_virtio_balloon_free_page_reporting() { 5586 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5587 let guest = Guest::new(Box::new(focal)); 5588 5589 //Let's start a 4G guest with balloon occupied 2G memory 5590 let mut child = GuestCommand::new(&guest) 5591 .args(["--cpus", "boot=1"]) 5592 .args(["--memory", "size=4G"]) 5593 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5594 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5595 .args(["--balloon", "size=0,free_page_reporting=on"]) 5596 .default_disks() 5597 .default_net() 5598 .capture_output() 5599 .spawn() 5600 .unwrap(); 5601 5602 let pid = child.id(); 5603 let r = std::panic::catch_unwind(|| { 5604 guest.wait_vm_boot(None).unwrap(); 5605 5606 // Check the initial RSS is less than 1GiB 5607 let rss = process_rss_kib(pid); 5608 println!("RSS {rss} < 1048576"); 5609 assert!(rss < 1048576); 5610 5611 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5612 // seconds 5613 let guest_ip = guest.network.guest_ip.clone(); 5614 thread::spawn(move || { 5615 ssh_command_ip( 5616 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5617 &guest_ip, 5618 DEFAULT_SSH_RETRIES, 5619 DEFAULT_SSH_TIMEOUT, 5620 ) 5621 .unwrap(); 5622 }); 5623 5624 // Wait for 50 seconds to make sure the stress command is consuming 5625 // the expected amount of memory. 5626 thread::sleep(std::time::Duration::new(50, 0)); 5627 let rss = process_rss_kib(pid); 5628 println!("RSS {rss} >= 2097152"); 5629 assert!(rss >= 2097152); 5630 5631 // Wait for an extra minute to make sure the stress command has 5632 // completed and that the guest reported the free pages to the VMM 5633 // through the virtio-balloon device. We expect the RSS to be under 5634 // 2GiB. 5635 thread::sleep(std::time::Duration::new(60, 0)); 5636 let rss = process_rss_kib(pid); 5637 println!("RSS {rss} < 2097152"); 5638 assert!(rss < 2097152); 5639 }); 5640 5641 let _ = child.kill(); 5642 let output = child.wait_with_output().unwrap(); 5643 5644 handle_child_output(r, &output); 5645 } 5646 5647 #[test] 5648 fn test_pmem_hotplug() { 5649 _test_pmem_hotplug(None) 5650 } 5651 5652 #[test] 5653 fn test_pmem_multi_segment_hotplug() { 5654 _test_pmem_hotplug(Some(15)) 5655 } 5656 5657 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5658 #[cfg(target_arch = "aarch64")] 5659 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5660 #[cfg(target_arch = "x86_64")] 5661 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5662 let focal = UbuntuDiskConfig::new(focal_image); 5663 let guest = Guest::new(Box::new(focal)); 5664 5665 #[cfg(target_arch = "x86_64")] 5666 let kernel_path = direct_kernel_boot_path(); 5667 #[cfg(target_arch = "aarch64")] 5668 let kernel_path = edk2_path(); 5669 5670 let api_socket = temp_api_path(&guest.tmp_dir); 5671 5672 let mut cmd = GuestCommand::new(&guest); 5673 5674 cmd.args(["--api-socket", &api_socket]) 5675 .args(["--cpus", "boot=1"]) 5676 .args(["--memory", "size=512M"]) 5677 .args(["--kernel", kernel_path.to_str().unwrap()]) 5678 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5679 .default_disks() 5680 .default_net() 5681 .capture_output(); 5682 5683 if pci_segment.is_some() { 5684 cmd.args([ 5685 "--platform", 5686 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5687 ]); 5688 } 5689 5690 let mut child = cmd.spawn().unwrap(); 5691 5692 let r = std::panic::catch_unwind(|| { 5693 guest.wait_vm_boot(None).unwrap(); 5694 5695 // Check /dev/pmem0 is not there 5696 assert_eq!( 5697 guest 5698 .ssh_command("lsblk | grep -c pmem0 || true") 5699 .unwrap() 5700 .trim() 5701 .parse::<u32>() 5702 .unwrap_or(1), 5703 0 5704 ); 5705 5706 let pmem_temp_file = TempFile::new().unwrap(); 5707 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5708 let (cmd_success, cmd_output) = remote_command_w_output( 5709 &api_socket, 5710 "add-pmem", 5711 Some(&format!( 5712 "file={},id=test0{}", 5713 pmem_temp_file.as_path().to_str().unwrap(), 5714 if let Some(pci_segment) = pci_segment { 5715 format!(",pci_segment={pci_segment}") 5716 } else { 5717 "".to_owned() 5718 } 5719 )), 5720 ); 5721 assert!(cmd_success); 5722 if let Some(pci_segment) = pci_segment { 5723 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5724 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5725 ))); 5726 } else { 5727 assert!(String::from_utf8_lossy(&cmd_output) 5728 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5729 } 5730 5731 // Check that /dev/pmem0 exists and the block size is 128M 5732 assert_eq!( 5733 guest 5734 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5735 .unwrap() 5736 .trim() 5737 .parse::<u32>() 5738 .unwrap_or_default(), 5739 1 5740 ); 5741 5742 guest.reboot_linux(0, None); 5743 5744 // Check still there after reboot 5745 assert_eq!( 5746 guest 5747 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5748 .unwrap() 5749 .trim() 5750 .parse::<u32>() 5751 .unwrap_or_default(), 5752 1 5753 ); 5754 5755 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5756 5757 thread::sleep(std::time::Duration::new(20, 0)); 5758 5759 // Check device has gone away 5760 assert_eq!( 5761 guest 5762 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5763 .unwrap() 5764 .trim() 5765 .parse::<u32>() 5766 .unwrap_or(1), 5767 0 5768 ); 5769 5770 guest.reboot_linux(1, None); 5771 5772 // Check still absent after reboot 5773 assert_eq!( 5774 guest 5775 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5776 .unwrap() 5777 .trim() 5778 .parse::<u32>() 5779 .unwrap_or(1), 5780 0 5781 ); 5782 }); 5783 5784 let _ = child.kill(); 5785 let output = child.wait_with_output().unwrap(); 5786 5787 handle_child_output(r, &output); 5788 } 5789 5790 #[test] 5791 fn test_net_hotplug() { 5792 _test_net_hotplug(None) 5793 } 5794 5795 #[test] 5796 fn test_net_multi_segment_hotplug() { 5797 _test_net_hotplug(Some(15)) 5798 } 5799 5800 fn _test_net_hotplug(pci_segment: Option<u16>) { 5801 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5802 let guest = Guest::new(Box::new(focal)); 5803 5804 #[cfg(target_arch = "x86_64")] 5805 let kernel_path = direct_kernel_boot_path(); 5806 #[cfg(target_arch = "aarch64")] 5807 let kernel_path = edk2_path(); 5808 5809 let api_socket = temp_api_path(&guest.tmp_dir); 5810 5811 // Boot without network 5812 let mut cmd = GuestCommand::new(&guest); 5813 5814 cmd.args(["--api-socket", &api_socket]) 5815 .args(["--cpus", "boot=1"]) 5816 .args(["--memory", "size=512M"]) 5817 .args(["--kernel", kernel_path.to_str().unwrap()]) 5818 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5819 .default_disks() 5820 .capture_output(); 5821 5822 if pci_segment.is_some() { 5823 cmd.args([ 5824 "--platform", 5825 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5826 ]); 5827 } 5828 5829 let mut child = cmd.spawn().unwrap(); 5830 5831 thread::sleep(std::time::Duration::new(20, 0)); 5832 5833 let r = std::panic::catch_unwind(|| { 5834 // Add network 5835 let (cmd_success, cmd_output) = remote_command_w_output( 5836 &api_socket, 5837 "add-net", 5838 Some( 5839 format!( 5840 "{}{},id=test0", 5841 guest.default_net_string(), 5842 if let Some(pci_segment) = pci_segment { 5843 format!(",pci_segment={pci_segment}") 5844 } else { 5845 "".to_owned() 5846 } 5847 ) 5848 .as_str(), 5849 ), 5850 ); 5851 assert!(cmd_success); 5852 5853 if let Some(pci_segment) = pci_segment { 5854 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5855 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5856 ))); 5857 } else { 5858 assert!(String::from_utf8_lossy(&cmd_output) 5859 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5860 } 5861 5862 thread::sleep(std::time::Duration::new(5, 0)); 5863 5864 // 1 network interfaces + default localhost ==> 2 interfaces 5865 assert_eq!( 5866 guest 5867 .ssh_command("ip -o link | wc -l") 5868 .unwrap() 5869 .trim() 5870 .parse::<u32>() 5871 .unwrap_or_default(), 5872 2 5873 ); 5874 5875 // Remove network 5876 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5877 thread::sleep(std::time::Duration::new(5, 0)); 5878 5879 let (cmd_success, cmd_output) = remote_command_w_output( 5880 &api_socket, 5881 "add-net", 5882 Some( 5883 format!( 5884 "{}{},id=test1", 5885 guest.default_net_string(), 5886 if let Some(pci_segment) = pci_segment { 5887 format!(",pci_segment={pci_segment}") 5888 } else { 5889 "".to_owned() 5890 } 5891 ) 5892 .as_str(), 5893 ), 5894 ); 5895 assert!(cmd_success); 5896 5897 if let Some(pci_segment) = pci_segment { 5898 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5899 "{{\"id\":\"test1\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5900 ))); 5901 } else { 5902 assert!(String::from_utf8_lossy(&cmd_output) 5903 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5904 } 5905 5906 thread::sleep(std::time::Duration::new(5, 0)); 5907 5908 // 1 network interfaces + default localhost ==> 2 interfaces 5909 assert_eq!( 5910 guest 5911 .ssh_command("ip -o link | wc -l") 5912 .unwrap() 5913 .trim() 5914 .parse::<u32>() 5915 .unwrap_or_default(), 5916 2 5917 ); 5918 5919 guest.reboot_linux(0, None); 5920 5921 // Check still there after reboot 5922 // 1 network interfaces + default localhost ==> 2 interfaces 5923 assert_eq!( 5924 guest 5925 .ssh_command("ip -o link | wc -l") 5926 .unwrap() 5927 .trim() 5928 .parse::<u32>() 5929 .unwrap_or_default(), 5930 2 5931 ); 5932 }); 5933 5934 let _ = child.kill(); 5935 let output = child.wait_with_output().unwrap(); 5936 5937 handle_child_output(r, &output); 5938 } 5939 5940 #[test] 5941 fn test_initramfs() { 5942 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5943 let guest = Guest::new(Box::new(focal)); 5944 let mut workload_path = dirs::home_dir().unwrap(); 5945 workload_path.push("workloads"); 5946 5947 #[cfg(target_arch = "x86_64")] 5948 let mut kernels = vec![direct_kernel_boot_path()]; 5949 #[cfg(target_arch = "aarch64")] 5950 let kernels = [direct_kernel_boot_path()]; 5951 5952 #[cfg(target_arch = "x86_64")] 5953 { 5954 let mut pvh_kernel_path = workload_path.clone(); 5955 pvh_kernel_path.push("vmlinux"); 5956 kernels.push(pvh_kernel_path); 5957 } 5958 5959 let mut initramfs_path = workload_path; 5960 initramfs_path.push("alpine_initramfs.img"); 5961 5962 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5963 let cmdline = format!("console=hvc0 quiet TEST_STRING={test_string}"); 5964 5965 kernels.iter().for_each(|k_path| { 5966 let mut child = GuestCommand::new(&guest) 5967 .args(["--kernel", k_path.to_str().unwrap()]) 5968 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 5969 .args(["--cmdline", &cmdline]) 5970 .capture_output() 5971 .spawn() 5972 .unwrap(); 5973 5974 thread::sleep(std::time::Duration::new(20, 0)); 5975 5976 let _ = child.kill(); 5977 let output = child.wait_with_output().unwrap(); 5978 5979 let r = std::panic::catch_unwind(|| { 5980 let s = String::from_utf8_lossy(&output.stdout); 5981 5982 assert_ne!(s.lines().position(|line| line == test_string), None); 5983 }); 5984 5985 handle_child_output(r, &output); 5986 }); 5987 } 5988 5989 // One thing to note about this test. The virtio-net device is heavily used 5990 // through each ssh command. There's no need to perform a dedicated test to 5991 // verify the migration went well for virtio-net. 5992 #[test] 5993 #[cfg(not(feature = "mshv"))] 5994 fn test_snapshot_restore_hotplug_virtiomem() { 5995 _test_snapshot_restore(true); 5996 } 5997 5998 #[test] 5999 fn test_snapshot_restore_basic() { 6000 _test_snapshot_restore(false); 6001 } 6002 6003 fn _test_snapshot_restore(use_hotplug: bool) { 6004 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6005 let guest = Guest::new(Box::new(focal)); 6006 let kernel_path = direct_kernel_boot_path(); 6007 6008 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 6009 6010 let net_id = "net123"; 6011 let net_params = format!( 6012 "id={},tap=,mac={},ip={},mask=255.255.255.0", 6013 net_id, guest.network.guest_mac, guest.network.host_ip 6014 ); 6015 let mut mem_params = "size=4G"; 6016 6017 if use_hotplug { 6018 mem_params = "size=4G,hotplug_method=virtio-mem,hotplug_size=32G" 6019 } 6020 6021 let cloudinit_params = format!( 6022 "path={},iommu=on", 6023 guest.disk_config.disk(DiskType::CloudInit).unwrap() 6024 ); 6025 6026 let socket = temp_vsock_path(&guest.tmp_dir); 6027 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6028 6029 let mut child = GuestCommand::new(&guest) 6030 .args(["--api-socket", &api_socket_source]) 6031 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6032 .args(["--cpus", "boot=4"]) 6033 .args(["--memory", mem_params]) 6034 .args(["--balloon", "size=0"]) 6035 .args(["--kernel", kernel_path.to_str().unwrap()]) 6036 .args([ 6037 "--disk", 6038 format!( 6039 "path={}", 6040 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 6041 ) 6042 .as_str(), 6043 cloudinit_params.as_str(), 6044 ]) 6045 .args(["--net", net_params.as_str()]) 6046 .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) 6047 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6048 .capture_output() 6049 .spawn() 6050 .unwrap(); 6051 6052 let console_text = String::from("On a branch floating down river a cricket, singing."); 6053 // Create the snapshot directory 6054 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 6055 6056 let r = std::panic::catch_unwind(|| { 6057 guest.wait_vm_boot(None).unwrap(); 6058 6059 // Check the number of vCPUs 6060 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 6061 // Check the guest RAM 6062 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6063 if use_hotplug { 6064 // Increase guest RAM with virtio-mem 6065 resize_command( 6066 &api_socket_source, 6067 None, 6068 Some(6 << 30), 6069 None, 6070 Some(&event_path), 6071 ); 6072 thread::sleep(std::time::Duration::new(5, 0)); 6073 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6074 // Use balloon to remove RAM from the VM 6075 resize_command( 6076 &api_socket_source, 6077 None, 6078 None, 6079 Some(1 << 30), 6080 Some(&event_path), 6081 ); 6082 thread::sleep(std::time::Duration::new(5, 0)); 6083 let total_memory = guest.get_total_memory().unwrap_or_default(); 6084 assert!(total_memory > 4_800_000); 6085 assert!(total_memory < 5_760_000); 6086 } 6087 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 6088 guest.check_devices_common(Some(&socket), Some(&console_text), None); 6089 6090 // x86_64: We check that removing and adding back the virtio-net device 6091 // does not break the snapshot/restore support for virtio-pci. 6092 // This is an important thing to test as the hotplug will 6093 // trigger a PCI BAR reprogramming, which is a good way of 6094 // checking if the stored resources are correctly restored. 6095 // Unplug the virtio-net device 6096 // AArch64: Device hotplug is currently not supported, skipping here. 6097 #[cfg(target_arch = "x86_64")] 6098 { 6099 assert!(remote_command( 6100 &api_socket_source, 6101 "remove-device", 6102 Some(net_id), 6103 )); 6104 thread::sleep(std::time::Duration::new(10, 0)); 6105 let latest_events = [&MetaEvent { 6106 event: "device-removed".to_string(), 6107 device_id: Some(net_id.to_string()), 6108 }]; 6109 // See: #5938 6110 thread::sleep(std::time::Duration::new(1, 0)); 6111 assert!(check_latest_events_exact(&latest_events, &event_path)); 6112 6113 // Plug the virtio-net device again 6114 assert!(remote_command( 6115 &api_socket_source, 6116 "add-net", 6117 Some(net_params.as_str()), 6118 )); 6119 thread::sleep(std::time::Duration::new(10, 0)); 6120 } 6121 6122 // Pause the VM 6123 assert!(remote_command(&api_socket_source, "pause", None)); 6124 let latest_events = [ 6125 &MetaEvent { 6126 event: "pausing".to_string(), 6127 device_id: None, 6128 }, 6129 &MetaEvent { 6130 event: "paused".to_string(), 6131 device_id: None, 6132 }, 6133 ]; 6134 // See: #5938 6135 thread::sleep(std::time::Duration::new(1, 0)); 6136 assert!(check_latest_events_exact(&latest_events, &event_path)); 6137 6138 // Take a snapshot from the VM 6139 assert!(remote_command( 6140 &api_socket_source, 6141 "snapshot", 6142 Some(format!("file://{snapshot_dir}").as_str()), 6143 )); 6144 6145 // Wait to make sure the snapshot is completed 6146 thread::sleep(std::time::Duration::new(10, 0)); 6147 6148 let latest_events = [ 6149 &MetaEvent { 6150 event: "snapshotting".to_string(), 6151 device_id: None, 6152 }, 6153 &MetaEvent { 6154 event: "snapshotted".to_string(), 6155 device_id: None, 6156 }, 6157 ]; 6158 // See: #5938 6159 thread::sleep(std::time::Duration::new(1, 0)); 6160 assert!(check_latest_events_exact(&latest_events, &event_path)); 6161 }); 6162 6163 // Shutdown the source VM and check console output 6164 let _ = child.kill(); 6165 let output = child.wait_with_output().unwrap(); 6166 handle_child_output(r, &output); 6167 6168 let r = std::panic::catch_unwind(|| { 6169 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 6170 }); 6171 6172 handle_child_output(r, &output); 6173 6174 // Remove the vsock socket file. 6175 Command::new("rm") 6176 .arg("-f") 6177 .arg(socket.as_str()) 6178 .output() 6179 .unwrap(); 6180 6181 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 6182 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 6183 6184 // Restore the VM from the snapshot 6185 let mut child = GuestCommand::new(&guest) 6186 .args(["--api-socket", &api_socket_restored]) 6187 .args([ 6188 "--event-monitor", 6189 format!("path={event_path_restored}").as_str(), 6190 ]) 6191 .args([ 6192 "--restore", 6193 format!("source_url=file://{snapshot_dir}").as_str(), 6194 ]) 6195 .capture_output() 6196 .spawn() 6197 .unwrap(); 6198 6199 // Wait for the VM to be restored 6200 thread::sleep(std::time::Duration::new(10, 0)); 6201 let expected_events = [ 6202 &MetaEvent { 6203 event: "starting".to_string(), 6204 device_id: None, 6205 }, 6206 &MetaEvent { 6207 event: "activated".to_string(), 6208 device_id: Some("__console".to_string()), 6209 }, 6210 &MetaEvent { 6211 event: "activated".to_string(), 6212 device_id: Some("__rng".to_string()), 6213 }, 6214 &MetaEvent { 6215 event: "restoring".to_string(), 6216 device_id: None, 6217 }, 6218 ]; 6219 assert!(check_sequential_events( 6220 &expected_events, 6221 &event_path_restored 6222 )); 6223 let latest_events = [&MetaEvent { 6224 event: "restored".to_string(), 6225 device_id: None, 6226 }]; 6227 assert!(check_latest_events_exact( 6228 &latest_events, 6229 &event_path_restored 6230 )); 6231 6232 let r = std::panic::catch_unwind(|| { 6233 // Resume the VM 6234 assert!(remote_command(&api_socket_restored, "resume", None)); 6235 // There is no way that we can ensure the 'write()' to the 6236 // event file is completed when the 'resume' request is 6237 // returned successfully, because the 'write()' was done 6238 // asynchronously from a different thread of Cloud 6239 // Hypervisor (e.g. the event-monitor thread). 6240 thread::sleep(std::time::Duration::new(1, 0)); 6241 let latest_events = [ 6242 &MetaEvent { 6243 event: "resuming".to_string(), 6244 device_id: None, 6245 }, 6246 &MetaEvent { 6247 event: "resumed".to_string(), 6248 device_id: None, 6249 }, 6250 ]; 6251 assert!(check_latest_events_exact( 6252 &latest_events, 6253 &event_path_restored 6254 )); 6255 6256 // Perform same checks to validate VM has been properly restored 6257 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 6258 let total_memory = guest.get_total_memory().unwrap_or_default(); 6259 if !use_hotplug { 6260 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6261 } else { 6262 assert!(total_memory > 4_800_000); 6263 assert!(total_memory < 5_760_000); 6264 // Deflate balloon to restore entire RAM to the VM 6265 resize_command(&api_socket_restored, None, None, Some(0), None); 6266 thread::sleep(std::time::Duration::new(5, 0)); 6267 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6268 // Decrease guest RAM with virtio-mem 6269 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 6270 thread::sleep(std::time::Duration::new(5, 0)); 6271 let total_memory = guest.get_total_memory().unwrap_or_default(); 6272 assert!(total_memory > 4_800_000); 6273 assert!(total_memory < 5_760_000); 6274 } 6275 6276 guest.check_devices_common(Some(&socket), Some(&console_text), None); 6277 }); 6278 // Shutdown the target VM and check console output 6279 let _ = child.kill(); 6280 let output = child.wait_with_output().unwrap(); 6281 handle_child_output(r, &output); 6282 6283 let r = std::panic::catch_unwind(|| { 6284 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 6285 }); 6286 6287 handle_child_output(r, &output); 6288 } 6289 6290 #[test] 6291 fn test_counters() { 6292 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6293 let guest = Guest::new(Box::new(focal)); 6294 let api_socket = temp_api_path(&guest.tmp_dir); 6295 6296 let mut cmd = GuestCommand::new(&guest); 6297 cmd.args(["--cpus", "boot=1"]) 6298 .args(["--memory", "size=512M"]) 6299 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 6300 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6301 .default_disks() 6302 .args(["--net", guest.default_net_string().as_str()]) 6303 .args(["--api-socket", &api_socket]) 6304 .capture_output(); 6305 6306 let mut child = cmd.spawn().unwrap(); 6307 6308 let r = std::panic::catch_unwind(|| { 6309 guest.wait_vm_boot(None).unwrap(); 6310 6311 let orig_counters = get_counters(&api_socket); 6312 guest 6313 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 6314 .unwrap(); 6315 6316 let new_counters = get_counters(&api_socket); 6317 6318 // Check that all the counters have increased 6319 assert!(new_counters > orig_counters); 6320 }); 6321 6322 let _ = child.kill(); 6323 let output = child.wait_with_output().unwrap(); 6324 6325 handle_child_output(r, &output); 6326 } 6327 6328 #[test] 6329 #[cfg(feature = "guest_debug")] 6330 fn test_coredump() { 6331 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6332 let guest = Guest::new(Box::new(focal)); 6333 let api_socket = temp_api_path(&guest.tmp_dir); 6334 6335 let mut cmd = GuestCommand::new(&guest); 6336 cmd.args(["--cpus", "boot=4"]) 6337 .args(["--memory", "size=4G"]) 6338 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6339 .default_disks() 6340 .args(["--net", guest.default_net_string().as_str()]) 6341 .args(["--api-socket", &api_socket]) 6342 .capture_output(); 6343 6344 let mut child = cmd.spawn().unwrap(); 6345 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6346 6347 let r = std::panic::catch_unwind(|| { 6348 guest.wait_vm_boot(None).unwrap(); 6349 6350 assert!(remote_command(&api_socket, "pause", None)); 6351 6352 assert!(remote_command( 6353 &api_socket, 6354 "coredump", 6355 Some(format!("file://{vmcore_file}").as_str()), 6356 )); 6357 6358 // the num of CORE notes should equals to vcpu 6359 let readelf_core_num_cmd = 6360 format!("readelf --all {vmcore_file} |grep CORE |grep -v Type |wc -l"); 6361 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 6362 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 6363 6364 // the num of QEMU notes should equals to vcpu 6365 let readelf_vmm_num_cmd = format!("readelf --all {vmcore_file} |grep QEMU |wc -l"); 6366 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 6367 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 6368 }); 6369 6370 let _ = child.kill(); 6371 let output = child.wait_with_output().unwrap(); 6372 6373 handle_child_output(r, &output); 6374 } 6375 6376 #[test] 6377 #[cfg(feature = "guest_debug")] 6378 fn test_coredump_no_pause() { 6379 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6380 let guest = Guest::new(Box::new(focal)); 6381 let api_socket = temp_api_path(&guest.tmp_dir); 6382 6383 let mut cmd = GuestCommand::new(&guest); 6384 cmd.args(["--cpus", "boot=4"]) 6385 .args(["--memory", "size=4G"]) 6386 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6387 .default_disks() 6388 .args(["--net", guest.default_net_string().as_str()]) 6389 .args(["--api-socket", &api_socket]) 6390 .capture_output(); 6391 6392 let mut child = cmd.spawn().unwrap(); 6393 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6394 6395 let r = std::panic::catch_unwind(|| { 6396 guest.wait_vm_boot(None).unwrap(); 6397 6398 assert!(remote_command( 6399 &api_socket, 6400 "coredump", 6401 Some(format!("file://{vmcore_file}").as_str()), 6402 )); 6403 6404 assert_eq!(vm_state(&api_socket), "Running"); 6405 }); 6406 6407 let _ = child.kill(); 6408 let output = child.wait_with_output().unwrap(); 6409 6410 handle_child_output(r, &output); 6411 } 6412 6413 #[test] 6414 fn test_watchdog() { 6415 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6416 let guest = Guest::new(Box::new(focal)); 6417 let api_socket = temp_api_path(&guest.tmp_dir); 6418 6419 let kernel_path = direct_kernel_boot_path(); 6420 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6421 6422 let mut cmd = GuestCommand::new(&guest); 6423 cmd.args(["--cpus", "boot=1"]) 6424 .args(["--memory", "size=512M"]) 6425 .args(["--kernel", kernel_path.to_str().unwrap()]) 6426 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6427 .default_disks() 6428 .args(["--net", guest.default_net_string().as_str()]) 6429 .args(["--watchdog"]) 6430 .args(["--api-socket", &api_socket]) 6431 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6432 .capture_output(); 6433 6434 let mut child = cmd.spawn().unwrap(); 6435 6436 let r = std::panic::catch_unwind(|| { 6437 guest.wait_vm_boot(None).unwrap(); 6438 6439 let mut expected_reboot_count = 1; 6440 6441 // Enable the watchdog with a 15s timeout 6442 enable_guest_watchdog(&guest, 15); 6443 6444 // Reboot and check that systemd has activated the watchdog 6445 guest.ssh_command("sudo reboot").unwrap(); 6446 guest.wait_vm_boot(None).unwrap(); 6447 expected_reboot_count += 1; 6448 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6449 assert_eq!( 6450 guest 6451 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6452 .unwrap() 6453 .trim() 6454 .parse::<u32>() 6455 .unwrap_or_default(), 6456 2 6457 ); 6458 6459 // Allow some normal time to elapse to check we don't get spurious reboots 6460 thread::sleep(std::time::Duration::new(40, 0)); 6461 // Check no reboot 6462 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6463 6464 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6465 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6466 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6467 guest.wait_vm_boot(Some(50)).unwrap(); 6468 // Check a reboot is triggered by the watchdog 6469 expected_reboot_count += 1; 6470 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6471 6472 #[cfg(target_arch = "x86_64")] 6473 { 6474 // Now pause the VM and remain offline for 30s 6475 assert!(remote_command(&api_socket, "pause", None)); 6476 let latest_events = [ 6477 &MetaEvent { 6478 event: "pausing".to_string(), 6479 device_id: None, 6480 }, 6481 &MetaEvent { 6482 event: "paused".to_string(), 6483 device_id: None, 6484 }, 6485 ]; 6486 assert!(check_latest_events_exact(&latest_events, &event_path)); 6487 assert!(remote_command(&api_socket, "resume", None)); 6488 6489 // Check no reboot 6490 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6491 } 6492 }); 6493 6494 let _ = child.kill(); 6495 let output = child.wait_with_output().unwrap(); 6496 6497 handle_child_output(r, &output); 6498 } 6499 6500 #[test] 6501 fn test_pvpanic() { 6502 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 6503 let guest = Guest::new(Box::new(jammy)); 6504 let api_socket = temp_api_path(&guest.tmp_dir); 6505 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6506 6507 let kernel_path = direct_kernel_boot_path(); 6508 6509 let mut cmd = GuestCommand::new(&guest); 6510 cmd.args(["--cpus", "boot=1"]) 6511 .args(["--memory", "size=512M"]) 6512 .args(["--kernel", kernel_path.to_str().unwrap()]) 6513 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6514 .default_disks() 6515 .args(["--net", guest.default_net_string().as_str()]) 6516 .args(["--pvpanic"]) 6517 .args(["--api-socket", &api_socket]) 6518 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6519 .capture_output(); 6520 6521 let mut child = cmd.spawn().unwrap(); 6522 6523 let r = std::panic::catch_unwind(|| { 6524 guest.wait_vm_boot(None).unwrap(); 6525 6526 // Trigger guest a panic 6527 make_guest_panic(&guest); 6528 6529 // Wait a while for guest 6530 thread::sleep(std::time::Duration::new(10, 0)); 6531 6532 let expected_sequential_events = [&MetaEvent { 6533 event: "panic".to_string(), 6534 device_id: None, 6535 }]; 6536 assert!(check_latest_events_exact( 6537 &expected_sequential_events, 6538 &event_path 6539 )); 6540 }); 6541 6542 let _ = child.kill(); 6543 let output = child.wait_with_output().unwrap(); 6544 6545 handle_child_output(r, &output); 6546 } 6547 6548 #[test] 6549 fn test_tap_from_fd() { 6550 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6551 let guest = Guest::new(Box::new(focal)); 6552 let kernel_path = direct_kernel_boot_path(); 6553 6554 // Create a TAP interface with multi-queue enabled 6555 let num_queue_pairs: usize = 2; 6556 6557 use std::str::FromStr; 6558 let taps = net_util::open_tap( 6559 Some("chtap0"), 6560 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6561 None, 6562 &mut None, 6563 None, 6564 num_queue_pairs, 6565 Some(libc::O_RDWR | libc::O_NONBLOCK), 6566 ) 6567 .unwrap(); 6568 6569 let mut child = GuestCommand::new(&guest) 6570 .args(["--cpus", &format!("boot={num_queue_pairs}")]) 6571 .args(["--memory", "size=512M"]) 6572 .args(["--kernel", kernel_path.to_str().unwrap()]) 6573 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6574 .default_disks() 6575 .args([ 6576 "--net", 6577 &format!( 6578 "fd=[{},{}],mac={},num_queues={}", 6579 taps[0].as_raw_fd(), 6580 taps[1].as_raw_fd(), 6581 guest.network.guest_mac, 6582 num_queue_pairs * 2 6583 ), 6584 ]) 6585 .capture_output() 6586 .spawn() 6587 .unwrap(); 6588 6589 let r = std::panic::catch_unwind(|| { 6590 guest.wait_vm_boot(None).unwrap(); 6591 6592 assert_eq!( 6593 guest 6594 .ssh_command("ip -o link | wc -l") 6595 .unwrap() 6596 .trim() 6597 .parse::<u32>() 6598 .unwrap_or_default(), 6599 2 6600 ); 6601 6602 guest.reboot_linux(0, None); 6603 6604 assert_eq!( 6605 guest 6606 .ssh_command("ip -o link | wc -l") 6607 .unwrap() 6608 .trim() 6609 .parse::<u32>() 6610 .unwrap_or_default(), 6611 2 6612 ); 6613 }); 6614 6615 let _ = child.kill(); 6616 let output = child.wait_with_output().unwrap(); 6617 6618 handle_child_output(r, &output); 6619 } 6620 6621 // By design, a guest VM won't be able to connect to the host 6622 // machine when using a macvtap network interface (while it can 6623 // communicate externally). As a workaround, this integration 6624 // test creates two macvtap interfaces in 'bridge' mode on the 6625 // same physical net interface, one for the guest and one for 6626 // the host. With additional setup on the IP address and the 6627 // routing table, it enables the communications between the 6628 // guest VM and the host machine. 6629 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6630 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6631 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6632 let guest = Guest::new(Box::new(focal)); 6633 let api_socket = temp_api_path(&guest.tmp_dir); 6634 6635 #[cfg(target_arch = "x86_64")] 6636 let kernel_path = direct_kernel_boot_path(); 6637 #[cfg(target_arch = "aarch64")] 6638 let kernel_path = edk2_path(); 6639 6640 let phy_net = "eth0"; 6641 6642 // Create a macvtap interface for the guest VM to use 6643 assert!(exec_host_command_status(&format!( 6644 "sudo ip link add link {phy_net} name {guest_macvtap_name} type macvtap mod bridge" 6645 )) 6646 .success()); 6647 assert!(exec_host_command_status(&format!( 6648 "sudo ip link set {} address {} up", 6649 guest_macvtap_name, guest.network.guest_mac 6650 )) 6651 .success()); 6652 assert!( 6653 exec_host_command_status(&format!("sudo ip link show {guest_macvtap_name}")).success() 6654 ); 6655 6656 let tap_index = 6657 fs::read_to_string(format!("/sys/class/net/{guest_macvtap_name}/ifindex")).unwrap(); 6658 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6659 6660 assert!(exec_host_command_status(&format!("sudo chown $UID.$UID {tap_device}")).success()); 6661 6662 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6663 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6664 assert!(tap_fd1 > 0); 6665 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6666 assert!(tap_fd2 > 0); 6667 6668 // Create a macvtap on the same physical net interface for 6669 // the host machine to use 6670 assert!(exec_host_command_status(&format!( 6671 "sudo ip link add link {phy_net} name {host_macvtap_name} type macvtap mod bridge" 6672 )) 6673 .success()); 6674 // Use default mask "255.255.255.0" 6675 assert!(exec_host_command_status(&format!( 6676 "sudo ip address add {}/24 dev {}", 6677 guest.network.host_ip, host_macvtap_name 6678 )) 6679 .success()); 6680 assert!( 6681 exec_host_command_status(&format!("sudo ip link set dev {host_macvtap_name} up")) 6682 .success() 6683 ); 6684 6685 let mut guest_command = GuestCommand::new(&guest); 6686 guest_command 6687 .args(["--cpus", "boot=2"]) 6688 .args(["--memory", "size=512M"]) 6689 .args(["--kernel", kernel_path.to_str().unwrap()]) 6690 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6691 .default_disks() 6692 .args(["--api-socket", &api_socket]); 6693 6694 let net_params = format!( 6695 "fd=[{},{}],mac={},num_queues=4", 6696 tap_fd1, tap_fd2, guest.network.guest_mac 6697 ); 6698 6699 if !hotplug { 6700 guest_command.args(["--net", &net_params]); 6701 } 6702 6703 let mut child = guest_command.capture_output().spawn().unwrap(); 6704 6705 if hotplug { 6706 // Give some time to the VMM process to listen to the API 6707 // socket. This is the only requirement to avoid the following 6708 // call to ch-remote from failing. 6709 thread::sleep(std::time::Duration::new(10, 0)); 6710 // Hotplug the virtio-net device 6711 let (cmd_success, cmd_output) = 6712 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6713 assert!(cmd_success); 6714 #[cfg(target_arch = "x86_64")] 6715 assert!(String::from_utf8_lossy(&cmd_output) 6716 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6717 #[cfg(target_arch = "aarch64")] 6718 assert!(String::from_utf8_lossy(&cmd_output) 6719 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6720 } 6721 6722 // The functional connectivity provided by the virtio-net device 6723 // gets tested through wait_vm_boot() as it expects to receive a 6724 // HTTP request, and through the SSH command as well. 6725 let r = std::panic::catch_unwind(|| { 6726 guest.wait_vm_boot(None).unwrap(); 6727 6728 assert_eq!( 6729 guest 6730 .ssh_command("ip -o link | wc -l") 6731 .unwrap() 6732 .trim() 6733 .parse::<u32>() 6734 .unwrap_or_default(), 6735 2 6736 ); 6737 6738 guest.reboot_linux(0, None); 6739 6740 assert_eq!( 6741 guest 6742 .ssh_command("ip -o link | wc -l") 6743 .unwrap() 6744 .trim() 6745 .parse::<u32>() 6746 .unwrap_or_default(), 6747 2 6748 ); 6749 }); 6750 6751 let _ = child.kill(); 6752 6753 exec_host_command_status(&format!("sudo ip link del {guest_macvtap_name}")); 6754 exec_host_command_status(&format!("sudo ip link del {host_macvtap_name}")); 6755 6756 let output = child.wait_with_output().unwrap(); 6757 6758 handle_child_output(r, &output); 6759 } 6760 6761 #[test] 6762 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6763 fn test_macvtap() { 6764 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6765 } 6766 6767 #[test] 6768 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6769 fn test_macvtap_hotplug() { 6770 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6771 } 6772 6773 #[test] 6774 #[cfg(not(feature = "mshv"))] 6775 fn test_ovs_dpdk() { 6776 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6777 let guest1 = Guest::new(Box::new(focal1)); 6778 6779 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6780 let guest2 = Guest::new(Box::new(focal2)); 6781 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6782 6783 let (mut child1, mut child2) = 6784 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6785 6786 // Create the snapshot directory 6787 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6788 6789 let r = std::panic::catch_unwind(|| { 6790 // Remove one of the two ports from the OVS bridge 6791 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6792 6793 // Spawn a new netcat listener in the first VM 6794 let guest_ip = guest1.network.guest_ip.clone(); 6795 thread::spawn(move || { 6796 ssh_command_ip( 6797 "nc -l 12345", 6798 &guest_ip, 6799 DEFAULT_SSH_RETRIES, 6800 DEFAULT_SSH_TIMEOUT, 6801 ) 6802 .unwrap(); 6803 }); 6804 6805 // Wait for the server to be listening 6806 thread::sleep(std::time::Duration::new(5, 0)); 6807 6808 // Check the connection fails this time 6809 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6810 6811 // Add the OVS port back 6812 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6813 6814 // And finally check the connection is functional again 6815 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6816 6817 // Pause the VM 6818 assert!(remote_command(&api_socket_source, "pause", None)); 6819 6820 // Take a snapshot from the VM 6821 assert!(remote_command( 6822 &api_socket_source, 6823 "snapshot", 6824 Some(format!("file://{snapshot_dir}").as_str()), 6825 )); 6826 6827 // Wait to make sure the snapshot is completed 6828 thread::sleep(std::time::Duration::new(10, 0)); 6829 }); 6830 6831 // Shutdown the source VM 6832 let _ = child2.kill(); 6833 let output = child2.wait_with_output().unwrap(); 6834 handle_child_output(r, &output); 6835 6836 // Remove the vhost-user socket file. 6837 Command::new("rm") 6838 .arg("-f") 6839 .arg("/tmp/dpdkvhostclient2") 6840 .output() 6841 .unwrap(); 6842 6843 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6844 // Restore the VM from the snapshot 6845 let mut child2 = GuestCommand::new(&guest2) 6846 .args(["--api-socket", &api_socket_restored]) 6847 .args([ 6848 "--restore", 6849 format!("source_url=file://{snapshot_dir}").as_str(), 6850 ]) 6851 .capture_output() 6852 .spawn() 6853 .unwrap(); 6854 6855 // Wait for the VM to be restored 6856 thread::sleep(std::time::Duration::new(10, 0)); 6857 6858 let r = std::panic::catch_unwind(|| { 6859 // Resume the VM 6860 assert!(remote_command(&api_socket_restored, "resume", None)); 6861 6862 // Spawn a new netcat listener in the first VM 6863 let guest_ip = guest1.network.guest_ip.clone(); 6864 thread::spawn(move || { 6865 ssh_command_ip( 6866 "nc -l 12345", 6867 &guest_ip, 6868 DEFAULT_SSH_RETRIES, 6869 DEFAULT_SSH_TIMEOUT, 6870 ) 6871 .unwrap(); 6872 }); 6873 6874 // Wait for the server to be listening 6875 thread::sleep(std::time::Duration::new(5, 0)); 6876 6877 // And check the connection is still functional after restore 6878 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6879 }); 6880 6881 let _ = child1.kill(); 6882 let _ = child2.kill(); 6883 6884 let output = child1.wait_with_output().unwrap(); 6885 child2.wait().unwrap(); 6886 6887 cleanup_ovs_dpdk(); 6888 6889 handle_child_output(r, &output); 6890 } 6891 6892 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6893 cleanup_spdk_nvme(); 6894 6895 assert!(exec_host_command_status(&format!( 6896 "mkdir -p {}", 6897 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6898 )) 6899 .success()); 6900 assert!(exec_host_command_status(&format!( 6901 "truncate {} -s 128M", 6902 nvme_dir.join("test-disk.raw").to_str().unwrap() 6903 )) 6904 .success()); 6905 assert!(exec_host_command_status(&format!( 6906 "mkfs.ext4 {}", 6907 nvme_dir.join("test-disk.raw").to_str().unwrap() 6908 )) 6909 .success()); 6910 6911 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6912 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6913 .args(["-i", "0", "-m", "0x1"]) 6914 .spawn() 6915 .unwrap(); 6916 thread::sleep(std::time::Duration::new(2, 0)); 6917 6918 assert!(exec_host_command_with_retries( 6919 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER", 6920 3, 6921 std::time::Duration::new(5, 0), 6922 )); 6923 assert!(exec_host_command_status(&format!( 6924 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6925 nvme_dir.join("test-disk.raw").to_str().unwrap() 6926 )) 6927 .success()); 6928 assert!(exec_host_command_status( 6929 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6930 ) 6931 .success()); 6932 assert!(exec_host_command_status( 6933 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6934 ) 6935 .success()); 6936 assert!(exec_host_command_status(&format!( 6937 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6938 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6939 )) 6940 .success()); 6941 } 6942 6943 fn cleanup_spdk_nvme() { 6944 exec_host_command_status("pkill -f nvmf_tgt"); 6945 } 6946 6947 #[test] 6948 fn test_vfio_user() { 6949 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6950 let jammy = UbuntuDiskConfig::new(jammy_image); 6951 let guest = Guest::new(Box::new(jammy)); 6952 6953 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6954 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6955 6956 let api_socket = temp_api_path(&guest.tmp_dir); 6957 let mut child = GuestCommand::new(&guest) 6958 .args(["--api-socket", &api_socket]) 6959 .args(["--cpus", "boot=1"]) 6960 .args(["--memory", "size=512M,shared=on,hugepages=on"]) 6961 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6962 .args(["--serial", "tty", "--console", "off"]) 6963 .default_disks() 6964 .default_net() 6965 .capture_output() 6966 .spawn() 6967 .unwrap(); 6968 6969 let r = std::panic::catch_unwind(|| { 6970 guest.wait_vm_boot(None).unwrap(); 6971 6972 // Hotplug the SPDK-NVMe device to the VM 6973 let (cmd_success, cmd_output) = remote_command_w_output( 6974 &api_socket, 6975 "add-user-device", 6976 Some(&format!( 6977 "socket={},id=vfio_user0", 6978 spdk_nvme_dir 6979 .as_path() 6980 .join("nvme-vfio-user/cntrl") 6981 .to_str() 6982 .unwrap(), 6983 )), 6984 ); 6985 assert!(cmd_success); 6986 assert!(String::from_utf8_lossy(&cmd_output) 6987 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6988 6989 thread::sleep(std::time::Duration::new(10, 0)); 6990 6991 // Check both if /dev/nvme exists and if the block size is 128M. 6992 assert_eq!( 6993 guest 6994 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6995 .unwrap() 6996 .trim() 6997 .parse::<u32>() 6998 .unwrap_or_default(), 6999 1 7000 ); 7001 7002 // Check changes persist after reboot 7003 assert_eq!( 7004 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 7005 "" 7006 ); 7007 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 7008 guest 7009 .ssh_command("echo test123 | sudo tee /mnt/test") 7010 .unwrap(); 7011 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 7012 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 7013 7014 guest.reboot_linux(0, None); 7015 assert_eq!( 7016 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 7017 "" 7018 ); 7019 assert_eq!( 7020 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 7021 "test123" 7022 ); 7023 }); 7024 7025 cleanup_spdk_nvme(); 7026 7027 let _ = child.kill(); 7028 let output = child.wait_with_output().unwrap(); 7029 7030 handle_child_output(r, &output); 7031 } 7032 7033 #[test] 7034 #[cfg(target_arch = "x86_64")] 7035 fn test_vdpa_block() { 7036 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 7037 assert!(exec_host_command_status("lsmod | grep vdpa_sim_blk").success()); 7038 7039 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7040 let guest = Guest::new(Box::new(focal)); 7041 let api_socket = temp_api_path(&guest.tmp_dir); 7042 7043 let kernel_path = direct_kernel_boot_path(); 7044 7045 let mut child = GuestCommand::new(&guest) 7046 .args(["--cpus", "boot=2"]) 7047 .args(["--memory", "size=512M,hugepages=on"]) 7048 .args(["--kernel", kernel_path.to_str().unwrap()]) 7049 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7050 .default_disks() 7051 .default_net() 7052 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 7053 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 7054 .args(["--api-socket", &api_socket]) 7055 .capture_output() 7056 .spawn() 7057 .unwrap(); 7058 7059 let r = std::panic::catch_unwind(|| { 7060 guest.wait_vm_boot(None).unwrap(); 7061 7062 // Check both if /dev/vdc exists and if the block size is 128M. 7063 assert_eq!( 7064 guest 7065 .ssh_command("lsblk | grep vdc | grep -c 128M") 7066 .unwrap() 7067 .trim() 7068 .parse::<u32>() 7069 .unwrap_or_default(), 7070 1 7071 ); 7072 7073 // Check the content of the block device after we wrote to it. 7074 // The vpda-sim-blk should let us read what we previously wrote. 7075 guest 7076 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 7077 .unwrap(); 7078 assert_eq!( 7079 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 7080 "foobar" 7081 ); 7082 7083 // Hotplug an extra vDPA block device behind the vIOMMU 7084 // Add a new vDPA device to the VM 7085 let (cmd_success, cmd_output) = remote_command_w_output( 7086 &api_socket, 7087 "add-vdpa", 7088 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 7089 ); 7090 assert!(cmd_success); 7091 assert!(String::from_utf8_lossy(&cmd_output) 7092 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 7093 7094 thread::sleep(std::time::Duration::new(10, 0)); 7095 7096 // Check IOMMU setup 7097 assert!(guest 7098 .does_device_vendor_pair_match("0x1057", "0x1af4") 7099 .unwrap_or_default()); 7100 assert_eq!( 7101 guest 7102 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 7103 .unwrap() 7104 .trim(), 7105 "0001:00:01.0" 7106 ); 7107 7108 // Check both if /dev/vdd exists and if the block size is 128M. 7109 assert_eq!( 7110 guest 7111 .ssh_command("lsblk | grep vdd | grep -c 128M") 7112 .unwrap() 7113 .trim() 7114 .parse::<u32>() 7115 .unwrap_or_default(), 7116 1 7117 ); 7118 7119 // Write some content to the block device we've just plugged. 7120 guest 7121 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 7122 .unwrap(); 7123 7124 // Check we can read the content back. 7125 assert_eq!( 7126 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 7127 "foobar" 7128 ); 7129 7130 // Unplug the device 7131 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 7132 assert!(cmd_success); 7133 thread::sleep(std::time::Duration::new(10, 0)); 7134 7135 // Check /dev/vdd doesn't exist anymore 7136 assert_eq!( 7137 guest 7138 .ssh_command("lsblk | grep -c vdd || true") 7139 .unwrap() 7140 .trim() 7141 .parse::<u32>() 7142 .unwrap_or(1), 7143 0 7144 ); 7145 }); 7146 7147 let _ = child.kill(); 7148 let output = child.wait_with_output().unwrap(); 7149 7150 handle_child_output(r, &output); 7151 } 7152 7153 #[test] 7154 #[cfg(target_arch = "x86_64")] 7155 #[ignore = "See #5756"] 7156 fn test_vdpa_net() { 7157 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 7158 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 7159 return; 7160 } 7161 7162 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7163 let guest = Guest::new(Box::new(focal)); 7164 7165 let kernel_path = direct_kernel_boot_path(); 7166 7167 let mut child = GuestCommand::new(&guest) 7168 .args(["--cpus", "boot=2"]) 7169 .args(["--memory", "size=512M,hugepages=on"]) 7170 .args(["--kernel", kernel_path.to_str().unwrap()]) 7171 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7172 .default_disks() 7173 .default_net() 7174 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 7175 .capture_output() 7176 .spawn() 7177 .unwrap(); 7178 7179 let r = std::panic::catch_unwind(|| { 7180 guest.wait_vm_boot(None).unwrap(); 7181 7182 // Check we can find network interface related to vDPA device 7183 assert_eq!( 7184 guest 7185 .ssh_command("ip -o link | grep -c ens6") 7186 .unwrap() 7187 .trim() 7188 .parse::<u32>() 7189 .unwrap_or(0), 7190 1 7191 ); 7192 7193 guest 7194 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 7195 .unwrap(); 7196 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 7197 7198 // Check there is no packet yet on both TX/RX of the network interface 7199 assert_eq!( 7200 guest 7201 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 7202 .unwrap() 7203 .trim() 7204 .parse::<u32>() 7205 .unwrap_or(0), 7206 2 7207 ); 7208 7209 // Send 6 packets with ping command 7210 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 7211 7212 // Check we can find 6 packets on both TX/RX of the network interface 7213 assert_eq!( 7214 guest 7215 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 7216 .unwrap() 7217 .trim() 7218 .parse::<u32>() 7219 .unwrap_or(0), 7220 2 7221 ); 7222 7223 // No need to check for hotplug as we already tested it through 7224 // test_vdpa_block() 7225 }); 7226 7227 let _ = child.kill(); 7228 let output = child.wait_with_output().unwrap(); 7229 7230 handle_child_output(r, &output); 7231 } 7232 7233 #[test] 7234 #[cfg(target_arch = "x86_64")] 7235 fn test_tpm() { 7236 let focal = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7237 let guest = Guest::new(Box::new(focal)); 7238 7239 let (mut swtpm_command, swtpm_socket_path) = prepare_swtpm_daemon(&guest.tmp_dir); 7240 7241 let mut guest_cmd = GuestCommand::new(&guest); 7242 guest_cmd 7243 .args(["--cpus", "boot=1"]) 7244 .args(["--memory", "size=512M"]) 7245 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7246 .args(["--tpm", &format!("socket={swtpm_socket_path}")]) 7247 .capture_output() 7248 .default_disks() 7249 .default_net(); 7250 7251 // Start swtpm daemon 7252 let mut swtpm_child = swtpm_command.spawn().unwrap(); 7253 thread::sleep(std::time::Duration::new(10, 0)); 7254 let mut child = guest_cmd.spawn().unwrap(); 7255 let r = std::panic::catch_unwind(|| { 7256 guest.wait_vm_boot(None).unwrap(); 7257 assert_eq!( 7258 guest.ssh_command("ls /dev/tpm0").unwrap().trim(), 7259 "/dev/tpm0" 7260 ); 7261 guest.ssh_command("sudo tpm2_selftest -f").unwrap(); 7262 guest 7263 .ssh_command("echo 'hello' > /tmp/checksum_test; ") 7264 .unwrap(); 7265 guest.ssh_command("cmp <(sudo tpm2_pcrevent /tmp/checksum_test | grep sha256 | awk '{print $2}') <(sha256sum /tmp/checksum_test| awk '{print $1}')").unwrap(); 7266 }); 7267 7268 let _ = swtpm_child.kill(); 7269 let _d_out = swtpm_child.wait_with_output().unwrap(); 7270 7271 let _ = child.kill(); 7272 let output = child.wait_with_output().unwrap(); 7273 7274 handle_child_output(r, &output); 7275 } 7276 7277 #[test] 7278 #[cfg(target_arch = "x86_64")] 7279 fn test_double_tty() { 7280 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7281 let guest = Guest::new(Box::new(focal)); 7282 let mut cmd = GuestCommand::new(&guest); 7283 let api_socket = temp_api_path(&guest.tmp_dir); 7284 let tty_str: &str = "console=hvc0 earlyprintk=ttyS0 "; 7285 // linux printk module enable console log. 7286 let con_dis_str: &str = "console [hvc0] enabled"; 7287 // linux printk module disable console log. 7288 let con_enb_str: &str = "bootconsole [earlyser0] disabled"; 7289 7290 let kernel_path = direct_kernel_boot_path(); 7291 7292 cmd.args(["--cpus", "boot=1"]) 7293 .args(["--memory", "size=512M"]) 7294 .args(["--kernel", kernel_path.to_str().unwrap()]) 7295 .args([ 7296 "--cmdline", 7297 DIRECT_KERNEL_BOOT_CMDLINE 7298 .replace("console=hvc0 ", tty_str) 7299 .as_str(), 7300 ]) 7301 .capture_output() 7302 .default_disks() 7303 .default_net() 7304 .args(["--serial", "tty"]) 7305 .args(["--console", "tty"]) 7306 .args(["--api-socket", &api_socket]); 7307 7308 let mut child = cmd.spawn().unwrap(); 7309 7310 let mut r = std::panic::catch_unwind(|| { 7311 guest.wait_vm_boot(None).unwrap(); 7312 }); 7313 7314 let _ = child.kill(); 7315 let output = child.wait_with_output().unwrap(); 7316 7317 if r.is_ok() { 7318 r = std::panic::catch_unwind(|| { 7319 let s = String::from_utf8_lossy(&output.stdout); 7320 assert!(s.contains(tty_str)); 7321 assert!(s.contains(con_dis_str)); 7322 assert!(s.contains(con_enb_str)); 7323 }); 7324 } 7325 7326 handle_child_output(r, &output); 7327 } 7328 } 7329 7330 mod dbus_api { 7331 use crate::*; 7332 7333 // Start cloud-hypervisor with no VM parameters, running both the HTTP 7334 // and DBus APIs. Alternate calls to the external APIs (HTTP and DBus) 7335 // to create a VM, boot it, and verify that it can be shut down and then 7336 // booted again. 7337 #[test] 7338 fn test_api_dbus_and_http_interleaved() { 7339 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7340 let guest = Guest::new(Box::new(focal)); 7341 let dbus_api = TargetApi::new_dbus_api(&guest.tmp_dir); 7342 let http_api = TargetApi::new_http_api(&guest.tmp_dir); 7343 7344 let mut child = GuestCommand::new(&guest) 7345 .args(dbus_api.guest_args()) 7346 .args(http_api.guest_args()) 7347 .capture_output() 7348 .spawn() 7349 .unwrap(); 7350 7351 thread::sleep(std::time::Duration::new(1, 0)); 7352 7353 // Verify API servers are running 7354 assert!(dbus_api.remote_command("ping", None)); 7355 assert!(http_api.remote_command("ping", None)); 7356 7357 // Create the VM first 7358 let cpu_count: u8 = 4; 7359 let request_body = guest.api_create_body( 7360 cpu_count, 7361 direct_kernel_boot_path().to_str().unwrap(), 7362 DIRECT_KERNEL_BOOT_CMDLINE, 7363 ); 7364 7365 let temp_config_path = guest.tmp_dir.as_path().join("config"); 7366 std::fs::write(&temp_config_path, request_body).unwrap(); 7367 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 7368 7369 let r = std::panic::catch_unwind(|| { 7370 // Create the VM 7371 assert!(dbus_api.remote_command("create", Some(create_config),)); 7372 7373 // Then boot it 7374 assert!(http_api.remote_command("boot", None)); 7375 guest.wait_vm_boot(None).unwrap(); 7376 7377 // Check that the VM booted as expected 7378 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7379 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7380 7381 // Sync and shutdown without powering off to prevent filesystem 7382 // corruption. 7383 guest.ssh_command("sync").unwrap(); 7384 guest.ssh_command("sudo shutdown -H now").unwrap(); 7385 7386 // Wait for the guest to be fully shutdown 7387 thread::sleep(std::time::Duration::new(20, 0)); 7388 7389 // Then shutdown the VM 7390 assert!(dbus_api.remote_command("shutdown", None)); 7391 7392 // Then boot it again 7393 assert!(http_api.remote_command("boot", None)); 7394 guest.wait_vm_boot(None).unwrap(); 7395 7396 // Check that the VM booted as expected 7397 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7398 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7399 }); 7400 7401 let _ = child.kill(); 7402 let output = child.wait_with_output().unwrap(); 7403 7404 handle_child_output(r, &output); 7405 } 7406 7407 #[test] 7408 fn test_api_dbus_create_boot() { 7409 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7410 let guest = Guest::new(Box::new(focal)); 7411 7412 _test_api_create_boot(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7413 } 7414 7415 #[test] 7416 fn test_api_dbus_shutdown() { 7417 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7418 let guest = Guest::new(Box::new(focal)); 7419 7420 _test_api_shutdown(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7421 } 7422 7423 #[test] 7424 fn test_api_dbus_delete() { 7425 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7426 let guest = Guest::new(Box::new(focal)); 7427 7428 _test_api_delete(TargetApi::new_dbus_api(&guest.tmp_dir), guest); 7429 } 7430 7431 #[test] 7432 fn test_api_dbus_pause_resume() { 7433 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7434 let guest = Guest::new(Box::new(focal)); 7435 7436 _test_api_pause_resume(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7437 } 7438 } 7439 7440 mod common_sequential { 7441 #[cfg(not(feature = "mshv"))] 7442 use crate::*; 7443 7444 #[test] 7445 #[cfg(not(feature = "mshv"))] 7446 fn test_memory_mergeable_on() { 7447 test_memory_mergeable(true) 7448 } 7449 } 7450 7451 mod windows { 7452 use crate::*; 7453 use once_cell::sync::Lazy; 7454 7455 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 7456 7457 struct WindowsGuest { 7458 guest: Guest, 7459 auth: PasswordAuth, 7460 } 7461 7462 trait FsType { 7463 const FS_FAT: u8; 7464 const FS_NTFS: u8; 7465 } 7466 impl FsType for WindowsGuest { 7467 const FS_FAT: u8 = 0; 7468 const FS_NTFS: u8 = 1; 7469 } 7470 7471 impl WindowsGuest { 7472 fn new() -> Self { 7473 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 7474 let guest = Guest::new(Box::new(disk)); 7475 let auth = PasswordAuth { 7476 username: String::from("administrator"), 7477 password: String::from("Admin123"), 7478 }; 7479 7480 WindowsGuest { guest, auth } 7481 } 7482 7483 fn guest(&self) -> &Guest { 7484 &self.guest 7485 } 7486 7487 fn ssh_cmd(&self, cmd: &str) -> String { 7488 ssh_command_ip_with_auth( 7489 cmd, 7490 &self.auth, 7491 &self.guest.network.guest_ip, 7492 DEFAULT_SSH_RETRIES, 7493 DEFAULT_SSH_TIMEOUT, 7494 ) 7495 .unwrap() 7496 } 7497 7498 fn cpu_count(&self) -> u8 { 7499 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 7500 .trim() 7501 .parse::<u8>() 7502 .unwrap_or(0) 7503 } 7504 7505 fn ram_size(&self) -> usize { 7506 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 7507 .trim() 7508 .parse::<usize>() 7509 .unwrap_or(0) 7510 } 7511 7512 fn netdev_count(&self) -> u8 { 7513 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7514 .trim() 7515 .parse::<u8>() 7516 .unwrap_or(0) 7517 } 7518 7519 fn disk_count(&self) -> u8 { 7520 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7521 .trim() 7522 .parse::<u8>() 7523 .unwrap_or(0) 7524 } 7525 7526 fn reboot(&self) { 7527 let _ = self.ssh_cmd("shutdown /r /t 0"); 7528 } 7529 7530 fn shutdown(&self) { 7531 let _ = self.ssh_cmd("shutdown /s /t 0"); 7532 } 7533 7534 fn run_dnsmasq(&self) -> std::process::Child { 7535 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 7536 let dhcp_host = format!( 7537 "--dhcp-host={},{}", 7538 self.guest.network.guest_mac, self.guest.network.guest_ip 7539 ); 7540 let dhcp_range = format!( 7541 "--dhcp-range=eth,{},{}", 7542 self.guest.network.guest_ip, self.guest.network.guest_ip 7543 ); 7544 7545 Command::new("dnsmasq") 7546 .arg("--no-daemon") 7547 .arg("--log-queries") 7548 .arg(listen_address.as_str()) 7549 .arg("--except-interface=lo") 7550 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 7551 .arg("--conf-file=/dev/null") 7552 .arg(dhcp_host.as_str()) 7553 .arg(dhcp_range.as_str()) 7554 .spawn() 7555 .unwrap() 7556 } 7557 7558 // TODO Cleanup image file explicitly after test, if there's some space issues. 7559 fn disk_new(&self, fs: u8, sz: usize) -> String { 7560 let mut guard = NEXT_DISK_ID.lock().unwrap(); 7561 let id = *guard; 7562 *guard = id + 1; 7563 7564 let img = PathBuf::from(format!("/tmp/test-hotplug-{id}.raw")); 7565 let _ = fs::remove_file(&img); 7566 7567 // Create an image file 7568 let out = Command::new("qemu-img") 7569 .args([ 7570 "create", 7571 "-f", 7572 "raw", 7573 img.to_str().unwrap(), 7574 format!("{sz}m").as_str(), 7575 ]) 7576 .output() 7577 .expect("qemu-img command failed") 7578 .stdout; 7579 println!("{out:?}"); 7580 7581 // Associate image to a loop device 7582 let out = Command::new("losetup") 7583 .args(["--show", "-f", img.to_str().unwrap()]) 7584 .output() 7585 .expect("failed to create loop device") 7586 .stdout; 7587 let _tmp = String::from_utf8_lossy(&out); 7588 let loop_dev = _tmp.trim(); 7589 println!("{out:?}"); 7590 7591 // Create a partition table 7592 // echo 'type=7' | sudo sfdisk "${LOOP}" 7593 let mut child = Command::new("sfdisk") 7594 .args([loop_dev]) 7595 .stdin(Stdio::piped()) 7596 .spawn() 7597 .unwrap(); 7598 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 7599 stdin 7600 .write_all("type=7".as_bytes()) 7601 .expect("failed to write stdin"); 7602 let out = child.wait_with_output().expect("sfdisk failed").stdout; 7603 println!("{out:?}"); 7604 7605 // Disengage the loop device 7606 let out = Command::new("losetup") 7607 .args(["-d", loop_dev]) 7608 .output() 7609 .expect("loop device not found") 7610 .stdout; 7611 println!("{out:?}"); 7612 7613 // Re-associate loop device pointing to the partition only 7614 let out = Command::new("losetup") 7615 .args([ 7616 "--show", 7617 "--offset", 7618 (512 * 2048).to_string().as_str(), 7619 "-f", 7620 img.to_str().unwrap(), 7621 ]) 7622 .output() 7623 .expect("failed to create loop device") 7624 .stdout; 7625 let _tmp = String::from_utf8_lossy(&out); 7626 let loop_dev = _tmp.trim(); 7627 println!("{out:?}"); 7628 7629 // Create filesystem. 7630 let fs_cmd = match fs { 7631 WindowsGuest::FS_FAT => "mkfs.msdos", 7632 WindowsGuest::FS_NTFS => "mkfs.ntfs", 7633 _ => panic!("Unknown filesystem type '{fs}'"), 7634 }; 7635 let out = Command::new(fs_cmd) 7636 .args([&loop_dev]) 7637 .output() 7638 .unwrap_or_else(|_| panic!("{fs_cmd} failed")) 7639 .stdout; 7640 println!("{out:?}"); 7641 7642 // Disengage the loop device 7643 let out = Command::new("losetup") 7644 .args(["-d", loop_dev]) 7645 .output() 7646 .unwrap_or_else(|_| panic!("loop device '{loop_dev}' not found")) 7647 .stdout; 7648 println!("{out:?}"); 7649 7650 img.to_str().unwrap().to_string() 7651 } 7652 7653 fn disks_set_rw(&self) { 7654 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 7655 } 7656 7657 fn disks_online(&self) { 7658 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 7659 } 7660 7661 fn disk_file_put(&self, fname: &str, data: &str) { 7662 let _ = self.ssh_cmd(&format!( 7663 "powershell -Command \"'{data}' | Set-Content -Path {fname}\"" 7664 )); 7665 } 7666 7667 fn disk_file_read(&self, fname: &str) -> String { 7668 self.ssh_cmd(&format!( 7669 "powershell -Command \"Get-Content -Path {fname}\"" 7670 )) 7671 } 7672 7673 fn wait_for_boot(&self) -> bool { 7674 let cmd = "dir /b c:\\ | find \"Windows\""; 7675 let tmo_max = 180; 7676 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 7677 // interval must be small. 7678 let tmo_int = 2; 7679 let out = ssh_command_ip_with_auth( 7680 cmd, 7681 &self.auth, 7682 &self.guest.network.guest_ip, 7683 { 7684 let mut ret = 1; 7685 let mut tmo_acc = 0; 7686 loop { 7687 tmo_acc += tmo_int * ret; 7688 if tmo_acc >= tmo_max { 7689 break; 7690 } 7691 ret += 1; 7692 } 7693 ret 7694 }, 7695 tmo_int, 7696 ) 7697 .unwrap(); 7698 7699 if "Windows" == out.trim() { 7700 return true; 7701 } 7702 7703 false 7704 } 7705 } 7706 7707 fn vcpu_threads_count(pid: u32) -> u8 { 7708 // ps -T -p 12345 | grep vcpu | wc -l 7709 let out = Command::new("ps") 7710 .args(["-T", "-p", format!("{pid}").as_str()]) 7711 .output() 7712 .expect("ps command failed") 7713 .stdout; 7714 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 7715 } 7716 7717 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 7718 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 7719 let out = Command::new("ps") 7720 .args(["-T", "-p", format!("{pid}").as_str()]) 7721 .output() 7722 .expect("ps command failed") 7723 .stdout; 7724 let mut n = 0; 7725 String::from_utf8_lossy(&out) 7726 .split_whitespace() 7727 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 7728 n 7729 } 7730 7731 fn disk_ctrl_threads_count(pid: u32) -> u8 { 7732 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 7733 let out = Command::new("ps") 7734 .args(["-T", "-p", format!("{pid}").as_str()]) 7735 .output() 7736 .expect("ps command failed") 7737 .stdout; 7738 let mut n = 0; 7739 String::from_utf8_lossy(&out) 7740 .split_whitespace() 7741 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 7742 n 7743 } 7744 7745 #[test] 7746 fn test_windows_guest() { 7747 let windows_guest = WindowsGuest::new(); 7748 7749 let mut child = GuestCommand::new(windows_guest.guest()) 7750 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7751 .args(["--memory", "size=4G"]) 7752 .args(["--kernel", edk2_path().to_str().unwrap()]) 7753 .args(["--serial", "tty"]) 7754 .args(["--console", "off"]) 7755 .default_disks() 7756 .default_net() 7757 .capture_output() 7758 .spawn() 7759 .unwrap(); 7760 7761 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7762 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7763 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7764 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7765 7766 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7767 7768 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7769 7770 let r = std::panic::catch_unwind(|| { 7771 // Wait to make sure Windows boots up 7772 assert!(windows_guest.wait_for_boot()); 7773 7774 windows_guest.shutdown(); 7775 }); 7776 7777 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7778 let _ = child.kill(); 7779 let output = child.wait_with_output().unwrap(); 7780 7781 let _ = child_dnsmasq.kill(); 7782 let _ = child_dnsmasq.wait(); 7783 7784 handle_child_output(r, &output); 7785 } 7786 7787 #[test] 7788 fn test_windows_guest_multiple_queues() { 7789 let windows_guest = WindowsGuest::new(); 7790 7791 let mut ovmf_path = dirs::home_dir().unwrap(); 7792 ovmf_path.push("workloads"); 7793 ovmf_path.push(OVMF_NAME); 7794 7795 let mut child = GuestCommand::new(windows_guest.guest()) 7796 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 7797 .args(["--memory", "size=4G"]) 7798 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7799 .args(["--serial", "tty"]) 7800 .args(["--console", "off"]) 7801 .args([ 7802 "--disk", 7803 format!( 7804 "path={},num_queues=4", 7805 windows_guest 7806 .guest() 7807 .disk_config 7808 .disk(DiskType::OperatingSystem) 7809 .unwrap() 7810 ) 7811 .as_str(), 7812 ]) 7813 .args([ 7814 "--net", 7815 format!( 7816 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 7817 windows_guest.guest().network.guest_mac, 7818 windows_guest.guest().network.host_ip 7819 ) 7820 .as_str(), 7821 ]) 7822 .capture_output() 7823 .spawn() 7824 .unwrap(); 7825 7826 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7827 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7828 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7829 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7830 7831 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7832 7833 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7834 7835 let r = std::panic::catch_unwind(|| { 7836 // Wait to make sure Windows boots up 7837 assert!(windows_guest.wait_for_boot()); 7838 7839 windows_guest.shutdown(); 7840 }); 7841 7842 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7843 let _ = child.kill(); 7844 let output = child.wait_with_output().unwrap(); 7845 7846 let _ = child_dnsmasq.kill(); 7847 let _ = child_dnsmasq.wait(); 7848 7849 handle_child_output(r, &output); 7850 } 7851 7852 #[test] 7853 #[cfg(not(feature = "mshv"))] 7854 #[ignore = "See #4327"] 7855 fn test_windows_guest_snapshot_restore() { 7856 let windows_guest = WindowsGuest::new(); 7857 7858 let mut ovmf_path = dirs::home_dir().unwrap(); 7859 ovmf_path.push("workloads"); 7860 ovmf_path.push(OVMF_NAME); 7861 7862 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7863 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 7864 7865 let mut child = GuestCommand::new(windows_guest.guest()) 7866 .args(["--api-socket", &api_socket_source]) 7867 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7868 .args(["--memory", "size=4G"]) 7869 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7870 .args(["--serial", "tty"]) 7871 .args(["--console", "off"]) 7872 .default_disks() 7873 .default_net() 7874 .capture_output() 7875 .spawn() 7876 .unwrap(); 7877 7878 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7879 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7880 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7881 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7882 7883 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7884 7885 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7886 7887 // Wait to make sure Windows boots up 7888 assert!(windows_guest.wait_for_boot()); 7889 7890 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 7891 7892 // Pause the VM 7893 assert!(remote_command(&api_socket_source, "pause", None)); 7894 7895 // Take a snapshot from the VM 7896 assert!(remote_command( 7897 &api_socket_source, 7898 "snapshot", 7899 Some(format!("file://{snapshot_dir}").as_str()), 7900 )); 7901 7902 // Wait to make sure the snapshot is completed 7903 thread::sleep(std::time::Duration::new(30, 0)); 7904 7905 let _ = child.kill(); 7906 child.wait().unwrap(); 7907 7908 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 7909 7910 // Restore the VM from the snapshot 7911 let mut child = GuestCommand::new(windows_guest.guest()) 7912 .args(["--api-socket", &api_socket_restored]) 7913 .args([ 7914 "--restore", 7915 format!("source_url=file://{snapshot_dir}").as_str(), 7916 ]) 7917 .capture_output() 7918 .spawn() 7919 .unwrap(); 7920 7921 // Wait for the VM to be restored 7922 thread::sleep(std::time::Duration::new(20, 0)); 7923 7924 let r = std::panic::catch_unwind(|| { 7925 // Resume the VM 7926 assert!(remote_command(&api_socket_restored, "resume", None)); 7927 7928 windows_guest.shutdown(); 7929 }); 7930 7931 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7932 let _ = child.kill(); 7933 let output = child.wait_with_output().unwrap(); 7934 7935 let _ = child_dnsmasq.kill(); 7936 let _ = child_dnsmasq.wait(); 7937 7938 handle_child_output(r, &output); 7939 } 7940 7941 #[test] 7942 #[cfg(not(feature = "mshv"))] 7943 #[cfg(not(target_arch = "aarch64"))] 7944 fn test_windows_guest_cpu_hotplug() { 7945 let windows_guest = WindowsGuest::new(); 7946 7947 let mut ovmf_path = dirs::home_dir().unwrap(); 7948 ovmf_path.push("workloads"); 7949 ovmf_path.push(OVMF_NAME); 7950 7951 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7952 let api_socket = temp_api_path(&tmp_dir); 7953 7954 let mut child = GuestCommand::new(windows_guest.guest()) 7955 .args(["--api-socket", &api_socket]) 7956 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 7957 .args(["--memory", "size=4G"]) 7958 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7959 .args(["--serial", "tty"]) 7960 .args(["--console", "off"]) 7961 .default_disks() 7962 .default_net() 7963 .capture_output() 7964 .spawn() 7965 .unwrap(); 7966 7967 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7968 7969 let r = std::panic::catch_unwind(|| { 7970 // Wait to make sure Windows boots up 7971 assert!(windows_guest.wait_for_boot()); 7972 7973 let vcpu_num = 2; 7974 // Check the initial number of CPUs the guest sees 7975 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7976 // Check the initial number of vcpu threads in the CH process 7977 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7978 7979 let vcpu_num = 6; 7980 // Hotplug some CPUs 7981 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7982 // Wait to make sure CPUs are added 7983 thread::sleep(std::time::Duration::new(10, 0)); 7984 // Check the guest sees the correct number 7985 assert_eq!(windows_guest.cpu_count(), vcpu_num); 7986 // Check the CH process has the correct number of vcpu threads 7987 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 7988 7989 let vcpu_num = 4; 7990 // Remove some CPUs. Note that Windows doesn't support hot-remove. 7991 resize_command(&api_socket, Some(vcpu_num), None, None, None); 7992 // Wait to make sure CPUs are removed 7993 thread::sleep(std::time::Duration::new(10, 0)); 7994 // Reboot to let Windows catch up 7995 windows_guest.reboot(); 7996 // Wait to make sure Windows completely rebooted 7997 thread::sleep(std::time::Duration::new(60, 0)); 7998 // Check the guest sees the correct number 7999 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8000 // Check the CH process has the correct number of vcpu threads 8001 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8002 8003 windows_guest.shutdown(); 8004 }); 8005 8006 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8007 let _ = child.kill(); 8008 let output = child.wait_with_output().unwrap(); 8009 8010 let _ = child_dnsmasq.kill(); 8011 let _ = child_dnsmasq.wait(); 8012 8013 handle_child_output(r, &output); 8014 } 8015 8016 #[test] 8017 #[cfg(not(feature = "mshv"))] 8018 #[cfg(not(target_arch = "aarch64"))] 8019 fn test_windows_guest_ram_hotplug() { 8020 let windows_guest = WindowsGuest::new(); 8021 8022 let mut ovmf_path = dirs::home_dir().unwrap(); 8023 ovmf_path.push("workloads"); 8024 ovmf_path.push(OVMF_NAME); 8025 8026 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8027 let api_socket = temp_api_path(&tmp_dir); 8028 8029 let mut child = GuestCommand::new(windows_guest.guest()) 8030 .args(["--api-socket", &api_socket]) 8031 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8032 .args(["--memory", "size=2G,hotplug_size=5G"]) 8033 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8034 .args(["--serial", "tty"]) 8035 .args(["--console", "off"]) 8036 .default_disks() 8037 .default_net() 8038 .capture_output() 8039 .spawn() 8040 .unwrap(); 8041 8042 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8043 8044 let r = std::panic::catch_unwind(|| { 8045 // Wait to make sure Windows boots up 8046 assert!(windows_guest.wait_for_boot()); 8047 8048 let ram_size = 2 * 1024 * 1024 * 1024; 8049 // Check the initial number of RAM the guest sees 8050 let current_ram_size = windows_guest.ram_size(); 8051 // This size seems to be reserved by the system and thus the 8052 // reported amount differs by this constant value. 8053 let reserved_ram_size = ram_size - current_ram_size; 8054 // Verify that there's not more than 4mb constant diff wasted 8055 // by the reserved ram. 8056 assert!(reserved_ram_size < 4 * 1024 * 1024); 8057 8058 let ram_size = 4 * 1024 * 1024 * 1024; 8059 // Hotplug some RAM 8060 resize_command(&api_socket, None, Some(ram_size), None, None); 8061 // Wait to make sure RAM has been added 8062 thread::sleep(std::time::Duration::new(10, 0)); 8063 // Check the guest sees the correct number 8064 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8065 8066 let ram_size = 3 * 1024 * 1024 * 1024; 8067 // Unplug some RAM. Note that hot-remove most likely won't work. 8068 resize_command(&api_socket, None, Some(ram_size), None, None); 8069 // Wait to make sure RAM has been added 8070 thread::sleep(std::time::Duration::new(10, 0)); 8071 // Reboot to let Windows catch up 8072 windows_guest.reboot(); 8073 // Wait to make sure guest completely rebooted 8074 thread::sleep(std::time::Duration::new(60, 0)); 8075 // Check the guest sees the correct number 8076 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8077 8078 windows_guest.shutdown(); 8079 }); 8080 8081 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8082 let _ = child.kill(); 8083 let output = child.wait_with_output().unwrap(); 8084 8085 let _ = child_dnsmasq.kill(); 8086 let _ = child_dnsmasq.wait(); 8087 8088 handle_child_output(r, &output); 8089 } 8090 8091 #[test] 8092 #[cfg(not(feature = "mshv"))] 8093 fn test_windows_guest_netdev_hotplug() { 8094 let windows_guest = WindowsGuest::new(); 8095 8096 let mut ovmf_path = dirs::home_dir().unwrap(); 8097 ovmf_path.push("workloads"); 8098 ovmf_path.push(OVMF_NAME); 8099 8100 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8101 let api_socket = temp_api_path(&tmp_dir); 8102 8103 let mut child = GuestCommand::new(windows_guest.guest()) 8104 .args(["--api-socket", &api_socket]) 8105 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8106 .args(["--memory", "size=4G"]) 8107 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8108 .args(["--serial", "tty"]) 8109 .args(["--console", "off"]) 8110 .default_disks() 8111 .default_net() 8112 .capture_output() 8113 .spawn() 8114 .unwrap(); 8115 8116 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8117 8118 let r = std::panic::catch_unwind(|| { 8119 // Wait to make sure Windows boots up 8120 assert!(windows_guest.wait_for_boot()); 8121 8122 // Initially present network device 8123 let netdev_num = 1; 8124 assert_eq!(windows_guest.netdev_count(), netdev_num); 8125 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8126 8127 // Hotplug network device 8128 let (cmd_success, cmd_output) = remote_command_w_output( 8129 &api_socket, 8130 "add-net", 8131 Some(windows_guest.guest().default_net_string().as_str()), 8132 ); 8133 assert!(cmd_success); 8134 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 8135 thread::sleep(std::time::Duration::new(5, 0)); 8136 // Verify the device is on the system 8137 let netdev_num = 2; 8138 assert_eq!(windows_guest.netdev_count(), netdev_num); 8139 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8140 8141 // Remove network device 8142 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 8143 assert!(cmd_success); 8144 thread::sleep(std::time::Duration::new(5, 0)); 8145 // Verify the device has been removed 8146 let netdev_num = 1; 8147 assert_eq!(windows_guest.netdev_count(), netdev_num); 8148 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8149 8150 windows_guest.shutdown(); 8151 }); 8152 8153 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8154 let _ = child.kill(); 8155 let output = child.wait_with_output().unwrap(); 8156 8157 let _ = child_dnsmasq.kill(); 8158 let _ = child_dnsmasq.wait(); 8159 8160 handle_child_output(r, &output); 8161 } 8162 8163 #[test] 8164 #[ignore = "See #6037"] 8165 #[cfg(not(feature = "mshv"))] 8166 #[cfg(not(target_arch = "aarch64"))] 8167 fn test_windows_guest_disk_hotplug() { 8168 let windows_guest = WindowsGuest::new(); 8169 8170 let mut ovmf_path = dirs::home_dir().unwrap(); 8171 ovmf_path.push("workloads"); 8172 ovmf_path.push(OVMF_NAME); 8173 8174 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8175 let api_socket = temp_api_path(&tmp_dir); 8176 8177 let mut child = GuestCommand::new(windows_guest.guest()) 8178 .args(["--api-socket", &api_socket]) 8179 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8180 .args(["--memory", "size=4G"]) 8181 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8182 .args(["--serial", "tty"]) 8183 .args(["--console", "off"]) 8184 .default_disks() 8185 .default_net() 8186 .capture_output() 8187 .spawn() 8188 .unwrap(); 8189 8190 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8191 8192 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 8193 8194 let r = std::panic::catch_unwind(|| { 8195 // Wait to make sure Windows boots up 8196 assert!(windows_guest.wait_for_boot()); 8197 8198 // Initially present disk device 8199 let disk_num = 1; 8200 assert_eq!(windows_guest.disk_count(), disk_num); 8201 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8202 8203 // Hotplug disk device 8204 let (cmd_success, cmd_output) = remote_command_w_output( 8205 &api_socket, 8206 "add-disk", 8207 Some(format!("path={disk},readonly=off").as_str()), 8208 ); 8209 assert!(cmd_success); 8210 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 8211 thread::sleep(std::time::Duration::new(5, 0)); 8212 // Online disk device 8213 windows_guest.disks_set_rw(); 8214 windows_guest.disks_online(); 8215 // Verify the device is on the system 8216 let disk_num = 2; 8217 assert_eq!(windows_guest.disk_count(), disk_num); 8218 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8219 8220 let data = "hello"; 8221 let fname = "d:\\world"; 8222 windows_guest.disk_file_put(fname, data); 8223 8224 // Unmount disk device 8225 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 8226 assert!(cmd_success); 8227 thread::sleep(std::time::Duration::new(5, 0)); 8228 // Verify the device has been removed 8229 let disk_num = 1; 8230 assert_eq!(windows_guest.disk_count(), disk_num); 8231 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8232 8233 // Remount and check the file exists with the expected contents 8234 let (cmd_success, _cmd_output) = remote_command_w_output( 8235 &api_socket, 8236 "add-disk", 8237 Some(format!("path={disk},readonly=off").as_str()), 8238 ); 8239 assert!(cmd_success); 8240 thread::sleep(std::time::Duration::new(5, 0)); 8241 let out = windows_guest.disk_file_read(fname); 8242 assert_eq!(data, out.trim()); 8243 8244 // Intentionally no unmount, it'll happen at shutdown. 8245 8246 windows_guest.shutdown(); 8247 }); 8248 8249 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8250 let _ = child.kill(); 8251 let output = child.wait_with_output().unwrap(); 8252 8253 let _ = child_dnsmasq.kill(); 8254 let _ = child_dnsmasq.wait(); 8255 8256 handle_child_output(r, &output); 8257 } 8258 8259 #[test] 8260 #[ignore = "See #6037"] 8261 #[cfg(not(feature = "mshv"))] 8262 #[cfg(not(target_arch = "aarch64"))] 8263 fn test_windows_guest_disk_hotplug_multi() { 8264 let windows_guest = WindowsGuest::new(); 8265 8266 let mut ovmf_path = dirs::home_dir().unwrap(); 8267 ovmf_path.push("workloads"); 8268 ovmf_path.push(OVMF_NAME); 8269 8270 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8271 let api_socket = temp_api_path(&tmp_dir); 8272 8273 let mut child = GuestCommand::new(windows_guest.guest()) 8274 .args(["--api-socket", &api_socket]) 8275 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8276 .args(["--memory", "size=2G"]) 8277 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8278 .args(["--serial", "tty"]) 8279 .args(["--console", "off"]) 8280 .default_disks() 8281 .default_net() 8282 .capture_output() 8283 .spawn() 8284 .unwrap(); 8285 8286 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8287 8288 // Predefined data to used at various test stages 8289 let disk_test_data: [[String; 4]; 2] = [ 8290 [ 8291 "_disk2".to_string(), 8292 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 8293 "d:\\world".to_string(), 8294 "hello".to_string(), 8295 ], 8296 [ 8297 "_disk3".to_string(), 8298 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 8299 "e:\\hello".to_string(), 8300 "world".to_string(), 8301 ], 8302 ]; 8303 8304 let r = std::panic::catch_unwind(|| { 8305 // Wait to make sure Windows boots up 8306 assert!(windows_guest.wait_for_boot()); 8307 8308 // Initially present disk device 8309 let disk_num = 1; 8310 assert_eq!(windows_guest.disk_count(), disk_num); 8311 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8312 8313 for it in &disk_test_data { 8314 let disk_id = it[0].as_str(); 8315 let disk = it[1].as_str(); 8316 // Hotplug disk device 8317 let (cmd_success, cmd_output) = remote_command_w_output( 8318 &api_socket, 8319 "add-disk", 8320 Some(format!("path={disk},readonly=off").as_str()), 8321 ); 8322 assert!(cmd_success); 8323 assert!(String::from_utf8_lossy(&cmd_output) 8324 .contains(format!("\"id\":\"{disk_id}\"").as_str())); 8325 thread::sleep(std::time::Duration::new(5, 0)); 8326 // Online disk devices 8327 windows_guest.disks_set_rw(); 8328 windows_guest.disks_online(); 8329 } 8330 // Verify the devices are on the system 8331 let disk_num = (disk_test_data.len() + 1) as u8; 8332 assert_eq!(windows_guest.disk_count(), disk_num); 8333 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8334 8335 // Put test data 8336 for it in &disk_test_data { 8337 let fname = it[2].as_str(); 8338 let data = it[3].as_str(); 8339 windows_guest.disk_file_put(fname, data); 8340 } 8341 8342 // Unmount disk devices 8343 for it in &disk_test_data { 8344 let disk_id = it[0].as_str(); 8345 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 8346 assert!(cmd_success); 8347 thread::sleep(std::time::Duration::new(5, 0)); 8348 } 8349 8350 // Verify the devices have been removed 8351 let disk_num = 1; 8352 assert_eq!(windows_guest.disk_count(), disk_num); 8353 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8354 8355 // Remount 8356 for it in &disk_test_data { 8357 let disk = it[1].as_str(); 8358 let (cmd_success, _cmd_output) = remote_command_w_output( 8359 &api_socket, 8360 "add-disk", 8361 Some(format!("path={disk},readonly=off").as_str()), 8362 ); 8363 assert!(cmd_success); 8364 thread::sleep(std::time::Duration::new(5, 0)); 8365 } 8366 8367 // Check the files exists with the expected contents 8368 for it in &disk_test_data { 8369 let fname = it[2].as_str(); 8370 let data = it[3].as_str(); 8371 let out = windows_guest.disk_file_read(fname); 8372 assert_eq!(data, out.trim()); 8373 } 8374 8375 // Intentionally no unmount, it'll happen at shutdown. 8376 8377 windows_guest.shutdown(); 8378 }); 8379 8380 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8381 let _ = child.kill(); 8382 let output = child.wait_with_output().unwrap(); 8383 8384 let _ = child_dnsmasq.kill(); 8385 let _ = child_dnsmasq.wait(); 8386 8387 handle_child_output(r, &output); 8388 } 8389 8390 #[test] 8391 #[cfg(not(feature = "mshv"))] 8392 #[cfg(not(target_arch = "aarch64"))] 8393 fn test_windows_guest_netdev_multi() { 8394 let windows_guest = WindowsGuest::new(); 8395 8396 let mut ovmf_path = dirs::home_dir().unwrap(); 8397 ovmf_path.push("workloads"); 8398 ovmf_path.push(OVMF_NAME); 8399 8400 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8401 let api_socket = temp_api_path(&tmp_dir); 8402 8403 let mut child = GuestCommand::new(windows_guest.guest()) 8404 .args(["--api-socket", &api_socket]) 8405 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8406 .args(["--memory", "size=4G"]) 8407 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8408 .args(["--serial", "tty"]) 8409 .args(["--console", "off"]) 8410 .default_disks() 8411 // The multi net dev config is borrowed from test_multiple_network_interfaces 8412 .args([ 8413 "--net", 8414 windows_guest.guest().default_net_string().as_str(), 8415 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 8416 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 8417 ]) 8418 .capture_output() 8419 .spawn() 8420 .unwrap(); 8421 8422 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8423 8424 let r = std::panic::catch_unwind(|| { 8425 // Wait to make sure Windows boots up 8426 assert!(windows_guest.wait_for_boot()); 8427 8428 let netdev_num = 3; 8429 assert_eq!(windows_guest.netdev_count(), netdev_num); 8430 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8431 8432 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 8433 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 8434 8435 windows_guest.shutdown(); 8436 }); 8437 8438 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8439 let _ = child.kill(); 8440 let output = child.wait_with_output().unwrap(); 8441 8442 let _ = child_dnsmasq.kill(); 8443 let _ = child_dnsmasq.wait(); 8444 8445 handle_child_output(r, &output); 8446 } 8447 } 8448 8449 #[cfg(target_arch = "x86_64")] 8450 mod sgx { 8451 use crate::*; 8452 8453 #[test] 8454 fn test_sgx() { 8455 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 8456 let jammy = UbuntuDiskConfig::new(jammy_image); 8457 let guest = Guest::new(Box::new(jammy)); 8458 8459 let mut child = GuestCommand::new(&guest) 8460 .args(["--cpus", "boot=1"]) 8461 .args(["--memory", "size=512M"]) 8462 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8463 .default_disks() 8464 .default_net() 8465 .args(["--sgx-epc", "id=epc0,size=64M"]) 8466 .capture_output() 8467 .spawn() 8468 .unwrap(); 8469 8470 let r = std::panic::catch_unwind(|| { 8471 guest.wait_vm_boot(None).unwrap(); 8472 8473 // Check if SGX is correctly detected in the guest. 8474 guest.check_sgx_support().unwrap(); 8475 8476 // Validate the SGX EPC section is 64MiB. 8477 assert_eq!( 8478 guest 8479 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 8480 .unwrap() 8481 .trim(), 8482 "0x0000000004000000" 8483 ); 8484 }); 8485 8486 let _ = child.kill(); 8487 let output = child.wait_with_output().unwrap(); 8488 8489 handle_child_output(r, &output); 8490 } 8491 } 8492 8493 #[cfg(target_arch = "x86_64")] 8494 mod vfio { 8495 use crate::*; 8496 8497 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 8498 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8499 let guest = Guest::new(Box::new(jammy)); 8500 let api_socket = temp_api_path(&guest.tmp_dir); 8501 8502 let mut child = GuestCommand::new(&guest) 8503 .args(["--cpus", "boot=4"]) 8504 .args([ 8505 "--memory", 8506 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(), 8507 ]) 8508 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8509 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8510 .args(["--api-socket", &api_socket]) 8511 .default_disks() 8512 .default_net() 8513 .capture_output() 8514 .spawn() 8515 .unwrap(); 8516 8517 let r = std::panic::catch_unwind(|| { 8518 guest.wait_vm_boot(None).unwrap(); 8519 8520 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8521 8522 guest.enable_memory_hotplug(); 8523 8524 // Add RAM to the VM 8525 let desired_ram = 6 << 30; 8526 resize_command(&api_socket, None, Some(desired_ram), None, None); 8527 thread::sleep(std::time::Duration::new(30, 0)); 8528 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8529 8530 // Check the VFIO device works when RAM is increased to 6GiB 8531 guest.check_nvidia_gpu(); 8532 }); 8533 8534 let _ = child.kill(); 8535 let output = child.wait_with_output().unwrap(); 8536 8537 handle_child_output(r, &output); 8538 } 8539 8540 #[test] 8541 fn test_nvidia_card_memory_hotplug_acpi() { 8542 test_nvidia_card_memory_hotplug("acpi") 8543 } 8544 8545 #[test] 8546 fn test_nvidia_card_memory_hotplug_virtio_mem() { 8547 test_nvidia_card_memory_hotplug("virtio-mem") 8548 } 8549 8550 #[test] 8551 fn test_nvidia_card_pci_hotplug() { 8552 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8553 let guest = Guest::new(Box::new(jammy)); 8554 let api_socket = temp_api_path(&guest.tmp_dir); 8555 8556 let mut child = GuestCommand::new(&guest) 8557 .args(["--cpus", "boot=4"]) 8558 .args(["--memory", "size=4G"]) 8559 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8560 .args(["--api-socket", &api_socket]) 8561 .default_disks() 8562 .default_net() 8563 .capture_output() 8564 .spawn() 8565 .unwrap(); 8566 8567 let r = std::panic::catch_unwind(|| { 8568 guest.wait_vm_boot(None).unwrap(); 8569 8570 // Hotplug the card to the VM 8571 let (cmd_success, cmd_output) = remote_command_w_output( 8572 &api_socket, 8573 "add-device", 8574 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 8575 ); 8576 assert!(cmd_success); 8577 assert!(String::from_utf8_lossy(&cmd_output) 8578 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 8579 8580 thread::sleep(std::time::Duration::new(10, 0)); 8581 8582 // Check the VFIO device works after hotplug 8583 guest.check_nvidia_gpu(); 8584 }); 8585 8586 let _ = child.kill(); 8587 let output = child.wait_with_output().unwrap(); 8588 8589 handle_child_output(r, &output); 8590 } 8591 8592 #[test] 8593 fn test_nvidia_card_reboot() { 8594 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8595 let guest = Guest::new(Box::new(jammy)); 8596 let api_socket = temp_api_path(&guest.tmp_dir); 8597 8598 let mut child = GuestCommand::new(&guest) 8599 .args(["--cpus", "boot=4"]) 8600 .args(["--memory", "size=4G"]) 8601 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8602 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8603 .args(["--api-socket", &api_socket]) 8604 .default_disks() 8605 .default_net() 8606 .capture_output() 8607 .spawn() 8608 .unwrap(); 8609 8610 let r = std::panic::catch_unwind(|| { 8611 guest.wait_vm_boot(None).unwrap(); 8612 8613 // Check the VFIO device works after boot 8614 guest.check_nvidia_gpu(); 8615 8616 guest.reboot_linux(0, None); 8617 8618 // Check the VFIO device works after reboot 8619 guest.check_nvidia_gpu(); 8620 }); 8621 8622 let _ = child.kill(); 8623 let output = child.wait_with_output().unwrap(); 8624 8625 handle_child_output(r, &output); 8626 } 8627 } 8628 8629 mod live_migration { 8630 use crate::*; 8631 8632 fn start_live_migration( 8633 migration_socket: &str, 8634 src_api_socket: &str, 8635 dest_api_socket: &str, 8636 local: bool, 8637 ) -> bool { 8638 // Start to receive migration from the destintion VM 8639 let mut receive_migration = Command::new(clh_command("ch-remote")) 8640 .args([ 8641 &format!("--api-socket={dest_api_socket}"), 8642 "receive-migration", 8643 &format! {"unix:{migration_socket}"}, 8644 ]) 8645 .stderr(Stdio::piped()) 8646 .stdout(Stdio::piped()) 8647 .spawn() 8648 .unwrap(); 8649 // Give it '1s' to make sure the 'migration_socket' file is properly created 8650 thread::sleep(std::time::Duration::new(1, 0)); 8651 // Start to send migration from the source VM 8652 8653 let mut args = [ 8654 format!("--api-socket={}", &src_api_socket), 8655 "send-migration".to_string(), 8656 format! {"unix:{migration_socket}"}, 8657 ] 8658 .to_vec(); 8659 8660 if local { 8661 args.insert(2, "--local".to_string()); 8662 } 8663 8664 let mut send_migration = Command::new(clh_command("ch-remote")) 8665 .args(&args) 8666 .stderr(Stdio::piped()) 8667 .stdout(Stdio::piped()) 8668 .spawn() 8669 .unwrap(); 8670 8671 // The 'send-migration' command should be executed successfully within the given timeout 8672 let send_success = if let Some(status) = send_migration 8673 .wait_timeout(std::time::Duration::from_secs(30)) 8674 .unwrap() 8675 { 8676 status.success() 8677 } else { 8678 false 8679 }; 8680 8681 if !send_success { 8682 let _ = send_migration.kill(); 8683 let output = send_migration.wait_with_output().unwrap(); 8684 eprintln!( 8685 "\n\n==== Start 'send_migration' output ==== \ 8686 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8687 \n\n==== End 'send_migration' output ====\n\n", 8688 String::from_utf8_lossy(&output.stdout), 8689 String::from_utf8_lossy(&output.stderr) 8690 ); 8691 } 8692 8693 // The 'receive-migration' command should be executed successfully within the given timeout 8694 let receive_success = if let Some(status) = receive_migration 8695 .wait_timeout(std::time::Duration::from_secs(30)) 8696 .unwrap() 8697 { 8698 status.success() 8699 } else { 8700 false 8701 }; 8702 8703 if !receive_success { 8704 let _ = receive_migration.kill(); 8705 let output = receive_migration.wait_with_output().unwrap(); 8706 eprintln!( 8707 "\n\n==== Start 'receive_migration' output ==== \ 8708 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8709 \n\n==== End 'receive_migration' output ====\n\n", 8710 String::from_utf8_lossy(&output.stdout), 8711 String::from_utf8_lossy(&output.stderr) 8712 ); 8713 } 8714 8715 send_success && receive_success 8716 } 8717 8718 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 8719 let mut src_vm = src_vm; 8720 let mut dest_vm = dest_vm; 8721 8722 let _ = src_vm.kill(); 8723 let src_output = src_vm.wait_with_output().unwrap(); 8724 eprintln!( 8725 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8726 String::from_utf8_lossy(&src_output.stdout) 8727 ); 8728 eprintln!( 8729 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8730 String::from_utf8_lossy(&src_output.stderr) 8731 ); 8732 let _ = dest_vm.kill(); 8733 let dest_output = dest_vm.wait_with_output().unwrap(); 8734 eprintln!( 8735 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8736 String::from_utf8_lossy(&dest_output.stdout) 8737 ); 8738 eprintln!( 8739 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8740 String::from_utf8_lossy(&dest_output.stderr) 8741 ); 8742 8743 if let Some(ovs_vm) = ovs_vm { 8744 let mut ovs_vm = ovs_vm; 8745 let _ = ovs_vm.kill(); 8746 let ovs_output = ovs_vm.wait_with_output().unwrap(); 8747 eprintln!( 8748 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 8749 String::from_utf8_lossy(&ovs_output.stdout) 8750 ); 8751 eprintln!( 8752 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 8753 String::from_utf8_lossy(&ovs_output.stderr) 8754 ); 8755 8756 cleanup_ovs_dpdk(); 8757 } 8758 8759 panic!("Test failed: {message}") 8760 } 8761 8762 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 8763 // same host. It ensures the following behaviors: 8764 // 1. The source VM is up and functional (including various virtio-devices are working properly); 8765 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 8766 // 3. The source VM terminated gracefully after live migration; 8767 // 4. The destination VM is functional (including various virtio-devices are working properly) after 8768 // live migration; 8769 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 8770 fn _test_live_migration(upgrade_test: bool, local: bool) { 8771 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8772 let guest = Guest::new(Box::new(focal)); 8773 let kernel_path = direct_kernel_boot_path(); 8774 let console_text = String::from("On a branch floating down river a cricket, singing."); 8775 let net_id = "net123"; 8776 let net_params = format!( 8777 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8778 net_id, guest.network.guest_mac, guest.network.host_ip 8779 ); 8780 8781 let memory_param: &[&str] = if local { 8782 &["--memory", "size=4G,shared=on"] 8783 } else { 8784 &["--memory", "size=4G"] 8785 }; 8786 8787 let boot_vcpus = 2; 8788 let max_vcpus = 4; 8789 8790 let pmem_temp_file = TempFile::new().unwrap(); 8791 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8792 std::process::Command::new("mkfs.ext4") 8793 .arg(pmem_temp_file.as_path()) 8794 .output() 8795 .expect("Expect creating disk image to succeed"); 8796 let pmem_path = String::from("/dev/pmem0"); 8797 8798 // Start the source VM 8799 let src_vm_path = if !upgrade_test { 8800 clh_command("cloud-hypervisor") 8801 } else { 8802 cloud_hypervisor_release_path() 8803 }; 8804 let src_api_socket = temp_api_path(&guest.tmp_dir); 8805 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8806 src_vm_cmd 8807 .args([ 8808 "--cpus", 8809 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8810 ]) 8811 .args(memory_param) 8812 .args(["--kernel", kernel_path.to_str().unwrap()]) 8813 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8814 .default_disks() 8815 .args(["--net", net_params.as_str()]) 8816 .args(["--api-socket", &src_api_socket]) 8817 .args([ 8818 "--pmem", 8819 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8820 ]); 8821 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8822 8823 // Start the destination VM 8824 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8825 dest_api_socket.push_str(".dest"); 8826 let mut dest_child = GuestCommand::new(&guest) 8827 .args(["--api-socket", &dest_api_socket]) 8828 .capture_output() 8829 .spawn() 8830 .unwrap(); 8831 8832 let r = std::panic::catch_unwind(|| { 8833 guest.wait_vm_boot(None).unwrap(); 8834 8835 // Make sure the source VM is functaionl 8836 // Check the number of vCPUs 8837 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8838 8839 // Check the guest RAM 8840 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8841 8842 // Check the guest virtio-devices, e.g. block, rng, console, and net 8843 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8844 8845 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8846 // to make sure that removing and adding back the virtio-net device does 8847 // not break the live-migration support for virtio-pci. 8848 #[cfg(target_arch = "x86_64")] 8849 { 8850 assert!(remote_command( 8851 &src_api_socket, 8852 "remove-device", 8853 Some(net_id), 8854 )); 8855 thread::sleep(std::time::Duration::new(10, 0)); 8856 8857 // Plug the virtio-net device again 8858 assert!(remote_command( 8859 &src_api_socket, 8860 "add-net", 8861 Some(net_params.as_str()), 8862 )); 8863 thread::sleep(std::time::Duration::new(10, 0)); 8864 } 8865 8866 // Start the live-migration 8867 let migration_socket = String::from( 8868 guest 8869 .tmp_dir 8870 .as_path() 8871 .join("live-migration.sock") 8872 .to_str() 8873 .unwrap(), 8874 ); 8875 8876 assert!( 8877 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8878 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8879 ); 8880 }); 8881 8882 // Check and report any errors occurred during the live-migration 8883 if r.is_err() { 8884 print_and_panic( 8885 src_child, 8886 dest_child, 8887 None, 8888 "Error occurred during live-migration", 8889 ); 8890 } 8891 8892 // Check the source vm has been terminated successful (give it '3s' to settle) 8893 thread::sleep(std::time::Duration::new(3, 0)); 8894 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8895 print_and_panic( 8896 src_child, 8897 dest_child, 8898 None, 8899 "source VM was not terminated successfully.", 8900 ); 8901 }; 8902 8903 // Post live-migration check to make sure the destination VM is funcational 8904 let r = std::panic::catch_unwind(|| { 8905 // Perform same checks to validate VM has been properly migrated 8906 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8907 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8908 8909 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8910 }); 8911 8912 // Clean-up the destination VM and make sure it terminated correctly 8913 let _ = dest_child.kill(); 8914 let dest_output = dest_child.wait_with_output().unwrap(); 8915 handle_child_output(r, &dest_output); 8916 8917 // Check the destination VM has the expected 'concole_text' from its output 8918 let r = std::panic::catch_unwind(|| { 8919 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8920 }); 8921 handle_child_output(r, &dest_output); 8922 } 8923 8924 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 8925 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8926 let guest = Guest::new(Box::new(focal)); 8927 let kernel_path = direct_kernel_boot_path(); 8928 let console_text = String::from("On a branch floating down river a cricket, singing."); 8929 let net_id = "net123"; 8930 let net_params = format!( 8931 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8932 net_id, guest.network.guest_mac, guest.network.host_ip 8933 ); 8934 8935 let memory_param: &[&str] = if local { 8936 &[ 8937 "--memory", 8938 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 8939 "--balloon", 8940 "size=0", 8941 ] 8942 } else { 8943 &[ 8944 "--memory", 8945 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 8946 "--balloon", 8947 "size=0", 8948 ] 8949 }; 8950 8951 let boot_vcpus = 2; 8952 let max_vcpus = 4; 8953 8954 let pmem_temp_file = TempFile::new().unwrap(); 8955 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8956 std::process::Command::new("mkfs.ext4") 8957 .arg(pmem_temp_file.as_path()) 8958 .output() 8959 .expect("Expect creating disk image to succeed"); 8960 let pmem_path = String::from("/dev/pmem0"); 8961 8962 // Start the source VM 8963 let src_vm_path = if !upgrade_test { 8964 clh_command("cloud-hypervisor") 8965 } else { 8966 cloud_hypervisor_release_path() 8967 }; 8968 let src_api_socket = temp_api_path(&guest.tmp_dir); 8969 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8970 src_vm_cmd 8971 .args([ 8972 "--cpus", 8973 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8974 ]) 8975 .args(memory_param) 8976 .args(["--kernel", kernel_path.to_str().unwrap()]) 8977 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8978 .default_disks() 8979 .args(["--net", net_params.as_str()]) 8980 .args(["--api-socket", &src_api_socket]) 8981 .args([ 8982 "--pmem", 8983 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8984 ]); 8985 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8986 8987 // Start the destination VM 8988 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8989 dest_api_socket.push_str(".dest"); 8990 let mut dest_child = GuestCommand::new(&guest) 8991 .args(["--api-socket", &dest_api_socket]) 8992 .capture_output() 8993 .spawn() 8994 .unwrap(); 8995 8996 let r = std::panic::catch_unwind(|| { 8997 guest.wait_vm_boot(None).unwrap(); 8998 8999 // Make sure the source VM is functaionl 9000 // Check the number of vCPUs 9001 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9002 9003 // Check the guest RAM 9004 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9005 // Increase the guest RAM 9006 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 9007 thread::sleep(std::time::Duration::new(5, 0)); 9008 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9009 // Use balloon to remove RAM from the VM 9010 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 9011 thread::sleep(std::time::Duration::new(5, 0)); 9012 let total_memory = guest.get_total_memory().unwrap_or_default(); 9013 assert!(total_memory > 4_800_000); 9014 assert!(total_memory < 5_760_000); 9015 9016 // Check the guest virtio-devices, e.g. block, rng, console, and net 9017 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9018 9019 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9020 // to make sure that removing and adding back the virtio-net device does 9021 // not break the live-migration support for virtio-pci. 9022 #[cfg(target_arch = "x86_64")] 9023 { 9024 assert!(remote_command( 9025 &src_api_socket, 9026 "remove-device", 9027 Some(net_id), 9028 )); 9029 thread::sleep(std::time::Duration::new(10, 0)); 9030 9031 // Plug the virtio-net device again 9032 assert!(remote_command( 9033 &src_api_socket, 9034 "add-net", 9035 Some(net_params.as_str()), 9036 )); 9037 thread::sleep(std::time::Duration::new(10, 0)); 9038 } 9039 9040 // Start the live-migration 9041 let migration_socket = String::from( 9042 guest 9043 .tmp_dir 9044 .as_path() 9045 .join("live-migration.sock") 9046 .to_str() 9047 .unwrap(), 9048 ); 9049 9050 assert!( 9051 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9052 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9053 ); 9054 }); 9055 9056 // Check and report any errors occurred during the live-migration 9057 if r.is_err() { 9058 print_and_panic( 9059 src_child, 9060 dest_child, 9061 None, 9062 "Error occurred during live-migration", 9063 ); 9064 } 9065 9066 // Check the source vm has been terminated successful (give it '3s' to settle) 9067 thread::sleep(std::time::Duration::new(3, 0)); 9068 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9069 print_and_panic( 9070 src_child, 9071 dest_child, 9072 None, 9073 "source VM was not terminated successfully.", 9074 ); 9075 }; 9076 9077 // Post live-migration check to make sure the destination VM is funcational 9078 let r = std::panic::catch_unwind(|| { 9079 // Perform same checks to validate VM has been properly migrated 9080 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9081 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9082 9083 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9084 9085 // Perform checks on guest RAM using balloon 9086 let total_memory = guest.get_total_memory().unwrap_or_default(); 9087 assert!(total_memory > 4_800_000); 9088 assert!(total_memory < 5_760_000); 9089 // Deflate balloon to restore entire RAM to the VM 9090 resize_command(&dest_api_socket, None, None, Some(0), None); 9091 thread::sleep(std::time::Duration::new(5, 0)); 9092 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9093 // Decrease guest RAM with virtio-mem 9094 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 9095 thread::sleep(std::time::Duration::new(5, 0)); 9096 let total_memory = guest.get_total_memory().unwrap_or_default(); 9097 assert!(total_memory > 4_800_000); 9098 assert!(total_memory < 5_760_000); 9099 }); 9100 9101 // Clean-up the destination VM and make sure it terminated correctly 9102 let _ = dest_child.kill(); 9103 let dest_output = dest_child.wait_with_output().unwrap(); 9104 handle_child_output(r, &dest_output); 9105 9106 // Check the destination VM has the expected 'concole_text' from its output 9107 let r = std::panic::catch_unwind(|| { 9108 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9109 }); 9110 handle_child_output(r, &dest_output); 9111 } 9112 9113 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 9114 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9115 let guest = Guest::new(Box::new(focal)); 9116 let kernel_path = direct_kernel_boot_path(); 9117 let console_text = String::from("On a branch floating down river a cricket, singing."); 9118 let net_id = "net123"; 9119 let net_params = format!( 9120 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9121 net_id, guest.network.guest_mac, guest.network.host_ip 9122 ); 9123 9124 let memory_param: &[&str] = if local { 9125 &[ 9126 "--memory", 9127 "size=0,hotplug_method=virtio-mem,shared=on", 9128 "--memory-zone", 9129 "id=mem0,size=1G,hotplug_size=4G,shared=on", 9130 "id=mem1,size=1G,hotplug_size=4G,shared=on", 9131 "id=mem2,size=2G,hotplug_size=4G,shared=on", 9132 "--numa", 9133 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9134 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9135 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9136 ] 9137 } else { 9138 &[ 9139 "--memory", 9140 "size=0,hotplug_method=virtio-mem", 9141 "--memory-zone", 9142 "id=mem0,size=1G,hotplug_size=4G", 9143 "id=mem1,size=1G,hotplug_size=4G", 9144 "id=mem2,size=2G,hotplug_size=4G", 9145 "--numa", 9146 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9147 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9148 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9149 ] 9150 }; 9151 9152 let boot_vcpus = 6; 9153 let max_vcpus = 12; 9154 9155 let pmem_temp_file = TempFile::new().unwrap(); 9156 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9157 std::process::Command::new("mkfs.ext4") 9158 .arg(pmem_temp_file.as_path()) 9159 .output() 9160 .expect("Expect creating disk image to succeed"); 9161 let pmem_path = String::from("/dev/pmem0"); 9162 9163 // Start the source VM 9164 let src_vm_path = if !upgrade_test { 9165 clh_command("cloud-hypervisor") 9166 } else { 9167 cloud_hypervisor_release_path() 9168 }; 9169 let src_api_socket = temp_api_path(&guest.tmp_dir); 9170 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9171 src_vm_cmd 9172 .args([ 9173 "--cpus", 9174 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9175 ]) 9176 .args(memory_param) 9177 .args(["--kernel", kernel_path.to_str().unwrap()]) 9178 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9179 .default_disks() 9180 .args(["--net", net_params.as_str()]) 9181 .args(["--api-socket", &src_api_socket]) 9182 .args([ 9183 "--pmem", 9184 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9185 ]); 9186 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9187 9188 // Start the destination VM 9189 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9190 dest_api_socket.push_str(".dest"); 9191 let mut dest_child = GuestCommand::new(&guest) 9192 .args(["--api-socket", &dest_api_socket]) 9193 .capture_output() 9194 .spawn() 9195 .unwrap(); 9196 9197 let r = std::panic::catch_unwind(|| { 9198 guest.wait_vm_boot(None).unwrap(); 9199 9200 // Make sure the source VM is functaionl 9201 // Check the number of vCPUs 9202 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9203 9204 // Check the guest RAM 9205 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 9206 9207 // Check the guest virtio-devices, e.g. block, rng, console, and net 9208 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9209 9210 // Check the NUMA parameters are applied correctly and resize 9211 // each zone to test the case where we migrate a VM with the 9212 // virtio-mem regions being used. 9213 { 9214 guest.check_numa_common( 9215 Some(&[960_000, 960_000, 1_920_000]), 9216 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9217 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9218 ); 9219 9220 // AArch64 currently does not support hotplug, and therefore we only 9221 // test hotplug-related function on x86_64 here. 9222 #[cfg(target_arch = "x86_64")] 9223 { 9224 guest.enable_memory_hotplug(); 9225 9226 // Resize every memory zone and check each associated NUMA node 9227 // has been assigned the right amount of memory. 9228 resize_zone_command(&src_api_socket, "mem0", "2G"); 9229 resize_zone_command(&src_api_socket, "mem1", "2G"); 9230 resize_zone_command(&src_api_socket, "mem2", "3G"); 9231 thread::sleep(std::time::Duration::new(5, 0)); 9232 9233 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 9234 } 9235 } 9236 9237 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9238 // to make sure that removing and adding back the virtio-net device does 9239 // not break the live-migration support for virtio-pci. 9240 #[cfg(target_arch = "x86_64")] 9241 { 9242 assert!(remote_command( 9243 &src_api_socket, 9244 "remove-device", 9245 Some(net_id), 9246 )); 9247 thread::sleep(std::time::Duration::new(10, 0)); 9248 9249 // Plug the virtio-net device again 9250 assert!(remote_command( 9251 &src_api_socket, 9252 "add-net", 9253 Some(net_params.as_str()), 9254 )); 9255 thread::sleep(std::time::Duration::new(10, 0)); 9256 } 9257 9258 // Start the live-migration 9259 let migration_socket = String::from( 9260 guest 9261 .tmp_dir 9262 .as_path() 9263 .join("live-migration.sock") 9264 .to_str() 9265 .unwrap(), 9266 ); 9267 9268 assert!( 9269 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9270 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9271 ); 9272 }); 9273 9274 // Check and report any errors occurred during the live-migration 9275 if r.is_err() { 9276 print_and_panic( 9277 src_child, 9278 dest_child, 9279 None, 9280 "Error occurred during live-migration", 9281 ); 9282 } 9283 9284 // Check the source vm has been terminated successful (give it '3s' to settle) 9285 thread::sleep(std::time::Duration::new(3, 0)); 9286 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9287 print_and_panic( 9288 src_child, 9289 dest_child, 9290 None, 9291 "source VM was not terminated successfully.", 9292 ); 9293 }; 9294 9295 // Post live-migration check to make sure the destination VM is funcational 9296 let r = std::panic::catch_unwind(|| { 9297 // Perform same checks to validate VM has been properly migrated 9298 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9299 #[cfg(target_arch = "x86_64")] 9300 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 9301 #[cfg(target_arch = "aarch64")] 9302 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9303 9304 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9305 9306 // Perform NUMA related checks 9307 { 9308 #[cfg(target_arch = "aarch64")] 9309 { 9310 guest.check_numa_common( 9311 Some(&[960_000, 960_000, 1_920_000]), 9312 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9313 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9314 ); 9315 } 9316 9317 // AArch64 currently does not support hotplug, and therefore we only 9318 // test hotplug-related function on x86_64 here. 9319 #[cfg(target_arch = "x86_64")] 9320 { 9321 guest.check_numa_common( 9322 Some(&[1_920_000, 1_920_000, 2_880_000]), 9323 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9324 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9325 ); 9326 9327 guest.enable_memory_hotplug(); 9328 9329 // Resize every memory zone and check each associated NUMA node 9330 // has been assigned the right amount of memory. 9331 resize_zone_command(&dest_api_socket, "mem0", "4G"); 9332 resize_zone_command(&dest_api_socket, "mem1", "4G"); 9333 resize_zone_command(&dest_api_socket, "mem2", "4G"); 9334 // Resize to the maximum amount of CPUs and check each NUMA 9335 // node has been assigned the right CPUs set. 9336 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 9337 thread::sleep(std::time::Duration::new(5, 0)); 9338 9339 guest.check_numa_common( 9340 Some(&[3_840_000, 3_840_000, 3_840_000]), 9341 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 9342 None, 9343 ); 9344 } 9345 } 9346 }); 9347 9348 // Clean-up the destination VM and make sure it terminated correctly 9349 let _ = dest_child.kill(); 9350 let dest_output = dest_child.wait_with_output().unwrap(); 9351 handle_child_output(r, &dest_output); 9352 9353 // Check the destination VM has the expected 'concole_text' from its output 9354 let r = std::panic::catch_unwind(|| { 9355 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9356 }); 9357 handle_child_output(r, &dest_output); 9358 } 9359 9360 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 9361 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9362 let guest = Guest::new(Box::new(focal)); 9363 let kernel_path = direct_kernel_boot_path(); 9364 let console_text = String::from("On a branch floating down river a cricket, singing."); 9365 let net_id = "net123"; 9366 let net_params = format!( 9367 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9368 net_id, guest.network.guest_mac, guest.network.host_ip 9369 ); 9370 9371 let memory_param: &[&str] = if local { 9372 &["--memory", "size=4G,shared=on"] 9373 } else { 9374 &["--memory", "size=4G"] 9375 }; 9376 9377 let boot_vcpus = 2; 9378 let max_vcpus = 4; 9379 9380 let pmem_temp_file = TempFile::new().unwrap(); 9381 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9382 std::process::Command::new("mkfs.ext4") 9383 .arg(pmem_temp_file.as_path()) 9384 .output() 9385 .expect("Expect creating disk image to succeed"); 9386 let pmem_path = String::from("/dev/pmem0"); 9387 9388 // Start the source VM 9389 let src_vm_path = if !upgrade_test { 9390 clh_command("cloud-hypervisor") 9391 } else { 9392 cloud_hypervisor_release_path() 9393 }; 9394 let src_api_socket = temp_api_path(&guest.tmp_dir); 9395 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9396 src_vm_cmd 9397 .args([ 9398 "--cpus", 9399 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9400 ]) 9401 .args(memory_param) 9402 .args(["--kernel", kernel_path.to_str().unwrap()]) 9403 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9404 .default_disks() 9405 .args(["--net", net_params.as_str()]) 9406 .args(["--api-socket", &src_api_socket]) 9407 .args([ 9408 "--pmem", 9409 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9410 ]) 9411 .args(["--watchdog"]); 9412 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9413 9414 // Start the destination VM 9415 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9416 dest_api_socket.push_str(".dest"); 9417 let mut dest_child = GuestCommand::new(&guest) 9418 .args(["--api-socket", &dest_api_socket]) 9419 .capture_output() 9420 .spawn() 9421 .unwrap(); 9422 9423 let r = std::panic::catch_unwind(|| { 9424 guest.wait_vm_boot(None).unwrap(); 9425 9426 // Make sure the source VM is functaionl 9427 // Check the number of vCPUs 9428 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9429 // Check the guest RAM 9430 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9431 // Check the guest virtio-devices, e.g. block, rng, console, and net 9432 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9433 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9434 // to make sure that removing and adding back the virtio-net device does 9435 // not break the live-migration support for virtio-pci. 9436 #[cfg(target_arch = "x86_64")] 9437 { 9438 assert!(remote_command( 9439 &src_api_socket, 9440 "remove-device", 9441 Some(net_id), 9442 )); 9443 thread::sleep(std::time::Duration::new(10, 0)); 9444 9445 // Plug the virtio-net device again 9446 assert!(remote_command( 9447 &src_api_socket, 9448 "add-net", 9449 Some(net_params.as_str()), 9450 )); 9451 thread::sleep(std::time::Duration::new(10, 0)); 9452 } 9453 9454 // Enable watchdog and ensure its functional 9455 let mut expected_reboot_count = 1; 9456 // Enable the watchdog with a 15s timeout 9457 enable_guest_watchdog(&guest, 15); 9458 // Reboot and check that systemd has activated the watchdog 9459 guest.ssh_command("sudo reboot").unwrap(); 9460 guest.wait_vm_boot(None).unwrap(); 9461 expected_reboot_count += 1; 9462 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9463 assert_eq!( 9464 guest 9465 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 9466 .unwrap() 9467 .trim() 9468 .parse::<u32>() 9469 .unwrap_or_default(), 9470 2 9471 ); 9472 // Allow some normal time to elapse to check we don't get spurious reboots 9473 thread::sleep(std::time::Duration::new(40, 0)); 9474 // Check no reboot 9475 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9476 9477 // Start the live-migration 9478 let migration_socket = String::from( 9479 guest 9480 .tmp_dir 9481 .as_path() 9482 .join("live-migration.sock") 9483 .to_str() 9484 .unwrap(), 9485 ); 9486 9487 assert!( 9488 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9489 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9490 ); 9491 }); 9492 9493 // Check and report any errors occurred during the live-migration 9494 if r.is_err() { 9495 print_and_panic( 9496 src_child, 9497 dest_child, 9498 None, 9499 "Error occurred during live-migration", 9500 ); 9501 } 9502 9503 // Check the source vm has been terminated successful (give it '3s' to settle) 9504 thread::sleep(std::time::Duration::new(3, 0)); 9505 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9506 print_and_panic( 9507 src_child, 9508 dest_child, 9509 None, 9510 "source VM was not terminated successfully.", 9511 ); 9512 }; 9513 9514 // Post live-migration check to make sure the destination VM is funcational 9515 let r = std::panic::catch_unwind(|| { 9516 // Perform same checks to validate VM has been properly migrated 9517 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9518 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9519 9520 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9521 9522 // Perform checks on watchdog 9523 let mut expected_reboot_count = 2; 9524 9525 // Allow some normal time to elapse to check we don't get spurious reboots 9526 thread::sleep(std::time::Duration::new(40, 0)); 9527 // Check no reboot 9528 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9529 9530 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 9531 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 9532 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 9533 guest.wait_vm_boot(Some(50)).unwrap(); 9534 // Check a reboot is triggered by the watchdog 9535 expected_reboot_count += 1; 9536 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9537 9538 #[cfg(target_arch = "x86_64")] 9539 { 9540 // Now pause the VM and remain offline for 30s 9541 assert!(remote_command(&dest_api_socket, "pause", None)); 9542 thread::sleep(std::time::Duration::new(30, 0)); 9543 assert!(remote_command(&dest_api_socket, "resume", None)); 9544 9545 // Check no reboot 9546 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9547 } 9548 }); 9549 9550 // Clean-up the destination VM and make sure it terminated correctly 9551 let _ = dest_child.kill(); 9552 let dest_output = dest_child.wait_with_output().unwrap(); 9553 handle_child_output(r, &dest_output); 9554 9555 // Check the destination VM has the expected 'concole_text' from its output 9556 let r = std::panic::catch_unwind(|| { 9557 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9558 }); 9559 handle_child_output(r, &dest_output); 9560 } 9561 9562 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 9563 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9564 let ovs_guest = Guest::new(Box::new(ovs_focal)); 9565 9566 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9567 let migration_guest = Guest::new(Box::new(migration_focal)); 9568 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 9569 9570 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 9571 let (mut ovs_child, mut src_child) = 9572 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 9573 9574 // Start the destination VM 9575 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 9576 dest_api_socket.push_str(".dest"); 9577 let mut dest_child = GuestCommand::new(&migration_guest) 9578 .args(["--api-socket", &dest_api_socket]) 9579 .capture_output() 9580 .spawn() 9581 .unwrap(); 9582 9583 let r = std::panic::catch_unwind(|| { 9584 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 9585 thread::sleep(std::time::Duration::new(1, 0)); 9586 9587 // Start the live-migration 9588 let migration_socket = String::from( 9589 migration_guest 9590 .tmp_dir 9591 .as_path() 9592 .join("live-migration.sock") 9593 .to_str() 9594 .unwrap(), 9595 ); 9596 9597 assert!( 9598 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9599 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9600 ); 9601 }); 9602 9603 // Check and report any errors occurred during the live-migration 9604 if r.is_err() { 9605 print_and_panic( 9606 src_child, 9607 dest_child, 9608 Some(ovs_child), 9609 "Error occurred during live-migration", 9610 ); 9611 } 9612 9613 // Check the source vm has been terminated successful (give it '3s' to settle) 9614 thread::sleep(std::time::Duration::new(3, 0)); 9615 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9616 print_and_panic( 9617 src_child, 9618 dest_child, 9619 Some(ovs_child), 9620 "source VM was not terminated successfully.", 9621 ); 9622 }; 9623 9624 // Post live-migration check to make sure the destination VM is funcational 9625 let r = std::panic::catch_unwind(|| { 9626 // Perform same checks to validate VM has been properly migrated 9627 // Spawn a new netcat listener in the OVS VM 9628 let guest_ip = ovs_guest.network.guest_ip.clone(); 9629 thread::spawn(move || { 9630 ssh_command_ip( 9631 "nc -l 12345", 9632 &guest_ip, 9633 DEFAULT_SSH_RETRIES, 9634 DEFAULT_SSH_TIMEOUT, 9635 ) 9636 .unwrap(); 9637 }); 9638 9639 // Wait for the server to be listening 9640 thread::sleep(std::time::Duration::new(5, 0)); 9641 9642 // And check the connection is still functional after live-migration 9643 migration_guest 9644 .ssh_command("nc -vz 172.100.0.1 12345") 9645 .unwrap(); 9646 }); 9647 9648 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 9649 let _ = dest_child.kill(); 9650 let _ = ovs_child.kill(); 9651 let dest_output = dest_child.wait_with_output().unwrap(); 9652 let ovs_output = ovs_child.wait_with_output().unwrap(); 9653 9654 cleanup_ovs_dpdk(); 9655 9656 handle_child_output(r, &dest_output); 9657 handle_child_output(Ok(()), &ovs_output); 9658 } 9659 9660 mod live_migration_parallel { 9661 use super::*; 9662 #[test] 9663 fn test_live_migration_basic() { 9664 _test_live_migration(false, false) 9665 } 9666 9667 #[test] 9668 fn test_live_migration_local() { 9669 _test_live_migration(false, true) 9670 } 9671 9672 #[test] 9673 #[cfg(not(feature = "mshv"))] 9674 fn test_live_migration_numa() { 9675 _test_live_migration_numa(false, false) 9676 } 9677 9678 #[test] 9679 #[cfg(not(feature = "mshv"))] 9680 fn test_live_migration_numa_local() { 9681 _test_live_migration_numa(false, true) 9682 } 9683 9684 #[test] 9685 fn test_live_migration_watchdog() { 9686 _test_live_migration_watchdog(false, false) 9687 } 9688 9689 #[test] 9690 fn test_live_migration_watchdog_local() { 9691 _test_live_migration_watchdog(false, true) 9692 } 9693 9694 #[test] 9695 fn test_live_migration_balloon() { 9696 _test_live_migration_balloon(false, false) 9697 } 9698 9699 #[test] 9700 fn test_live_migration_balloon_local() { 9701 _test_live_migration_balloon(false, true) 9702 } 9703 9704 #[test] 9705 #[ignore = "See #6134"] 9706 fn test_live_upgrade_basic() { 9707 _test_live_migration(true, false) 9708 } 9709 9710 #[test] 9711 #[ignore = "See #6134"] 9712 fn test_live_upgrade_local() { 9713 _test_live_migration(true, true) 9714 } 9715 9716 #[test] 9717 #[ignore = "See #6134"] 9718 #[cfg(not(feature = "mshv"))] 9719 fn test_live_upgrade_numa() { 9720 _test_live_migration_numa(true, false) 9721 } 9722 9723 #[test] 9724 #[ignore = "See #6134"] 9725 #[cfg(not(feature = "mshv"))] 9726 fn test_live_upgrade_numa_local() { 9727 _test_live_migration_numa(true, true) 9728 } 9729 9730 #[test] 9731 #[ignore = "See #6134"] 9732 fn test_live_upgrade_watchdog() { 9733 _test_live_migration_watchdog(true, false) 9734 } 9735 9736 #[test] 9737 #[ignore = "See #6134"] 9738 fn test_live_upgrade_watchdog_local() { 9739 _test_live_migration_watchdog(true, true) 9740 } 9741 9742 #[test] 9743 #[ignore = "See #6134"] 9744 fn test_live_upgrade_balloon() { 9745 _test_live_migration_balloon(true, false) 9746 } 9747 9748 #[test] 9749 #[ignore = "See #6134"] 9750 fn test_live_upgrade_balloon_local() { 9751 _test_live_migration_balloon(true, true) 9752 } 9753 } 9754 9755 mod live_migration_sequential { 9756 #[cfg(target_arch = "x86_64")] 9757 #[cfg(not(feature = "mshv"))] 9758 use super::*; 9759 9760 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 9761 #[test] 9762 #[ignore = "See #5532"] 9763 #[cfg(target_arch = "x86_64")] 9764 #[cfg(not(feature = "mshv"))] 9765 fn test_live_migration_ovs_dpdk() { 9766 _test_live_migration_ovs_dpdk(false, false); 9767 } 9768 9769 #[test] 9770 #[cfg(target_arch = "x86_64")] 9771 #[cfg(not(feature = "mshv"))] 9772 fn test_live_migration_ovs_dpdk_local() { 9773 _test_live_migration_ovs_dpdk(false, true); 9774 } 9775 9776 #[test] 9777 #[ignore = "See #5532"] 9778 #[cfg(target_arch = "x86_64")] 9779 #[cfg(not(feature = "mshv"))] 9780 fn test_live_upgrade_ovs_dpdk() { 9781 _test_live_migration_ovs_dpdk(true, false); 9782 } 9783 9784 #[test] 9785 #[ignore = "See #5532"] 9786 #[cfg(target_arch = "x86_64")] 9787 #[cfg(not(feature = "mshv"))] 9788 fn test_live_upgrade_ovs_dpdk_local() { 9789 _test_live_migration_ovs_dpdk(true, true); 9790 } 9791 } 9792 } 9793 9794 #[cfg(target_arch = "aarch64")] 9795 mod aarch64_acpi { 9796 use crate::*; 9797 9798 #[test] 9799 fn test_simple_launch_acpi() { 9800 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9801 9802 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 9803 let guest = Guest::new(disk_config); 9804 9805 let mut child = GuestCommand::new(&guest) 9806 .args(["--cpus", "boot=1"]) 9807 .args(["--memory", "size=512M"]) 9808 .args(["--kernel", edk2_path().to_str().unwrap()]) 9809 .default_disks() 9810 .default_net() 9811 .args(["--serial", "tty", "--console", "off"]) 9812 .capture_output() 9813 .spawn() 9814 .unwrap(); 9815 9816 let r = std::panic::catch_unwind(|| { 9817 guest.wait_vm_boot(Some(120)).unwrap(); 9818 9819 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 9820 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 9821 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 9822 }); 9823 9824 let _ = child.kill(); 9825 let output = child.wait_with_output().unwrap(); 9826 9827 handle_child_output(r, &output); 9828 }); 9829 } 9830 9831 #[test] 9832 fn test_guest_numa_nodes_acpi() { 9833 _test_guest_numa_nodes(true); 9834 } 9835 9836 #[test] 9837 fn test_cpu_topology_421_acpi() { 9838 test_cpu_topology(4, 2, 1, true); 9839 } 9840 9841 #[test] 9842 fn test_cpu_topology_142_acpi() { 9843 test_cpu_topology(1, 4, 2, true); 9844 } 9845 9846 #[test] 9847 fn test_cpu_topology_262_acpi() { 9848 test_cpu_topology(2, 6, 2, true); 9849 } 9850 9851 #[test] 9852 fn test_power_button_acpi() { 9853 _test_power_button(true); 9854 } 9855 9856 #[test] 9857 fn test_virtio_iommu() { 9858 _test_virtio_iommu(true) 9859 } 9860 } 9861 9862 mod rate_limiter { 9863 use super::*; 9864 9865 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 9866 // compared to given 'limit' rate. 9867 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 9868 let upper_limit = limit * (1_f64 + difference); 9869 let lower_limit = limit * (1_f64 - difference); 9870 9871 if measured > lower_limit && measured < upper_limit { 9872 return true; 9873 } 9874 9875 eprintln!( 9876 "\n\n==== Start 'check_rate_limit' failed ==== \ 9877 \n\nmeasured={measured}, , lower_limit={lower_limit}, upper_limit={upper_limit} \ 9878 \n\n==== End 'check_rate_limit' failed ====\n\n" 9879 ); 9880 9881 false 9882 } 9883 9884 fn _test_rate_limiter_net(rx: bool) { 9885 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9886 let guest = Guest::new(Box::new(focal)); 9887 9888 let test_timeout = 10; 9889 let num_queues = 2; 9890 let queue_size = 256; 9891 let bw_size = 10485760_u64; // bytes 9892 let bw_refill_time = 100; // ms 9893 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 9894 9895 let net_params = format!( 9896 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 9897 guest.network.guest_mac, 9898 guest.network.host_ip, 9899 num_queues, 9900 queue_size, 9901 bw_size, 9902 bw_refill_time, 9903 ); 9904 9905 let mut child = GuestCommand::new(&guest) 9906 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 9907 .args(["--memory", "size=4G"]) 9908 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9909 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9910 .default_disks() 9911 .args(["--net", net_params.as_str()]) 9912 .capture_output() 9913 .spawn() 9914 .unwrap(); 9915 9916 let r = std::panic::catch_unwind(|| { 9917 guest.wait_vm_boot(None).unwrap(); 9918 let measured_bps = 9919 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, true) 9920 .unwrap(); 9921 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 9922 }); 9923 9924 let _ = child.kill(); 9925 let output = child.wait_with_output().unwrap(); 9926 handle_child_output(r, &output); 9927 } 9928 9929 #[test] 9930 fn test_rate_limiter_net_rx() { 9931 _test_rate_limiter_net(true); 9932 } 9933 9934 #[test] 9935 fn test_rate_limiter_net_tx() { 9936 _test_rate_limiter_net(false); 9937 } 9938 9939 fn _test_rate_limiter_block(bandwidth: bool, num_queues: u32) { 9940 let test_timeout = 10; 9941 let fio_ops = FioOps::RandRW; 9942 9943 let bw_size = if bandwidth { 9944 10485760_u64 // bytes 9945 } else { 9946 100_u64 // I/O 9947 }; 9948 let bw_refill_time = 100; // ms 9949 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 9950 9951 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9952 let guest = Guest::new(Box::new(focal)); 9953 let api_socket = temp_api_path(&guest.tmp_dir); 9954 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 9955 let blk_rate_limiter_test_img = 9956 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 9957 9958 // Create the test block image 9959 assert!(exec_host_command_output(&format!( 9960 "dd if=/dev/zero of={blk_rate_limiter_test_img} bs=1M count=1024" 9961 )) 9962 .status 9963 .success()); 9964 9965 let test_blk_params = if bandwidth { 9966 format!( 9967 "path={blk_rate_limiter_test_img},num_queues={num_queues},bw_size={bw_size},bw_refill_time={bw_refill_time}" 9968 ) 9969 } else { 9970 format!( 9971 "path={blk_rate_limiter_test_img},num_queues={num_queues},ops_size={bw_size},ops_refill_time={bw_refill_time}" 9972 ) 9973 }; 9974 9975 let mut child = GuestCommand::new(&guest) 9976 .args(["--cpus", &format!("boot={num_queues}")]) 9977 .args(["--memory", "size=4G"]) 9978 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9979 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9980 .args([ 9981 "--disk", 9982 format!( 9983 "path={}", 9984 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 9985 ) 9986 .as_str(), 9987 format!( 9988 "path={}", 9989 guest.disk_config.disk(DiskType::CloudInit).unwrap() 9990 ) 9991 .as_str(), 9992 test_blk_params.as_str(), 9993 ]) 9994 .default_net() 9995 .args(["--api-socket", &api_socket]) 9996 .capture_output() 9997 .spawn() 9998 .unwrap(); 9999 10000 let r = std::panic::catch_unwind(|| { 10001 guest.wait_vm_boot(None).unwrap(); 10002 10003 let fio_command = format!( 10004 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 10005 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10006 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10007 ); 10008 let output = guest.ssh_command(&fio_command).unwrap(); 10009 10010 // Parse fio output 10011 let measured_rate = if bandwidth { 10012 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 10013 } else { 10014 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 10015 }; 10016 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10017 }); 10018 10019 let _ = child.kill(); 10020 let output = child.wait_with_output().unwrap(); 10021 handle_child_output(r, &output); 10022 } 10023 10024 fn _test_rate_limiter_group_block(bandwidth: bool, num_queues: u32, num_disks: u32) { 10025 let test_timeout = 10; 10026 let fio_ops = FioOps::RandRW; 10027 10028 let bw_size = if bandwidth { 10029 10485760_u64 // bytes 10030 } else { 10031 100_u64 // I/O 10032 }; 10033 let bw_refill_time = 100; // ms 10034 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10035 10036 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10037 let guest = Guest::new(Box::new(focal)); 10038 let api_socket = temp_api_path(&guest.tmp_dir); 10039 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10040 10041 let rate_limit_group_arg = if bandwidth { 10042 format!("id=group0,bw_size={bw_size},bw_refill_time={bw_refill_time}") 10043 } else { 10044 format!("id=group0,ops_size={bw_size},ops_refill_time={bw_refill_time}") 10045 }; 10046 10047 let mut disk_args = vec![ 10048 "--disk".to_string(), 10049 format!( 10050 "path={}", 10051 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10052 ), 10053 format!( 10054 "path={}", 10055 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10056 ), 10057 ]; 10058 10059 for i in 0..num_disks { 10060 let test_img_path = String::from( 10061 test_img_dir 10062 .as_path() 10063 .join(format!("blk{}.img", i)) 10064 .to_str() 10065 .unwrap(), 10066 ); 10067 10068 assert!(exec_host_command_output(&format!( 10069 "dd if=/dev/zero of={test_img_path} bs=1M count=1024" 10070 )) 10071 .status 10072 .success()); 10073 10074 disk_args.push(format!( 10075 "path={test_img_path},num_queues={num_queues},rate_limit_group=group0" 10076 )); 10077 } 10078 10079 let mut child = GuestCommand::new(&guest) 10080 .args(["--cpus", &format!("boot={}", num_queues * num_disks)]) 10081 .args(["--memory", "size=4G"]) 10082 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10083 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10084 .args(["--rate-limit-group", &rate_limit_group_arg]) 10085 .args(disk_args) 10086 .default_net() 10087 .args(["--api-socket", &api_socket]) 10088 .capture_output() 10089 .spawn() 10090 .unwrap(); 10091 10092 let r = std::panic::catch_unwind(|| { 10093 guest.wait_vm_boot(None).unwrap(); 10094 10095 let mut fio_command = format!( 10096 "sudo fio --name=global --output-format=json \ 10097 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10098 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10099 ); 10100 10101 // Generate additional argument for each disk: 10102 // --name=job0 --filename=/dev/vdc \ 10103 // --name=job1 --filename=/dev/vdd \ 10104 // --name=job2 --filename=/dev/vde \ 10105 // ... 10106 for i in 0..num_disks { 10107 let c: char = 'c'; 10108 let arg = format!( 10109 " --name=job{i} --filename=/dev/vd{}", 10110 char::from_u32((c as u32) + i).unwrap() 10111 ); 10112 fio_command += &arg; 10113 } 10114 let output = guest.ssh_command(&fio_command).unwrap(); 10115 10116 // Parse fio output 10117 let measured_rate = if bandwidth { 10118 parse_fio_output(&output, &fio_ops, num_queues * num_disks).unwrap() 10119 } else { 10120 parse_fio_output_iops(&output, &fio_ops, num_queues * num_disks).unwrap() 10121 }; 10122 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10123 }); 10124 10125 let _ = child.kill(); 10126 let output = child.wait_with_output().unwrap(); 10127 handle_child_output(r, &output); 10128 } 10129 10130 #[test] 10131 fn test_rate_limiter_block_bandwidth() { 10132 _test_rate_limiter_block(true, 1); 10133 _test_rate_limiter_block(true, 2) 10134 } 10135 10136 #[test] 10137 fn test_rate_limiter_group_block_bandwidth() { 10138 _test_rate_limiter_group_block(true, 1, 1); 10139 _test_rate_limiter_group_block(true, 2, 1); 10140 _test_rate_limiter_group_block(true, 1, 2); 10141 _test_rate_limiter_group_block(true, 2, 2); 10142 } 10143 10144 #[test] 10145 fn test_rate_limiter_block_iops() { 10146 _test_rate_limiter_block(false, 1); 10147 _test_rate_limiter_block(false, 2); 10148 } 10149 10150 #[test] 10151 fn test_rate_limiter_group_block_iops() { 10152 _test_rate_limiter_group_block(false, 1, 1); 10153 _test_rate_limiter_group_block(false, 2, 1); 10154 _test_rate_limiter_group_block(false, 1, 2); 10155 _test_rate_limiter_group_block(false, 2, 2); 10156 } 10157 } 10158