1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 use std::sync::Mutex; 28 use std::thread; 29 use test_infra::*; 30 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 31 use wait_timeout::ChildExt; 32 33 // Constant taken from the VMM crate. 34 const MAX_NUM_PCI_SEGMENTS: u16 = 96; 35 36 #[cfg(target_arch = "x86_64")] 37 mod x86_64 { 38 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 39 pub const JAMMY_NVIDIA_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-nvidia.raw"; 40 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 41 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 42 "focal-server-cloudimg-amd64-custom-20210609-0-backing.qcow2"; 43 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 44 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 45 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20230119-0.raw"; 46 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2022-amd64-2.raw"; 47 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 48 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 49 } 50 51 #[cfg(target_arch = "x86_64")] 52 use x86_64::*; 53 54 #[cfg(target_arch = "aarch64")] 55 mod aarch64 { 56 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 57 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 58 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 59 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 60 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 61 "focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2"; 62 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 63 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 64 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 65 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 66 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 67 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 68 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 69 } 70 71 #[cfg(target_arch = "aarch64")] 72 use aarch64::*; 73 74 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 75 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 76 77 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 78 79 // This enum exists to make it more convenient to 80 // implement test for both D-Bus and REST APIs. 81 enum TargetApi { 82 // API socket 83 HttpApi(String), 84 // well known service name, object path 85 DBusApi(String, String), 86 } 87 88 impl TargetApi { 89 fn new_http_api(tmp_dir: &TempDir) -> Self { 90 Self::HttpApi(temp_api_path(tmp_dir)) 91 } 92 93 fn new_dbus_api(tmp_dir: &TempDir) -> Self { 94 // `tmp_dir` is in the form of "/tmp/chXXXXXX" 95 // and we take the `chXXXXXX` part as a unique identifier for the guest 96 let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); 97 98 Self::DBusApi( 99 format!("org.cloudhypervisor.{id}"), 100 format!("/org/cloudhypervisor/{id}"), 101 ) 102 } 103 104 fn guest_args(&self) -> Vec<String> { 105 match self { 106 TargetApi::HttpApi(api_socket) => { 107 vec![format!("--api-socket={}", api_socket.as_str())] 108 } 109 TargetApi::DBusApi(service_name, object_path) => { 110 vec![ 111 format!("--dbus-service-name={}", service_name.as_str()), 112 format!("--dbus-object-path={}", object_path.as_str()), 113 ] 114 } 115 } 116 } 117 118 fn remote_args(&self) -> Vec<String> { 119 // `guest_args` and `remote_args` are consistent with each other 120 self.guest_args() 121 } 122 123 fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { 124 let mut cmd = Command::new(clh_command("ch-remote")); 125 cmd.args(self.remote_args()); 126 cmd.arg(command); 127 128 if let Some(arg) = arg { 129 cmd.arg(arg); 130 } 131 132 let output = cmd.output().unwrap(); 133 if output.status.success() { 134 true 135 } else { 136 eprintln!("Error running ch-remote command: {:?}", &cmd); 137 let stderr = String::from_utf8_lossy(&output.stderr); 138 eprintln!("stderr: {stderr}"); 139 false 140 } 141 } 142 } 143 144 // Start cloud-hypervisor with no VM parameters, only the API server running. 145 // From the API: Create a VM, boot it and check that it looks as expected. 146 fn _test_api_create_boot(target_api: TargetApi, guest: Guest) { 147 let mut child = GuestCommand::new(&guest) 148 .args(target_api.guest_args()) 149 .capture_output() 150 .spawn() 151 .unwrap(); 152 153 thread::sleep(std::time::Duration::new(1, 0)); 154 155 // Verify API server is running 156 assert!(target_api.remote_command("ping", None)); 157 158 // Create the VM first 159 let cpu_count: u8 = 4; 160 let request_body = guest.api_create_body( 161 cpu_count, 162 direct_kernel_boot_path().to_str().unwrap(), 163 DIRECT_KERNEL_BOOT_CMDLINE, 164 ); 165 166 let temp_config_path = guest.tmp_dir.as_path().join("config"); 167 std::fs::write(&temp_config_path, request_body).unwrap(); 168 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 169 170 assert!(target_api.remote_command("create", Some(create_config),)); 171 172 // Then boot it 173 assert!(target_api.remote_command("boot", None)); 174 thread::sleep(std::time::Duration::new(20, 0)); 175 176 let r = std::panic::catch_unwind(|| { 177 // Check that the VM booted as expected 178 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 179 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 180 }); 181 182 let _ = child.kill(); 183 let output = child.wait_with_output().unwrap(); 184 185 handle_child_output(r, &output); 186 } 187 188 // Start cloud-hypervisor with no VM parameters, only the API server running. 189 // From the API: Create a VM, boot it and check it can be shutdown and then 190 // booted again 191 fn _test_api_shutdown(target_api: TargetApi, guest: Guest) { 192 let mut child = GuestCommand::new(&guest) 193 .args(target_api.guest_args()) 194 .capture_output() 195 .spawn() 196 .unwrap(); 197 198 thread::sleep(std::time::Duration::new(1, 0)); 199 200 // Verify API server is running 201 assert!(target_api.remote_command("ping", None)); 202 203 // Create the VM first 204 let cpu_count: u8 = 4; 205 let request_body = guest.api_create_body( 206 cpu_count, 207 direct_kernel_boot_path().to_str().unwrap(), 208 DIRECT_KERNEL_BOOT_CMDLINE, 209 ); 210 211 let temp_config_path = guest.tmp_dir.as_path().join("config"); 212 std::fs::write(&temp_config_path, request_body).unwrap(); 213 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 214 215 let r = std::panic::catch_unwind(|| { 216 assert!(target_api.remote_command("create", Some(create_config))); 217 218 // Then boot it 219 assert!(target_api.remote_command("boot", None)); 220 221 guest.wait_vm_boot(None).unwrap(); 222 223 // Check that the VM booted as expected 224 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 225 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 226 227 // Sync and shutdown without powering off to prevent filesystem 228 // corruption. 229 guest.ssh_command("sync").unwrap(); 230 guest.ssh_command("sudo shutdown -H now").unwrap(); 231 232 // Wait for the guest to be fully shutdown 233 thread::sleep(std::time::Duration::new(20, 0)); 234 235 // Then shut it down 236 assert!(target_api.remote_command("shutdown", None)); 237 238 // Then boot it again 239 assert!(target_api.remote_command("boot", None)); 240 241 guest.wait_vm_boot(None).unwrap(); 242 243 // Check that the VM booted as expected 244 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 245 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 246 }); 247 248 let _ = child.kill(); 249 let output = child.wait_with_output().unwrap(); 250 251 handle_child_output(r, &output); 252 } 253 254 // Start cloud-hypervisor with no VM parameters, only the API server running. 255 // From the API: Create a VM, boot it and check it can be deleted and then recreated 256 // booted again. 257 fn _test_api_delete(target_api: TargetApi, guest: Guest) { 258 let mut child = GuestCommand::new(&guest) 259 .args(target_api.guest_args()) 260 .capture_output() 261 .spawn() 262 .unwrap(); 263 264 thread::sleep(std::time::Duration::new(1, 0)); 265 266 // Verify API server is running 267 assert!(target_api.remote_command("ping", None)); 268 269 // Create the VM first 270 let cpu_count: u8 = 4; 271 let request_body = guest.api_create_body( 272 cpu_count, 273 direct_kernel_boot_path().to_str().unwrap(), 274 DIRECT_KERNEL_BOOT_CMDLINE, 275 ); 276 let temp_config_path = guest.tmp_dir.as_path().join("config"); 277 std::fs::write(&temp_config_path, request_body).unwrap(); 278 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 279 280 let r = std::panic::catch_unwind(|| { 281 assert!(target_api.remote_command("create", Some(create_config))); 282 283 // Then boot it 284 assert!(target_api.remote_command("boot", None)); 285 286 guest.wait_vm_boot(None).unwrap(); 287 288 // Check that the VM booted as expected 289 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 290 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 291 292 // Sync and shutdown without powering off to prevent filesystem 293 // corruption. 294 guest.ssh_command("sync").unwrap(); 295 guest.ssh_command("sudo shutdown -H now").unwrap(); 296 297 // Wait for the guest to be fully shutdown 298 thread::sleep(std::time::Duration::new(20, 0)); 299 300 // Then delete it 301 assert!(target_api.remote_command("delete", None)); 302 303 assert!(target_api.remote_command("create", Some(create_config))); 304 305 // Then boot it again 306 assert!(target_api.remote_command("boot", None)); 307 308 guest.wait_vm_boot(None).unwrap(); 309 310 // Check that the VM booted as expected 311 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 312 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 313 }); 314 315 let _ = child.kill(); 316 let output = child.wait_with_output().unwrap(); 317 318 handle_child_output(r, &output); 319 } 320 321 // Start cloud-hypervisor with no VM parameters, only the API server running. 322 // From the API: Create a VM, boot it and check that it looks as expected. 323 // Then we pause the VM, check that it's no longer available. 324 // Finally we resume the VM and check that it's available. 325 fn _test_api_pause_resume(target_api: TargetApi, guest: Guest) { 326 let mut child = GuestCommand::new(&guest) 327 .args(target_api.guest_args()) 328 .capture_output() 329 .spawn() 330 .unwrap(); 331 332 thread::sleep(std::time::Duration::new(1, 0)); 333 334 // Verify API server is running 335 assert!(target_api.remote_command("ping", None)); 336 337 // Create the VM first 338 let cpu_count: u8 = 4; 339 let request_body = guest.api_create_body( 340 cpu_count, 341 direct_kernel_boot_path().to_str().unwrap(), 342 DIRECT_KERNEL_BOOT_CMDLINE, 343 ); 344 345 let temp_config_path = guest.tmp_dir.as_path().join("config"); 346 std::fs::write(&temp_config_path, request_body).unwrap(); 347 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 348 349 assert!(target_api.remote_command("create", Some(create_config))); 350 351 // Then boot it 352 assert!(target_api.remote_command("boot", None)); 353 thread::sleep(std::time::Duration::new(20, 0)); 354 355 let r = std::panic::catch_unwind(|| { 356 // Check that the VM booted as expected 357 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 358 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 359 360 // We now pause the VM 361 assert!(target_api.remote_command("pause", None)); 362 363 // Check pausing again fails 364 assert!(!target_api.remote_command("pause", None)); 365 366 thread::sleep(std::time::Duration::new(2, 0)); 367 368 // SSH into the VM should fail 369 assert!(ssh_command_ip( 370 "grep -c processor /proc/cpuinfo", 371 &guest.network.guest_ip, 372 2, 373 5 374 ) 375 .is_err()); 376 377 // Resume the VM 378 assert!(target_api.remote_command("resume", None)); 379 380 // Check resuming again fails 381 assert!(!target_api.remote_command("resume", None)); 382 383 thread::sleep(std::time::Duration::new(2, 0)); 384 385 // Now we should be able to SSH back in and get the right number of CPUs 386 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 387 }); 388 389 let _ = child.kill(); 390 let output = child.wait_with_output().unwrap(); 391 392 handle_child_output(r, &output); 393 } 394 395 fn _test_pty_interaction(pty_path: PathBuf) { 396 let mut cf = std::fs::OpenOptions::new() 397 .write(true) 398 .read(true) 399 .open(pty_path) 400 .unwrap(); 401 402 // Some dumb sleeps but we don't want to write 403 // before the console is up and we don't want 404 // to try and write the next line before the 405 // login process is ready. 406 thread::sleep(std::time::Duration::new(5, 0)); 407 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 408 thread::sleep(std::time::Duration::new(2, 0)); 409 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 410 thread::sleep(std::time::Duration::new(2, 0)); 411 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 412 thread::sleep(std::time::Duration::new(2, 0)); 413 414 // read pty and ensure they have a login shell 415 // some fairly hacky workarounds to avoid looping 416 // forever in case the channel is blocked getting output 417 let ptyc = pty_read(cf); 418 let mut empty = 0; 419 let mut prev = String::new(); 420 loop { 421 thread::sleep(std::time::Duration::new(2, 0)); 422 match ptyc.try_recv() { 423 Ok(line) => { 424 empty = 0; 425 prev = prev + &line; 426 if prev.contains("test_pty_console") { 427 break; 428 } 429 } 430 Err(mpsc::TryRecvError::Empty) => { 431 empty += 1; 432 assert!(empty <= 5, "No login on pty"); 433 } 434 _ => { 435 panic!("No login on pty") 436 } 437 } 438 } 439 } 440 441 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 442 let mut workload_path = dirs::home_dir().unwrap(); 443 workload_path.push("workloads"); 444 445 let mut virtiofsd_path = workload_path; 446 virtiofsd_path.push("virtiofsd"); 447 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 448 449 let virtiofsd_socket_path = 450 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 451 452 // Start the daemon 453 let child = Command::new(virtiofsd_path.as_str()) 454 .args(["--shared-dir", shared_dir]) 455 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 456 .args(["--cache", "never"]) 457 .spawn() 458 .unwrap(); 459 460 thread::sleep(std::time::Duration::new(10, 0)); 461 462 (child, virtiofsd_socket_path) 463 } 464 465 fn prepare_vubd( 466 tmp_dir: &TempDir, 467 blk_img: &str, 468 num_queues: usize, 469 rdonly: bool, 470 direct: bool, 471 ) -> (std::process::Child, String) { 472 let mut workload_path = dirs::home_dir().unwrap(); 473 workload_path.push("workloads"); 474 475 let mut blk_file_path = workload_path; 476 blk_file_path.push(blk_img); 477 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 478 479 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 480 481 // Start the daemon 482 let child = Command::new(clh_command("vhost_user_block")) 483 .args([ 484 "--block-backend", 485 format!( 486 "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" 487 ) 488 .as_str(), 489 ]) 490 .spawn() 491 .unwrap(); 492 493 thread::sleep(std::time::Duration::new(10, 0)); 494 495 (child, vubd_socket_path) 496 } 497 498 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 499 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 500 } 501 502 fn temp_api_path(tmp_dir: &TempDir) -> String { 503 String::from( 504 tmp_dir 505 .as_path() 506 .join("cloud-hypervisor.sock") 507 .to_str() 508 .unwrap(), 509 ) 510 } 511 512 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 513 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 514 } 515 516 // Creates the directory and returns the path. 517 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 518 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 519 std::fs::create_dir(&snapshot_dir).unwrap(); 520 snapshot_dir 521 } 522 523 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 524 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 525 vmcore_file 526 } 527 528 // Creates the path for direct kernel boot and return the path. 529 // For x86_64, this function returns the vmlinux kernel path. 530 // For AArch64, this function returns the PE kernel path. 531 fn direct_kernel_boot_path() -> PathBuf { 532 let mut workload_path = dirs::home_dir().unwrap(); 533 workload_path.push("workloads"); 534 535 let mut kernel_path = workload_path; 536 #[cfg(target_arch = "x86_64")] 537 kernel_path.push("vmlinux"); 538 #[cfg(target_arch = "aarch64")] 539 kernel_path.push("Image"); 540 541 kernel_path 542 } 543 544 fn edk2_path() -> PathBuf { 545 let mut workload_path = dirs::home_dir().unwrap(); 546 workload_path.push("workloads"); 547 let mut edk2_path = workload_path; 548 edk2_path.push(OVMF_NAME); 549 550 edk2_path 551 } 552 553 fn cloud_hypervisor_release_path() -> String { 554 let mut workload_path = dirs::home_dir().unwrap(); 555 workload_path.push("workloads"); 556 557 let mut ch_release_path = workload_path; 558 #[cfg(target_arch = "x86_64")] 559 ch_release_path.push("cloud-hypervisor-static"); 560 #[cfg(target_arch = "aarch64")] 561 ch_release_path.push("cloud-hypervisor-static-aarch64"); 562 563 ch_release_path.into_os_string().into_string().unwrap() 564 } 565 566 fn prepare_vhost_user_net_daemon( 567 tmp_dir: &TempDir, 568 ip: &str, 569 tap: Option<&str>, 570 mtu: Option<u16>, 571 num_queues: usize, 572 client_mode: bool, 573 ) -> (std::process::Command, String) { 574 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 575 576 // Start the daemon 577 let mut net_params = format!( 578 "ip={ip},mask=255.255.255.0,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" 579 ); 580 581 if let Some(tap) = tap { 582 net_params.push_str(format!(",tap={tap}").as_str()); 583 } 584 585 if let Some(mtu) = mtu { 586 net_params.push_str(format!(",mtu={mtu}").as_str()); 587 } 588 589 let mut command = Command::new(clh_command("vhost_user_net")); 590 command.args(["--net-backend", net_params.as_str()]); 591 592 (command, vunet_socket_path) 593 } 594 595 fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { 596 let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); 597 let swtpm_socket_path = String::from( 598 tmp_dir 599 .as_path() 600 .join("swtpm") 601 .join("swtpm.sock") 602 .to_str() 603 .unwrap(), 604 ); 605 std::fs::create_dir(&swtpm_tpm_dir).unwrap(); 606 607 let mut swtpm_command = Command::new("swtpm"); 608 let swtpm_args = [ 609 "socket", 610 "--tpmstate", 611 &format!("dir={swtpm_tpm_dir}"), 612 "--ctrl", 613 &format!("type=unixio,path={swtpm_socket_path}"), 614 "--flags", 615 "startup-clear", 616 "--tpm2", 617 ]; 618 swtpm_command.args(swtpm_args); 619 620 (swtpm_command, swtpm_socket_path) 621 } 622 623 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 624 let mut cmd = Command::new(clh_command("ch-remote")); 625 cmd.args([&format!("--api-socket={api_socket}"), command]); 626 627 if let Some(arg) = arg { 628 cmd.arg(arg); 629 } 630 let output = cmd.output().unwrap(); 631 if output.status.success() { 632 true 633 } else { 634 eprintln!("Error running ch-remote command: {:?}", &cmd); 635 let stderr = String::from_utf8_lossy(&output.stderr); 636 eprintln!("stderr: {stderr}"); 637 false 638 } 639 } 640 641 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 642 let mut cmd = Command::new(clh_command("ch-remote")); 643 cmd.args([&format!("--api-socket={api_socket}"), command]); 644 645 if let Some(arg) = arg { 646 cmd.arg(arg); 647 } 648 649 let output = cmd.output().expect("Failed to launch ch-remote"); 650 651 (output.status.success(), output.stdout) 652 } 653 654 fn resize_command( 655 api_socket: &str, 656 desired_vcpus: Option<u8>, 657 desired_ram: Option<usize>, 658 desired_balloon: Option<usize>, 659 event_file: Option<&str>, 660 ) -> bool { 661 let mut cmd = Command::new(clh_command("ch-remote")); 662 cmd.args([&format!("--api-socket={api_socket}"), "resize"]); 663 664 if let Some(desired_vcpus) = desired_vcpus { 665 cmd.arg(format!("--cpus={desired_vcpus}")); 666 } 667 668 if let Some(desired_ram) = desired_ram { 669 cmd.arg(format!("--memory={desired_ram}")); 670 } 671 672 if let Some(desired_balloon) = desired_balloon { 673 cmd.arg(format!("--balloon={desired_balloon}")); 674 } 675 676 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 677 678 if let Some(event_path) = event_file { 679 let latest_events = [ 680 &MetaEvent { 681 event: "resizing".to_string(), 682 device_id: None, 683 }, 684 &MetaEvent { 685 event: "resized".to_string(), 686 device_id: None, 687 }, 688 ]; 689 // See: #5938 690 thread::sleep(std::time::Duration::new(1, 0)); 691 assert!(check_latest_events_exact(&latest_events, event_path)); 692 } 693 694 ret 695 } 696 697 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 698 let mut cmd = Command::new(clh_command("ch-remote")); 699 cmd.args([ 700 &format!("--api-socket={api_socket}"), 701 "resize-zone", 702 &format!("--id={id}"), 703 &format!("--size={desired_size}"), 704 ]); 705 706 cmd.status().expect("Failed to launch ch-remote").success() 707 } 708 709 // setup OVS-DPDK bridge and ports 710 fn setup_ovs_dpdk() { 711 // setup OVS-DPDK 712 assert!(exec_host_command_status("service openvswitch-switch start").success()); 713 assert!(exec_host_command_status("ovs-vsctl init").success()); 714 assert!( 715 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 716 .success() 717 ); 718 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 719 720 // Create OVS-DPDK bridge and ports 721 assert!(exec_host_command_status( 722 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 723 ) 724 .success()); 725 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 726 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 727 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 728 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 729 } 730 fn cleanup_ovs_dpdk() { 731 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 732 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 733 } 734 // Setup two guests and ensure they are connected through ovs-dpdk 735 fn setup_ovs_dpdk_guests( 736 guest1: &Guest, 737 guest2: &Guest, 738 api_socket: &str, 739 release_binary: bool, 740 ) -> (Child, Child) { 741 setup_ovs_dpdk(); 742 743 let clh_path = if !release_binary { 744 clh_command("cloud-hypervisor") 745 } else { 746 cloud_hypervisor_release_path() 747 }; 748 749 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 750 .args(["--cpus", "boot=2"]) 751 .args(["--memory", "size=0,shared=on"]) 752 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 753 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 754 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 755 .default_disks() 756 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 757 .capture_output() 758 .spawn() 759 .unwrap(); 760 761 #[cfg(target_arch = "x86_64")] 762 let guest_net_iface = "ens5"; 763 #[cfg(target_arch = "aarch64")] 764 let guest_net_iface = "enp0s5"; 765 766 let r = std::panic::catch_unwind(|| { 767 guest1.wait_vm_boot(None).unwrap(); 768 769 guest1 770 .ssh_command(&format!( 771 "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" 772 )) 773 .unwrap(); 774 guest1 775 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 776 .unwrap(); 777 778 let guest_ip = guest1.network.guest_ip.clone(); 779 thread::spawn(move || { 780 ssh_command_ip( 781 "nc -l 12345", 782 &guest_ip, 783 DEFAULT_SSH_RETRIES, 784 DEFAULT_SSH_TIMEOUT, 785 ) 786 .unwrap(); 787 }); 788 }); 789 if r.is_err() { 790 cleanup_ovs_dpdk(); 791 792 let _ = child1.kill(); 793 let output = child1.wait_with_output().unwrap(); 794 handle_child_output(r, &output); 795 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 796 } 797 798 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 799 .args(["--api-socket", api_socket]) 800 .args(["--cpus", "boot=2"]) 801 .args(["--memory", "size=0,shared=on"]) 802 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 803 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 804 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 805 .default_disks() 806 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 807 .capture_output() 808 .spawn() 809 .unwrap(); 810 811 let r = std::panic::catch_unwind(|| { 812 guest2.wait_vm_boot(None).unwrap(); 813 814 guest2 815 .ssh_command(&format!( 816 "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" 817 )) 818 .unwrap(); 819 guest2 820 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 821 .unwrap(); 822 823 // Check the connection works properly between the two VMs 824 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 825 }); 826 if r.is_err() { 827 cleanup_ovs_dpdk(); 828 829 let _ = child1.kill(); 830 let _ = child2.kill(); 831 let output = child2.wait_with_output().unwrap(); 832 handle_child_output(r, &output); 833 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 834 } 835 836 (child1, child2) 837 } 838 839 enum FwType { 840 Ovmf, 841 RustHypervisorFirmware, 842 } 843 844 fn fw_path(_fw_type: FwType) -> String { 845 let mut workload_path = dirs::home_dir().unwrap(); 846 workload_path.push("workloads"); 847 848 let mut fw_path = workload_path; 849 #[cfg(target_arch = "aarch64")] 850 fw_path.push("CLOUDHV_EFI.fd"); 851 #[cfg(target_arch = "x86_64")] 852 { 853 match _fw_type { 854 FwType::Ovmf => fw_path.push(OVMF_NAME), 855 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 856 } 857 } 858 859 fw_path.to_str().unwrap().to_string() 860 } 861 862 #[derive(Debug)] 863 struct MetaEvent { 864 event: String, 865 device_id: Option<String>, 866 } 867 868 impl MetaEvent { 869 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 870 let mut matched = false; 871 if v["event"].as_str().unwrap() == self.event { 872 if let Some(device_id) = &self.device_id { 873 if v["properties"]["id"].as_str().unwrap() == device_id { 874 matched = true 875 } 876 } else { 877 matched = true; 878 } 879 } 880 matched 881 } 882 } 883 884 // Parse the event_monitor file based on the format that each event 885 // is followed by a double newline 886 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 887 let content = fs::read(event_file).unwrap(); 888 let mut ret = Vec::new(); 889 for entry in String::from_utf8_lossy(&content) 890 .trim() 891 .split("\n\n") 892 .collect::<Vec<&str>>() 893 { 894 ret.push(serde_json::from_str(entry).unwrap()); 895 } 896 897 ret 898 } 899 900 // Return true if all events from the input 'expected_events' are matched sequentially 901 // with events from the 'event_file' 902 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 903 let json_events = parse_event_file(event_file); 904 let len = expected_events.len(); 905 let mut idx = 0; 906 for e in &json_events { 907 if idx == len { 908 break; 909 } 910 if expected_events[idx].match_with_json_event(e) { 911 idx += 1; 912 } 913 } 914 915 let ret = idx == len; 916 917 if !ret { 918 eprintln!( 919 "\n\n==== Start 'check_sequential_events' failed ==== \ 920 \n\nexpected_events={:?}\nactual_events={:?} \ 921 \n\n==== End 'check_sequential_events' failed ====", 922 expected_events, json_events, 923 ); 924 } 925 926 ret 927 } 928 929 // Return true if all events from the input 'expected_events' are matched exactly 930 // with events from the 'event_file' 931 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 932 let json_events = parse_event_file(event_file); 933 assert!(expected_events.len() <= json_events.len()); 934 let json_events = &json_events[..expected_events.len()]; 935 936 for (idx, e) in json_events.iter().enumerate() { 937 if !expected_events[idx].match_with_json_event(e) { 938 eprintln!( 939 "\n\n==== Start 'check_sequential_events_exact' failed ==== \ 940 \n\nexpected_events={:?}\nactual_events={:?} \ 941 \n\n==== End 'check_sequential_events_exact' failed ====", 942 expected_events, json_events, 943 ); 944 945 return false; 946 } 947 } 948 949 true 950 } 951 952 // Return true if events from the input 'latest_events' are matched exactly 953 // with the most recent events from the 'event_file' 954 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 955 let json_events = parse_event_file(event_file); 956 assert!(latest_events.len() <= json_events.len()); 957 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 958 959 for (idx, e) in json_events.iter().enumerate() { 960 if !latest_events[idx].match_with_json_event(e) { 961 eprintln!( 962 "\n\n==== Start 'check_latest_events_exact' failed ==== \ 963 \n\nexpected_events={:?}\nactual_events={:?} \ 964 \n\n==== End 'check_latest_events_exact' failed ====", 965 latest_events, json_events, 966 ); 967 968 return false; 969 } 970 } 971 972 true 973 } 974 975 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 976 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 977 let guest = Guest::new(Box::new(focal)); 978 let total_vcpus = threads_per_core * cores_per_package * packages; 979 let direct_kernel_boot_path = direct_kernel_boot_path(); 980 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 981 let fw_path = fw_path(FwType::RustHypervisorFirmware); 982 if use_fw { 983 kernel_path = fw_path.as_str(); 984 } 985 986 let mut child = GuestCommand::new(&guest) 987 .args([ 988 "--cpus", 989 &format!( 990 "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" 991 ), 992 ]) 993 .args(["--memory", "size=512M"]) 994 .args(["--kernel", kernel_path]) 995 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 996 .default_disks() 997 .default_net() 998 .capture_output() 999 .spawn() 1000 .unwrap(); 1001 1002 let r = std::panic::catch_unwind(|| { 1003 guest.wait_vm_boot(None).unwrap(); 1004 assert_eq!( 1005 guest.get_cpu_count().unwrap_or_default(), 1006 u32::from(total_vcpus) 1007 ); 1008 assert_eq!( 1009 guest 1010 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1011 .unwrap() 1012 .trim() 1013 .parse::<u8>() 1014 .unwrap_or(0), 1015 threads_per_core 1016 ); 1017 1018 assert_eq!( 1019 guest 1020 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1021 .unwrap() 1022 .trim() 1023 .parse::<u8>() 1024 .unwrap_or(0), 1025 cores_per_package 1026 ); 1027 1028 assert_eq!( 1029 guest 1030 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1031 .unwrap() 1032 .trim() 1033 .parse::<u8>() 1034 .unwrap_or(0), 1035 packages 1036 ); 1037 1038 #[cfg(target_arch = "x86_64")] 1039 { 1040 let mut cpu_id = 0; 1041 for package_id in 0..packages { 1042 for core_id in 0..cores_per_package { 1043 for _ in 0..threads_per_core { 1044 assert_eq!( 1045 guest 1046 .ssh_command(&format!("cat /sys/devices/system/cpu/cpu{cpu_id}/topology/physical_package_id")) 1047 .unwrap() 1048 .trim() 1049 .parse::<u8>() 1050 .unwrap_or(0), 1051 package_id 1052 ); 1053 1054 assert_eq!( 1055 guest 1056 .ssh_command(&format!( 1057 "cat /sys/devices/system/cpu/cpu{cpu_id}/topology/core_id" 1058 )) 1059 .unwrap() 1060 .trim() 1061 .parse::<u8>() 1062 .unwrap_or(0), 1063 core_id 1064 ); 1065 1066 cpu_id += 1; 1067 } 1068 } 1069 } 1070 } 1071 }); 1072 1073 let _ = child.kill(); 1074 let output = child.wait_with_output().unwrap(); 1075 1076 handle_child_output(r, &output); 1077 } 1078 1079 #[allow(unused_variables)] 1080 fn _test_guest_numa_nodes(acpi: bool) { 1081 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1082 let guest = Guest::new(Box::new(focal)); 1083 let api_socket = temp_api_path(&guest.tmp_dir); 1084 #[cfg(target_arch = "x86_64")] 1085 let kernel_path = direct_kernel_boot_path(); 1086 #[cfg(target_arch = "aarch64")] 1087 let kernel_path = if acpi { 1088 edk2_path() 1089 } else { 1090 direct_kernel_boot_path() 1091 }; 1092 1093 let mut child = GuestCommand::new(&guest) 1094 .args(["--cpus", "boot=6,max=12"]) 1095 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 1096 .args([ 1097 "--memory-zone", 1098 "id=mem0,size=1G,hotplug_size=3G", 1099 "id=mem1,size=2G,hotplug_size=3G", 1100 "id=mem2,size=3G,hotplug_size=3G", 1101 ]) 1102 .args([ 1103 "--numa", 1104 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 1105 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 1106 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 1107 ]) 1108 .args(["--kernel", kernel_path.to_str().unwrap()]) 1109 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1110 .args(["--api-socket", &api_socket]) 1111 .capture_output() 1112 .default_disks() 1113 .default_net() 1114 .spawn() 1115 .unwrap(); 1116 1117 let r = std::panic::catch_unwind(|| { 1118 guest.wait_vm_boot(None).unwrap(); 1119 1120 guest.check_numa_common( 1121 Some(&[960_000, 1_920_000, 2_880_000]), 1122 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 1123 Some(&["10 15 20", "20 10 25", "25 30 10"]), 1124 ); 1125 1126 // AArch64 currently does not support hotplug, and therefore we only 1127 // test hotplug-related function on x86_64 here. 1128 #[cfg(target_arch = "x86_64")] 1129 { 1130 guest.enable_memory_hotplug(); 1131 1132 // Resize every memory zone and check each associated NUMA node 1133 // has been assigned the right amount of memory. 1134 resize_zone_command(&api_socket, "mem0", "4G"); 1135 resize_zone_command(&api_socket, "mem1", "4G"); 1136 resize_zone_command(&api_socket, "mem2", "4G"); 1137 // Resize to the maximum amount of CPUs and check each NUMA 1138 // node has been assigned the right CPUs set. 1139 resize_command(&api_socket, Some(12), None, None, None); 1140 thread::sleep(std::time::Duration::new(5, 0)); 1141 1142 guest.check_numa_common( 1143 Some(&[3_840_000, 3_840_000, 3_840_000]), 1144 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 1145 None, 1146 ); 1147 } 1148 }); 1149 1150 let _ = child.kill(); 1151 let output = child.wait_with_output().unwrap(); 1152 1153 handle_child_output(r, &output); 1154 } 1155 1156 #[allow(unused_variables)] 1157 fn _test_power_button(acpi: bool) { 1158 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1159 let guest = Guest::new(Box::new(focal)); 1160 let mut cmd = GuestCommand::new(&guest); 1161 let api_socket = temp_api_path(&guest.tmp_dir); 1162 1163 #[cfg(target_arch = "x86_64")] 1164 let kernel_path = direct_kernel_boot_path(); 1165 #[cfg(target_arch = "aarch64")] 1166 let kernel_path = if acpi { 1167 edk2_path() 1168 } else { 1169 direct_kernel_boot_path() 1170 }; 1171 1172 cmd.args(["--cpus", "boot=1"]) 1173 .args(["--memory", "size=512M"]) 1174 .args(["--kernel", kernel_path.to_str().unwrap()]) 1175 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1176 .capture_output() 1177 .default_disks() 1178 .default_net() 1179 .args(["--api-socket", &api_socket]); 1180 1181 let child = cmd.spawn().unwrap(); 1182 1183 let r = std::panic::catch_unwind(|| { 1184 guest.wait_vm_boot(None).unwrap(); 1185 assert!(remote_command(&api_socket, "power-button", None)); 1186 }); 1187 1188 let output = child.wait_with_output().unwrap(); 1189 assert!(output.status.success()); 1190 handle_child_output(r, &output); 1191 } 1192 1193 type PrepareNetDaemon = dyn Fn( 1194 &TempDir, 1195 &str, 1196 Option<&str>, 1197 Option<u16>, 1198 usize, 1199 bool, 1200 ) -> (std::process::Command, String); 1201 1202 fn test_vhost_user_net( 1203 tap: Option<&str>, 1204 num_queues: usize, 1205 prepare_daemon: &PrepareNetDaemon, 1206 generate_host_mac: bool, 1207 client_mode_daemon: bool, 1208 ) { 1209 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1210 let guest = Guest::new(Box::new(focal)); 1211 let api_socket = temp_api_path(&guest.tmp_dir); 1212 1213 let kernel_path = direct_kernel_boot_path(); 1214 1215 let host_mac = if generate_host_mac { 1216 Some(MacAddr::local_random()) 1217 } else { 1218 None 1219 }; 1220 1221 let mtu = Some(3000); 1222 1223 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 1224 &guest.tmp_dir, 1225 &guest.network.host_ip, 1226 tap, 1227 mtu, 1228 num_queues, 1229 client_mode_daemon, 1230 ); 1231 1232 let net_params = format!( 1233 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 1234 guest.network.guest_mac, 1235 vunet_socket_path, 1236 num_queues, 1237 if let Some(host_mac) = host_mac { 1238 format!(",host_mac={host_mac}") 1239 } else { 1240 "".to_owned() 1241 }, 1242 if client_mode_daemon { 1243 "server" 1244 } else { 1245 "client" 1246 }, 1247 ); 1248 1249 let mut ch_command = GuestCommand::new(&guest); 1250 ch_command 1251 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 1252 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1253 .args(["--kernel", kernel_path.to_str().unwrap()]) 1254 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1255 .default_disks() 1256 .args(["--net", net_params.as_str()]) 1257 .args(["--api-socket", &api_socket]) 1258 .capture_output(); 1259 1260 let mut daemon_child: std::process::Child; 1261 let mut child: std::process::Child; 1262 1263 if client_mode_daemon { 1264 child = ch_command.spawn().unwrap(); 1265 // Make sure the VMM is waiting for the backend to connect 1266 thread::sleep(std::time::Duration::new(10, 0)); 1267 daemon_child = daemon_command.spawn().unwrap(); 1268 } else { 1269 daemon_child = daemon_command.spawn().unwrap(); 1270 // Make sure the backend is waiting for the VMM to connect 1271 thread::sleep(std::time::Duration::new(10, 0)); 1272 child = ch_command.spawn().unwrap(); 1273 } 1274 1275 let r = std::panic::catch_unwind(|| { 1276 guest.wait_vm_boot(None).unwrap(); 1277 1278 if let Some(tap_name) = tap { 1279 let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); 1280 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1281 } 1282 1283 if let Some(host_mac) = tap { 1284 let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); 1285 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1286 } 1287 1288 #[cfg(target_arch = "aarch64")] 1289 let iface = "enp0s4"; 1290 #[cfg(target_arch = "x86_64")] 1291 let iface = "ens4"; 1292 1293 assert_eq!( 1294 guest 1295 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 1296 .unwrap() 1297 .trim(), 1298 "3000" 1299 ); 1300 1301 // 1 network interface + default localhost ==> 2 interfaces 1302 // It's important to note that this test is fully exercising the 1303 // vhost-user-net implementation and the associated backend since 1304 // it does not define any --net network interface. That means all 1305 // the ssh communication in that test happens through the network 1306 // interface backed by vhost-user-net. 1307 assert_eq!( 1308 guest 1309 .ssh_command("ip -o link | wc -l") 1310 .unwrap() 1311 .trim() 1312 .parse::<u32>() 1313 .unwrap_or_default(), 1314 2 1315 ); 1316 1317 // The following pci devices will appear on guest with PCI-MSI 1318 // interrupt vectors assigned. 1319 // 1 virtio-console with 3 vectors: config, Rx, Tx 1320 // 1 virtio-blk with 2 vectors: config, Request 1321 // 1 virtio-blk with 2 vectors: config, Request 1322 // 1 virtio-rng with 2 vectors: config, Request 1323 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1324 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1325 // Based on the above, the total vectors should 14. 1326 #[cfg(target_arch = "x86_64")] 1327 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1328 #[cfg(target_arch = "aarch64")] 1329 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1330 assert_eq!( 1331 guest 1332 .ssh_command(grep_cmd) 1333 .unwrap() 1334 .trim() 1335 .parse::<u32>() 1336 .unwrap_or_default(), 1337 10 + (num_queues as u32) 1338 ); 1339 1340 // ACPI feature is needed. 1341 #[cfg(target_arch = "x86_64")] 1342 { 1343 guest.enable_memory_hotplug(); 1344 1345 // Add RAM to the VM 1346 let desired_ram = 1024 << 20; 1347 resize_command(&api_socket, None, Some(desired_ram), None, None); 1348 1349 thread::sleep(std::time::Duration::new(10, 0)); 1350 1351 // Here by simply checking the size (through ssh), we validate 1352 // the connection is still working, which means vhost-user-net 1353 // keeps working after the resize. 1354 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1355 } 1356 }); 1357 1358 let _ = child.kill(); 1359 let output = child.wait_with_output().unwrap(); 1360 1361 thread::sleep(std::time::Duration::new(5, 0)); 1362 let _ = daemon_child.kill(); 1363 let _ = daemon_child.wait(); 1364 1365 handle_child_output(r, &output); 1366 } 1367 1368 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1369 1370 fn test_vhost_user_blk( 1371 num_queues: usize, 1372 readonly: bool, 1373 direct: bool, 1374 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1375 ) { 1376 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1377 let guest = Guest::new(Box::new(focal)); 1378 let api_socket = temp_api_path(&guest.tmp_dir); 1379 1380 let kernel_path = direct_kernel_boot_path(); 1381 1382 let (blk_params, daemon_child) = { 1383 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1384 // Start the daemon 1385 let (daemon_child, vubd_socket_path) = 1386 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1387 1388 ( 1389 format!( 1390 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1391 ), 1392 Some(daemon_child), 1393 ) 1394 }; 1395 1396 let mut child = GuestCommand::new(&guest) 1397 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1398 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1399 .args(["--kernel", kernel_path.to_str().unwrap()]) 1400 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1401 .args([ 1402 "--disk", 1403 format!( 1404 "path={}", 1405 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1406 ) 1407 .as_str(), 1408 format!( 1409 "path={}", 1410 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1411 ) 1412 .as_str(), 1413 blk_params.as_str(), 1414 ]) 1415 .default_net() 1416 .args(["--api-socket", &api_socket]) 1417 .capture_output() 1418 .spawn() 1419 .unwrap(); 1420 1421 let r = std::panic::catch_unwind(|| { 1422 guest.wait_vm_boot(None).unwrap(); 1423 1424 // Check both if /dev/vdc exists and if the block size is 16M. 1425 assert_eq!( 1426 guest 1427 .ssh_command("lsblk | grep vdc | grep -c 16M") 1428 .unwrap() 1429 .trim() 1430 .parse::<u32>() 1431 .unwrap_or_default(), 1432 1 1433 ); 1434 1435 // Check if this block is RO or RW. 1436 assert_eq!( 1437 guest 1438 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1439 .unwrap() 1440 .trim() 1441 .parse::<u32>() 1442 .unwrap_or_default(), 1443 readonly as u32 1444 ); 1445 1446 // Check if the number of queues in /sys/block/vdc/mq matches the 1447 // expected num_queues. 1448 assert_eq!( 1449 guest 1450 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1451 .unwrap() 1452 .trim() 1453 .parse::<u32>() 1454 .unwrap_or_default(), 1455 num_queues as u32 1456 ); 1457 1458 // Mount the device 1459 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1460 guest.ssh_command("mkdir mount_image").unwrap(); 1461 guest 1462 .ssh_command( 1463 format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), 1464 ) 1465 .unwrap(); 1466 1467 // Check the content of the block device. The file "foo" should 1468 // contain "bar". 1469 assert_eq!( 1470 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1471 "bar" 1472 ); 1473 1474 // ACPI feature is needed. 1475 #[cfg(target_arch = "x86_64")] 1476 { 1477 guest.enable_memory_hotplug(); 1478 1479 // Add RAM to the VM 1480 let desired_ram = 1024 << 20; 1481 resize_command(&api_socket, None, Some(desired_ram), None, None); 1482 1483 thread::sleep(std::time::Duration::new(10, 0)); 1484 1485 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1486 1487 // Check again the content of the block device after the resize 1488 // has been performed. 1489 assert_eq!( 1490 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1491 "bar" 1492 ); 1493 } 1494 1495 // Unmount the device 1496 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1497 guest.ssh_command("rm -r mount_image").unwrap(); 1498 }); 1499 1500 let _ = child.kill(); 1501 let output = child.wait_with_output().unwrap(); 1502 1503 if let Some(mut daemon_child) = daemon_child { 1504 thread::sleep(std::time::Duration::new(5, 0)); 1505 let _ = daemon_child.kill(); 1506 let _ = daemon_child.wait(); 1507 } 1508 1509 handle_child_output(r, &output); 1510 } 1511 1512 fn test_boot_from_vhost_user_blk( 1513 num_queues: usize, 1514 readonly: bool, 1515 direct: bool, 1516 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1517 ) { 1518 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1519 let guest = Guest::new(Box::new(focal)); 1520 1521 let kernel_path = direct_kernel_boot_path(); 1522 1523 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1524 1525 let (blk_boot_params, daemon_child) = { 1526 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1527 // Start the daemon 1528 let (daemon_child, vubd_socket_path) = prepare_daemon( 1529 &guest.tmp_dir, 1530 disk_path.as_str(), 1531 num_queues, 1532 readonly, 1533 direct, 1534 ); 1535 1536 ( 1537 format!( 1538 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1539 ), 1540 Some(daemon_child), 1541 ) 1542 }; 1543 1544 let mut child = GuestCommand::new(&guest) 1545 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1546 .args(["--memory", "size=512M,shared=on"]) 1547 .args(["--kernel", kernel_path.to_str().unwrap()]) 1548 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1549 .args([ 1550 "--disk", 1551 blk_boot_params.as_str(), 1552 format!( 1553 "path={}", 1554 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1555 ) 1556 .as_str(), 1557 ]) 1558 .default_net() 1559 .capture_output() 1560 .spawn() 1561 .unwrap(); 1562 1563 let r = std::panic::catch_unwind(|| { 1564 guest.wait_vm_boot(None).unwrap(); 1565 1566 // Just check the VM booted correctly. 1567 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1568 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1569 }); 1570 let _ = child.kill(); 1571 let output = child.wait_with_output().unwrap(); 1572 1573 if let Some(mut daemon_child) = daemon_child { 1574 thread::sleep(std::time::Duration::new(5, 0)); 1575 let _ = daemon_child.kill(); 1576 let _ = daemon_child.wait(); 1577 } 1578 1579 handle_child_output(r, &output); 1580 } 1581 1582 fn _test_virtio_fs( 1583 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1584 hotplug: bool, 1585 pci_segment: Option<u16>, 1586 ) { 1587 #[cfg(target_arch = "aarch64")] 1588 let focal_image = if hotplug { 1589 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1590 } else { 1591 FOCAL_IMAGE_NAME.to_string() 1592 }; 1593 #[cfg(target_arch = "x86_64")] 1594 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1595 let focal = UbuntuDiskConfig::new(focal_image); 1596 let guest = Guest::new(Box::new(focal)); 1597 let api_socket = temp_api_path(&guest.tmp_dir); 1598 1599 let mut workload_path = dirs::home_dir().unwrap(); 1600 workload_path.push("workloads"); 1601 1602 let mut shared_dir = workload_path; 1603 shared_dir.push("shared_dir"); 1604 1605 #[cfg(target_arch = "x86_64")] 1606 let kernel_path = direct_kernel_boot_path(); 1607 #[cfg(target_arch = "aarch64")] 1608 let kernel_path = if hotplug { 1609 edk2_path() 1610 } else { 1611 direct_kernel_boot_path() 1612 }; 1613 1614 let (mut daemon_child, virtiofsd_socket_path) = 1615 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1616 1617 let mut guest_command = GuestCommand::new(&guest); 1618 guest_command 1619 .args(["--cpus", "boot=1"]) 1620 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1621 .args(["--kernel", kernel_path.to_str().unwrap()]) 1622 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1623 .default_disks() 1624 .default_net() 1625 .args(["--api-socket", &api_socket]); 1626 if pci_segment.is_some() { 1627 guest_command.args([ 1628 "--platform", 1629 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 1630 ]); 1631 } 1632 1633 let fs_params = format!( 1634 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1635 virtiofsd_socket_path, 1636 if let Some(pci_segment) = pci_segment { 1637 format!(",pci_segment={pci_segment}") 1638 } else { 1639 "".to_owned() 1640 } 1641 ); 1642 1643 if !hotplug { 1644 guest_command.args(["--fs", fs_params.as_str()]); 1645 } 1646 1647 let mut child = guest_command.capture_output().spawn().unwrap(); 1648 1649 let r = std::panic::catch_unwind(|| { 1650 guest.wait_vm_boot(None).unwrap(); 1651 1652 if hotplug { 1653 // Add fs to the VM 1654 let (cmd_success, cmd_output) = 1655 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1656 assert!(cmd_success); 1657 1658 if let Some(pci_segment) = pci_segment { 1659 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1660 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1661 ))); 1662 } else { 1663 assert!(String::from_utf8_lossy(&cmd_output) 1664 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1665 } 1666 1667 thread::sleep(std::time::Duration::new(10, 0)); 1668 } 1669 1670 // Mount shared directory through virtio_fs filesystem 1671 guest 1672 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1673 .unwrap(); 1674 1675 // Check file1 exists and its content is "foo" 1676 assert_eq!( 1677 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1678 "foo" 1679 ); 1680 // Check file2 does not exist 1681 guest 1682 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1683 .unwrap(); 1684 1685 // Check file3 exists and its content is "bar" 1686 assert_eq!( 1687 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1688 "bar" 1689 ); 1690 1691 // ACPI feature is needed. 1692 #[cfg(target_arch = "x86_64")] 1693 { 1694 guest.enable_memory_hotplug(); 1695 1696 // Add RAM to the VM 1697 let desired_ram = 1024 << 20; 1698 resize_command(&api_socket, None, Some(desired_ram), None, None); 1699 1700 thread::sleep(std::time::Duration::new(30, 0)); 1701 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1702 1703 // After the resize, check again that file1 exists and its 1704 // content is "foo". 1705 assert_eq!( 1706 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1707 "foo" 1708 ); 1709 } 1710 1711 if hotplug { 1712 // Remove from VM 1713 guest.ssh_command("sudo umount mount_dir").unwrap(); 1714 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1715 } 1716 }); 1717 1718 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1719 thread::sleep(std::time::Duration::new(10, 0)); 1720 let (daemon_child, virtiofsd_socket_path) = 1721 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1722 1723 let r = std::panic::catch_unwind(|| { 1724 thread::sleep(std::time::Duration::new(10, 0)); 1725 let fs_params = format!( 1726 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1727 virtiofsd_socket_path, 1728 if let Some(pci_segment) = pci_segment { 1729 format!(",pci_segment={pci_segment}") 1730 } else { 1731 "".to_owned() 1732 } 1733 ); 1734 1735 // Add back and check it works 1736 let (cmd_success, cmd_output) = 1737 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1738 assert!(cmd_success); 1739 if let Some(pci_segment) = pci_segment { 1740 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1741 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1742 ))); 1743 } else { 1744 assert!(String::from_utf8_lossy(&cmd_output) 1745 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1746 } 1747 1748 thread::sleep(std::time::Duration::new(10, 0)); 1749 // Mount shared directory through virtio_fs filesystem 1750 guest 1751 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1752 .unwrap(); 1753 1754 // Check file1 exists and its content is "foo" 1755 assert_eq!( 1756 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1757 "foo" 1758 ); 1759 }); 1760 1761 (r, Some(daemon_child)) 1762 } else { 1763 (r, None) 1764 }; 1765 1766 let _ = child.kill(); 1767 let output = child.wait_with_output().unwrap(); 1768 1769 let _ = daemon_child.kill(); 1770 let _ = daemon_child.wait(); 1771 1772 if let Some(mut daemon_child) = hotplug_daemon_child { 1773 let _ = daemon_child.kill(); 1774 let _ = daemon_child.wait(); 1775 } 1776 1777 handle_child_output(r, &output); 1778 } 1779 1780 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1781 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1782 let guest = Guest::new(Box::new(focal)); 1783 1784 let kernel_path = direct_kernel_boot_path(); 1785 1786 let pmem_temp_file = TempFile::new().unwrap(); 1787 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1788 1789 std::process::Command::new("mkfs.ext4") 1790 .arg(pmem_temp_file.as_path()) 1791 .output() 1792 .expect("Expect creating disk image to succeed"); 1793 1794 let mut child = GuestCommand::new(&guest) 1795 .args(["--cpus", "boot=1"]) 1796 .args(["--memory", "size=512M"]) 1797 .args(["--kernel", kernel_path.to_str().unwrap()]) 1798 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1799 .default_disks() 1800 .default_net() 1801 .args([ 1802 "--pmem", 1803 format!( 1804 "file={}{}{}", 1805 pmem_temp_file.as_path().to_str().unwrap(), 1806 if specify_size { ",size=128M" } else { "" }, 1807 if discard_writes { 1808 ",discard_writes=on" 1809 } else { 1810 "" 1811 } 1812 ) 1813 .as_str(), 1814 ]) 1815 .capture_output() 1816 .spawn() 1817 .unwrap(); 1818 1819 let r = std::panic::catch_unwind(|| { 1820 guest.wait_vm_boot(None).unwrap(); 1821 1822 // Check for the presence of /dev/pmem0 1823 assert_eq!( 1824 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1825 "/dev/pmem0" 1826 ); 1827 1828 // Check changes persist after reboot 1829 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1830 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1831 guest 1832 .ssh_command("echo test123 | sudo tee /mnt/test") 1833 .unwrap(); 1834 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1835 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1836 1837 guest.reboot_linux(0, None); 1838 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1839 assert_eq!( 1840 guest 1841 .ssh_command("sudo cat /mnt/test || true") 1842 .unwrap() 1843 .trim(), 1844 if discard_writes { "" } else { "test123" } 1845 ); 1846 }); 1847 1848 let _ = child.kill(); 1849 let output = child.wait_with_output().unwrap(); 1850 1851 handle_child_output(r, &output); 1852 } 1853 1854 fn get_fd_count(pid: u32) -> usize { 1855 fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() 1856 } 1857 1858 fn _test_virtio_vsock(hotplug: bool) { 1859 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1860 let guest = Guest::new(Box::new(focal)); 1861 1862 #[cfg(target_arch = "x86_64")] 1863 let kernel_path = direct_kernel_boot_path(); 1864 #[cfg(target_arch = "aarch64")] 1865 let kernel_path = if hotplug { 1866 edk2_path() 1867 } else { 1868 direct_kernel_boot_path() 1869 }; 1870 1871 let socket = temp_vsock_path(&guest.tmp_dir); 1872 let api_socket = temp_api_path(&guest.tmp_dir); 1873 1874 let mut cmd = GuestCommand::new(&guest); 1875 cmd.args(["--api-socket", &api_socket]); 1876 cmd.args(["--cpus", "boot=1"]); 1877 cmd.args(["--memory", "size=512M"]); 1878 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1879 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1880 cmd.default_disks(); 1881 cmd.default_net(); 1882 1883 if !hotplug { 1884 cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); 1885 } 1886 1887 let mut child = cmd.capture_output().spawn().unwrap(); 1888 1889 let r = std::panic::catch_unwind(|| { 1890 guest.wait_vm_boot(None).unwrap(); 1891 1892 if hotplug { 1893 let (cmd_success, cmd_output) = remote_command_w_output( 1894 &api_socket, 1895 "add-vsock", 1896 Some(format!("cid=3,socket={socket},id=test0").as_str()), 1897 ); 1898 assert!(cmd_success); 1899 assert!(String::from_utf8_lossy(&cmd_output) 1900 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1901 thread::sleep(std::time::Duration::new(10, 0)); 1902 // Check adding a second one fails 1903 assert!(!remote_command( 1904 &api_socket, 1905 "add-vsock", 1906 Some("cid=1234,socket=/tmp/fail") 1907 )); 1908 } 1909 1910 // Validate vsock works as expected. 1911 guest.check_vsock(socket.as_str()); 1912 guest.reboot_linux(0, None); 1913 // Validate vsock still works after a reboot. 1914 guest.check_vsock(socket.as_str()); 1915 1916 if hotplug { 1917 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1918 } 1919 }); 1920 1921 let _ = child.kill(); 1922 let output = child.wait_with_output().unwrap(); 1923 1924 handle_child_output(r, &output); 1925 } 1926 1927 fn get_ksm_pages_shared() -> u32 { 1928 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1929 .unwrap() 1930 .trim() 1931 .parse::<u32>() 1932 .unwrap() 1933 } 1934 1935 fn test_memory_mergeable(mergeable: bool) { 1936 let memory_param = if mergeable { 1937 "mergeable=on" 1938 } else { 1939 "mergeable=off" 1940 }; 1941 1942 // We are assuming the rest of the system in our CI is not using mergeable memory 1943 let ksm_ps_init = get_ksm_pages_shared(); 1944 assert!(ksm_ps_init == 0); 1945 1946 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1947 let guest1 = Guest::new(Box::new(focal1)); 1948 let mut child1 = GuestCommand::new(&guest1) 1949 .args(["--cpus", "boot=1"]) 1950 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1951 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1952 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1953 .default_disks() 1954 .args(["--net", guest1.default_net_string().as_str()]) 1955 .args(["--serial", "tty", "--console", "off"]) 1956 .capture_output() 1957 .spawn() 1958 .unwrap(); 1959 1960 let r = std::panic::catch_unwind(|| { 1961 guest1.wait_vm_boot(None).unwrap(); 1962 }); 1963 if r.is_err() { 1964 let _ = child1.kill(); 1965 let output = child1.wait_with_output().unwrap(); 1966 handle_child_output(r, &output); 1967 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1968 } 1969 1970 let ksm_ps_guest1 = get_ksm_pages_shared(); 1971 1972 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1973 let guest2 = Guest::new(Box::new(focal2)); 1974 let mut child2 = GuestCommand::new(&guest2) 1975 .args(["--cpus", "boot=1"]) 1976 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1977 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1978 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1979 .default_disks() 1980 .args(["--net", guest2.default_net_string().as_str()]) 1981 .args(["--serial", "tty", "--console", "off"]) 1982 .capture_output() 1983 .spawn() 1984 .unwrap(); 1985 1986 let r = std::panic::catch_unwind(|| { 1987 guest2.wait_vm_boot(None).unwrap(); 1988 let ksm_ps_guest2 = get_ksm_pages_shared(); 1989 1990 if mergeable { 1991 println!( 1992 "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" 1993 ); 1994 // We are expecting the number of shared pages to increase as the number of VM increases 1995 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1996 } else { 1997 assert!(ksm_ps_guest1 == 0); 1998 assert!(ksm_ps_guest2 == 0); 1999 } 2000 }); 2001 2002 let _ = child1.kill(); 2003 let _ = child2.kill(); 2004 2005 let output = child1.wait_with_output().unwrap(); 2006 child2.wait().unwrap(); 2007 2008 handle_child_output(r, &output); 2009 } 2010 2011 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 2012 let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); 2013 let reader = io::BufReader::new(smaps); 2014 2015 let mut skip_map: bool = false; 2016 let mut region_name: String = "".to_string(); 2017 let mut region_maps = HashMap::new(); 2018 for line in reader.lines() { 2019 let l = line.unwrap(); 2020 2021 if l.contains('-') { 2022 let values: Vec<&str> = l.split_whitespace().collect(); 2023 region_name = values.last().unwrap().trim().to_string(); 2024 if region_name == "0" { 2025 region_name = "anonymous".to_string() 2026 } 2027 } 2028 2029 // Each section begins with something that looks like: 2030 // Size: 2184 kB 2031 if l.starts_with("Size:") { 2032 let values: Vec<&str> = l.split_whitespace().collect(); 2033 let map_size = values[1].parse::<u32>().unwrap(); 2034 // We skip the assigned guest RAM map, its RSS is only 2035 // dependent on the guest actual memory usage. 2036 // Everything else can be added to the VMM overhead. 2037 skip_map = map_size >= guest_memory_size; 2038 continue; 2039 } 2040 2041 // If this is a map we're taking into account, then we only 2042 // count the RSS. The sum of all counted RSS is the VMM overhead. 2043 if !skip_map && l.starts_with("Rss:") { 2044 let values: Vec<&str> = l.split_whitespace().collect(); 2045 let value = values[1].trim().parse::<u32>().unwrap(); 2046 *region_maps.entry(region_name.clone()).or_insert(0) += value; 2047 } 2048 } 2049 2050 region_maps 2051 } 2052 2053 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 2054 let mut total = 0; 2055 2056 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 2057 eprintln!("{region_name}: {value}"); 2058 total += value; 2059 } 2060 2061 total 2062 } 2063 2064 fn process_rss_kib(pid: u32) -> usize { 2065 let command = format!("ps -q {pid} -o rss="); 2066 let rss = exec_host_command_output(&command); 2067 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 2068 } 2069 2070 // 10MB is our maximum accepted overhead. 2071 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 2072 2073 #[derive(PartialEq, Eq, PartialOrd)] 2074 struct Counters { 2075 rx_bytes: u64, 2076 rx_frames: u64, 2077 tx_bytes: u64, 2078 tx_frames: u64, 2079 read_bytes: u64, 2080 write_bytes: u64, 2081 read_ops: u64, 2082 write_ops: u64, 2083 } 2084 2085 fn get_counters(api_socket: &str) -> Counters { 2086 // Get counters 2087 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 2088 assert!(cmd_success); 2089 2090 let counters: HashMap<&str, HashMap<&str, u64>> = 2091 serde_json::from_slice(&cmd_output).unwrap_or_default(); 2092 2093 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 2094 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 2095 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 2096 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 2097 2098 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 2099 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 2100 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 2101 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 2102 2103 Counters { 2104 rx_bytes, 2105 rx_frames, 2106 tx_bytes, 2107 tx_frames, 2108 read_bytes, 2109 write_bytes, 2110 read_ops, 2111 write_ops, 2112 } 2113 } 2114 2115 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 2116 let (tx, rx) = mpsc::channel::<String>(); 2117 thread::spawn(move || loop { 2118 thread::sleep(std::time::Duration::new(1, 0)); 2119 let mut buf = [0; 512]; 2120 match pty.read(&mut buf) { 2121 Ok(_bytes) => { 2122 let output = std::str::from_utf8(&buf).unwrap().to_string(); 2123 match tx.send(output) { 2124 Ok(_) => (), 2125 Err(_) => break, 2126 } 2127 } 2128 Err(_) => break, 2129 } 2130 }); 2131 rx 2132 } 2133 2134 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 2135 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2136 assert!(cmd_success); 2137 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2138 assert_eq!("Pty", info["config"][pty_type]["mode"]); 2139 PathBuf::from( 2140 info["config"][pty_type]["file"] 2141 .as_str() 2142 .expect("Missing pty path"), 2143 ) 2144 } 2145 2146 // VFIO test network setup. 2147 // We reserve a different IP class for it: 172.18.0.0/24. 2148 #[cfg(target_arch = "x86_64")] 2149 fn setup_vfio_network_interfaces() { 2150 // 'vfio-br0' 2151 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 2152 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 2153 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 2154 // 'vfio-tap0' 2155 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 2156 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 2157 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 2158 // 'vfio-tap1' 2159 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 2160 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 2161 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 2162 // 'vfio-tap2' 2163 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 2164 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 2165 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 2166 // 'vfio-tap3' 2167 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 2168 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 2169 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 2170 } 2171 2172 // Tear VFIO test network down 2173 #[cfg(target_arch = "x86_64")] 2174 fn cleanup_vfio_network_interfaces() { 2175 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 2176 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 2177 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 2178 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 2179 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 2180 } 2181 2182 fn balloon_size(api_socket: &str) -> u64 { 2183 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2184 assert!(cmd_success); 2185 2186 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2187 let total_mem = &info["config"]["memory"]["size"] 2188 .to_string() 2189 .parse::<u64>() 2190 .unwrap(); 2191 let actual_mem = &info["memory_actual_size"] 2192 .to_string() 2193 .parse::<u64>() 2194 .unwrap(); 2195 total_mem - actual_mem 2196 } 2197 2198 fn vm_state(api_socket: &str) -> String { 2199 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2200 assert!(cmd_success); 2201 2202 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2203 let state = &info["state"].as_str().unwrap(); 2204 2205 state.to_string() 2206 } 2207 2208 // This test validates that it can find the virtio-iommu device at first. 2209 // It also verifies that both disks and the network card are attached to 2210 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 2211 // The last interesting part of this test is that it exercises the network 2212 // interface attached to the virtual IOMMU since this is the one used to 2213 // send all commands through SSH. 2214 fn _test_virtio_iommu(acpi: bool) { 2215 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 2216 // Focal image is still old. 2217 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 2218 // the kernel binary has been updated. 2219 #[cfg(target_arch = "aarch64")] 2220 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 2221 #[cfg(target_arch = "x86_64")] 2222 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2223 let focal = UbuntuDiskConfig::new(focal_image); 2224 let guest = Guest::new(Box::new(focal)); 2225 2226 #[cfg(target_arch = "x86_64")] 2227 let kernel_path = direct_kernel_boot_path(); 2228 #[cfg(target_arch = "aarch64")] 2229 let kernel_path = if acpi { 2230 edk2_path() 2231 } else { 2232 direct_kernel_boot_path() 2233 }; 2234 2235 let mut child = GuestCommand::new(&guest) 2236 .args(["--cpus", "boot=1"]) 2237 .args(["--memory", "size=512M"]) 2238 .args(["--kernel", kernel_path.to_str().unwrap()]) 2239 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2240 .args([ 2241 "--disk", 2242 format!( 2243 "path={},iommu=on", 2244 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2245 ) 2246 .as_str(), 2247 format!( 2248 "path={},iommu=on", 2249 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2250 ) 2251 .as_str(), 2252 ]) 2253 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 2254 .capture_output() 2255 .spawn() 2256 .unwrap(); 2257 2258 let r = std::panic::catch_unwind(|| { 2259 guest.wait_vm_boot(None).unwrap(); 2260 2261 // Verify the virtio-iommu device is present. 2262 assert!(guest 2263 .does_device_vendor_pair_match("0x1057", "0x1af4") 2264 .unwrap_or_default()); 2265 2266 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 2267 // different with ACPI. 2268 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 2269 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 2270 // and appear under folder '/sys/kernel/iommu_groups/'. 2271 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 2272 // which is the console. The first disk "0000:00:02.0" is in group '1'. 2273 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 2274 // contains "0000:00:02.0" which is the first disk. 2275 // 2276 // Verify the iommu group of the first disk. 2277 let iommu_group = !acpi as i32; 2278 assert_eq!( 2279 guest 2280 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2281 .unwrap() 2282 .trim(), 2283 "0000:00:02.0" 2284 ); 2285 2286 // Verify the iommu group of the second disk. 2287 let iommu_group = if acpi { 1 } else { 2 }; 2288 assert_eq!( 2289 guest 2290 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2291 .unwrap() 2292 .trim(), 2293 "0000:00:03.0" 2294 ); 2295 2296 // Verify the iommu group of the network card. 2297 let iommu_group = if acpi { 2 } else { 3 }; 2298 assert_eq!( 2299 guest 2300 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2301 .unwrap() 2302 .trim(), 2303 "0000:00:04.0" 2304 ); 2305 }); 2306 2307 let _ = child.kill(); 2308 let output = child.wait_with_output().unwrap(); 2309 2310 handle_child_output(r, &output); 2311 } 2312 2313 fn get_reboot_count(guest: &Guest) -> u32 { 2314 guest 2315 .ssh_command("sudo last | grep -c reboot") 2316 .unwrap() 2317 .trim() 2318 .parse::<u32>() 2319 .unwrap_or_default() 2320 } 2321 2322 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 2323 // Check for PCI device 2324 assert!(guest 2325 .does_device_vendor_pair_match("0x1063", "0x1af4") 2326 .unwrap_or_default()); 2327 2328 // Enable systemd watchdog 2329 guest 2330 .ssh_command(&format!( 2331 "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" 2332 )) 2333 .unwrap(); 2334 2335 guest.ssh_command("sudo systemctl daemon-reexec").unwrap(); 2336 } 2337 2338 fn make_guest_panic(guest: &Guest) { 2339 // Check for pvpanic device 2340 assert!(guest 2341 .does_device_vendor_pair_match("0x0011", "0x1b36") 2342 .unwrap_or_default()); 2343 2344 // Trigger guest a panic 2345 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 2346 } 2347 2348 mod common_parallel { 2349 use std::{fs::OpenOptions, io::SeekFrom}; 2350 2351 use crate::*; 2352 2353 #[test] 2354 #[cfg(target_arch = "x86_64")] 2355 fn test_focal_hypervisor_fw() { 2356 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 2357 } 2358 2359 #[test] 2360 #[cfg(target_arch = "x86_64")] 2361 fn test_focal_ovmf() { 2362 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 2363 } 2364 2365 #[cfg(target_arch = "x86_64")] 2366 fn test_simple_launch(fw_path: String, disk_path: &str) { 2367 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 2368 let guest = Guest::new(disk_config); 2369 let event_path = temp_event_monitor_path(&guest.tmp_dir); 2370 2371 let mut child = GuestCommand::new(&guest) 2372 .args(["--cpus", "boot=1"]) 2373 .args(["--memory", "size=512M"]) 2374 .args(["--kernel", fw_path.as_str()]) 2375 .default_disks() 2376 .default_net() 2377 .args(["--serial", "tty", "--console", "off"]) 2378 .args(["--event-monitor", format!("path={event_path}").as_str()]) 2379 .capture_output() 2380 .spawn() 2381 .unwrap(); 2382 2383 let r = std::panic::catch_unwind(|| { 2384 guest.wait_vm_boot(Some(120)).unwrap(); 2385 2386 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2387 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2388 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2389 2390 let expected_sequential_events = [ 2391 &MetaEvent { 2392 event: "starting".to_string(), 2393 device_id: None, 2394 }, 2395 &MetaEvent { 2396 event: "booting".to_string(), 2397 device_id: None, 2398 }, 2399 &MetaEvent { 2400 event: "booted".to_string(), 2401 device_id: None, 2402 }, 2403 &MetaEvent { 2404 event: "activated".to_string(), 2405 device_id: Some("_disk0".to_string()), 2406 }, 2407 &MetaEvent { 2408 event: "reset".to_string(), 2409 device_id: Some("_disk0".to_string()), 2410 }, 2411 ]; 2412 assert!(check_sequential_events( 2413 &expected_sequential_events, 2414 &event_path 2415 )); 2416 2417 // It's been observed on the Bionic image that udev and snapd 2418 // services can cause some delay in the VM's shutdown. Disabling 2419 // them improves the reliability of this test. 2420 let _ = guest.ssh_command("sudo systemctl disable udev"); 2421 let _ = guest.ssh_command("sudo systemctl stop udev"); 2422 let _ = guest.ssh_command("sudo systemctl disable snapd"); 2423 let _ = guest.ssh_command("sudo systemctl stop snapd"); 2424 2425 guest.ssh_command("sudo poweroff").unwrap(); 2426 thread::sleep(std::time::Duration::new(20, 0)); 2427 let latest_events = [ 2428 &MetaEvent { 2429 event: "shutdown".to_string(), 2430 device_id: None, 2431 }, 2432 &MetaEvent { 2433 event: "deleted".to_string(), 2434 device_id: None, 2435 }, 2436 &MetaEvent { 2437 event: "shutdown".to_string(), 2438 device_id: None, 2439 }, 2440 ]; 2441 assert!(check_latest_events_exact(&latest_events, &event_path)); 2442 }); 2443 2444 let _ = child.kill(); 2445 let output = child.wait_with_output().unwrap(); 2446 2447 handle_child_output(r, &output); 2448 } 2449 2450 #[test] 2451 fn test_multi_cpu() { 2452 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2453 let jammy = UbuntuDiskConfig::new(jammy_image); 2454 let guest = Guest::new(Box::new(jammy)); 2455 2456 let mut cmd = GuestCommand::new(&guest); 2457 cmd.args(["--cpus", "boot=2,max=4"]) 2458 .args(["--memory", "size=512M"]) 2459 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2460 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2461 .capture_output() 2462 .default_disks() 2463 .default_net(); 2464 2465 let mut child = cmd.spawn().unwrap(); 2466 2467 let r = std::panic::catch_unwind(|| { 2468 guest.wait_vm_boot(Some(120)).unwrap(); 2469 2470 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2471 2472 assert_eq!( 2473 guest 2474 .ssh_command( 2475 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2476 ) 2477 .unwrap() 2478 .trim(), 2479 "smp: Brought up 1 node, 2 CPUs" 2480 ); 2481 }); 2482 2483 let _ = child.kill(); 2484 let output = child.wait_with_output().unwrap(); 2485 2486 handle_child_output(r, &output); 2487 } 2488 2489 #[test] 2490 fn test_cpu_topology_421() { 2491 test_cpu_topology(4, 2, 1, false); 2492 } 2493 2494 #[test] 2495 fn test_cpu_topology_142() { 2496 test_cpu_topology(1, 4, 2, false); 2497 } 2498 2499 #[test] 2500 fn test_cpu_topology_262() { 2501 test_cpu_topology(2, 6, 2, false); 2502 } 2503 2504 #[test] 2505 #[cfg(target_arch = "x86_64")] 2506 #[cfg(not(feature = "mshv"))] 2507 fn test_cpu_physical_bits() { 2508 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2509 let guest = Guest::new(Box::new(focal)); 2510 let max_phys_bits: u8 = 36; 2511 let mut child = GuestCommand::new(&guest) 2512 .args(["--cpus", &format!("max_phys_bits={max_phys_bits}")]) 2513 .args(["--memory", "size=512M"]) 2514 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2515 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2516 .default_disks() 2517 .default_net() 2518 .capture_output() 2519 .spawn() 2520 .unwrap(); 2521 2522 let r = std::panic::catch_unwind(|| { 2523 guest.wait_vm_boot(None).unwrap(); 2524 2525 assert!( 2526 guest 2527 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2528 .unwrap() 2529 .trim() 2530 .parse::<u8>() 2531 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2532 ); 2533 }); 2534 2535 let _ = child.kill(); 2536 let output = child.wait_with_output().unwrap(); 2537 2538 handle_child_output(r, &output); 2539 } 2540 2541 #[test] 2542 fn test_cpu_affinity() { 2543 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2544 let guest = Guest::new(Box::new(focal)); 2545 2546 // We need the host to have at least 4 CPUs if we want to be able 2547 // to run this test. 2548 let host_cpus_count = exec_host_command_output("nproc"); 2549 assert!( 2550 String::from_utf8_lossy(&host_cpus_count.stdout) 2551 .trim() 2552 .parse::<u16>() 2553 .unwrap_or(0) 2554 >= 4 2555 ); 2556 2557 let mut child = GuestCommand::new(&guest) 2558 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2559 .args(["--memory", "size=512M"]) 2560 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2561 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2562 .default_disks() 2563 .default_net() 2564 .capture_output() 2565 .spawn() 2566 .unwrap(); 2567 2568 let r = std::panic::catch_unwind(|| { 2569 guest.wait_vm_boot(None).unwrap(); 2570 let pid = child.id(); 2571 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2572 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2573 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2574 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2575 }); 2576 2577 let _ = child.kill(); 2578 let output = child.wait_with_output().unwrap(); 2579 handle_child_output(r, &output); 2580 } 2581 2582 #[test] 2583 fn test_virtio_queue_affinity() { 2584 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2585 let guest = Guest::new(Box::new(focal)); 2586 2587 // We need the host to have at least 4 CPUs if we want to be able 2588 // to run this test. 2589 let host_cpus_count = exec_host_command_output("nproc"); 2590 assert!( 2591 String::from_utf8_lossy(&host_cpus_count.stdout) 2592 .trim() 2593 .parse::<u16>() 2594 .unwrap_or(0) 2595 >= 4 2596 ); 2597 2598 let mut child = GuestCommand::new(&guest) 2599 .args(["--cpus", "boot=4"]) 2600 .args(["--memory", "size=512M"]) 2601 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2602 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2603 .args([ 2604 "--disk", 2605 format!( 2606 "path={}", 2607 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2608 ) 2609 .as_str(), 2610 format!( 2611 "path={},num_queues=4,queue_affinity=[0@[0,2],1@[1,3],2@[1],3@[3]]", 2612 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2613 ) 2614 .as_str(), 2615 ]) 2616 .default_net() 2617 .capture_output() 2618 .spawn() 2619 .unwrap(); 2620 2621 let r = std::panic::catch_unwind(|| { 2622 guest.wait_vm_boot(None).unwrap(); 2623 let pid = child.id(); 2624 let taskset_q0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2625 assert_eq!(String::from_utf8_lossy(&taskset_q0.stdout).trim(), "0,2"); 2626 let taskset_q1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2627 assert_eq!(String::from_utf8_lossy(&taskset_q1.stdout).trim(), "1,3"); 2628 let taskset_q2 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q2 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2629 assert_eq!(String::from_utf8_lossy(&taskset_q2.stdout).trim(), "1"); 2630 let taskset_q3 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q3 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2631 assert_eq!(String::from_utf8_lossy(&taskset_q3.stdout).trim(), "3"); 2632 }); 2633 2634 let _ = child.kill(); 2635 let output = child.wait_with_output().unwrap(); 2636 handle_child_output(r, &output); 2637 } 2638 2639 #[test] 2640 #[cfg(not(feature = "mshv"))] 2641 fn test_large_vm() { 2642 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2643 let guest = Guest::new(Box::new(focal)); 2644 let mut cmd = GuestCommand::new(&guest); 2645 cmd.args(["--cpus", "boot=48"]) 2646 .args(["--memory", "size=5120M"]) 2647 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2648 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2649 .args(["--serial", "tty"]) 2650 .args(["--console", "off"]) 2651 .capture_output() 2652 .default_disks() 2653 .default_net(); 2654 2655 let mut child = cmd.spawn().unwrap(); 2656 2657 guest.wait_vm_boot(None).unwrap(); 2658 2659 let r = std::panic::catch_unwind(|| { 2660 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2661 assert_eq!( 2662 guest 2663 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2664 .unwrap() 2665 .trim(), 2666 "0-47" 2667 ); 2668 2669 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2670 }); 2671 2672 let _ = child.kill(); 2673 let output = child.wait_with_output().unwrap(); 2674 2675 handle_child_output(r, &output); 2676 } 2677 2678 #[test] 2679 #[cfg(not(feature = "mshv"))] 2680 fn test_huge_memory() { 2681 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2682 let guest = Guest::new(Box::new(focal)); 2683 let mut cmd = GuestCommand::new(&guest); 2684 cmd.args(["--cpus", "boot=1"]) 2685 .args(["--memory", "size=128G"]) 2686 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2687 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2688 .capture_output() 2689 .default_disks() 2690 .default_net(); 2691 2692 let mut child = cmd.spawn().unwrap(); 2693 2694 guest.wait_vm_boot(Some(120)).unwrap(); 2695 2696 let r = std::panic::catch_unwind(|| { 2697 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2698 }); 2699 2700 let _ = child.kill(); 2701 let output = child.wait_with_output().unwrap(); 2702 2703 handle_child_output(r, &output); 2704 } 2705 2706 #[test] 2707 fn test_power_button() { 2708 _test_power_button(false); 2709 } 2710 2711 #[test] 2712 #[cfg(not(feature = "mshv"))] 2713 fn test_user_defined_memory_regions() { 2714 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2715 let guest = Guest::new(Box::new(focal)); 2716 let api_socket = temp_api_path(&guest.tmp_dir); 2717 2718 let kernel_path = direct_kernel_boot_path(); 2719 2720 let mut child = GuestCommand::new(&guest) 2721 .args(["--cpus", "boot=1"]) 2722 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2723 .args([ 2724 "--memory-zone", 2725 "id=mem0,size=1G,hotplug_size=2G", 2726 "id=mem1,size=1G,shared=on", 2727 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2728 ]) 2729 .args(["--kernel", kernel_path.to_str().unwrap()]) 2730 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2731 .args(["--api-socket", &api_socket]) 2732 .capture_output() 2733 .default_disks() 2734 .default_net() 2735 .spawn() 2736 .unwrap(); 2737 2738 let r = std::panic::catch_unwind(|| { 2739 guest.wait_vm_boot(None).unwrap(); 2740 2741 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2742 2743 guest.enable_memory_hotplug(); 2744 2745 resize_zone_command(&api_socket, "mem0", "3G"); 2746 thread::sleep(std::time::Duration::new(5, 0)); 2747 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2748 resize_zone_command(&api_socket, "mem2", "3G"); 2749 thread::sleep(std::time::Duration::new(5, 0)); 2750 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2751 resize_zone_command(&api_socket, "mem0", "2G"); 2752 thread::sleep(std::time::Duration::new(5, 0)); 2753 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2754 resize_zone_command(&api_socket, "mem2", "2G"); 2755 thread::sleep(std::time::Duration::new(5, 0)); 2756 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2757 2758 guest.reboot_linux(0, None); 2759 2760 // Check the amount of RAM after reboot 2761 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2762 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2763 2764 // Check if we can still resize down to the initial 'boot'size 2765 resize_zone_command(&api_socket, "mem0", "1G"); 2766 thread::sleep(std::time::Duration::new(5, 0)); 2767 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2768 resize_zone_command(&api_socket, "mem2", "1G"); 2769 thread::sleep(std::time::Duration::new(5, 0)); 2770 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2771 }); 2772 2773 let _ = child.kill(); 2774 let output = child.wait_with_output().unwrap(); 2775 2776 handle_child_output(r, &output); 2777 } 2778 2779 #[test] 2780 #[cfg(not(feature = "mshv"))] 2781 fn test_guest_numa_nodes() { 2782 _test_guest_numa_nodes(false); 2783 } 2784 2785 #[test] 2786 #[cfg(target_arch = "x86_64")] 2787 fn test_iommu_segments() { 2788 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2789 let focal = UbuntuDiskConfig::new(focal_image); 2790 let guest = Guest::new(Box::new(focal)); 2791 2792 // Prepare another disk file for the virtio-disk device 2793 let test_disk_path = String::from( 2794 guest 2795 .tmp_dir 2796 .as_path() 2797 .join("test-disk.raw") 2798 .to_str() 2799 .unwrap(), 2800 ); 2801 assert!( 2802 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2803 ); 2804 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2805 2806 let api_socket = temp_api_path(&guest.tmp_dir); 2807 let mut cmd = GuestCommand::new(&guest); 2808 2809 cmd.args(["--cpus", "boot=1"]) 2810 .args(["--api-socket", &api_socket]) 2811 .args(["--memory", "size=512M"]) 2812 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2813 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2814 .args([ 2815 "--platform", 2816 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS},iommu_segments=[1]"), 2817 ]) 2818 .default_disks() 2819 .capture_output() 2820 .default_net(); 2821 2822 let mut child = cmd.spawn().unwrap(); 2823 2824 guest.wait_vm_boot(None).unwrap(); 2825 2826 let r = std::panic::catch_unwind(|| { 2827 let (cmd_success, cmd_output) = remote_command_w_output( 2828 &api_socket, 2829 "add-disk", 2830 Some( 2831 format!( 2832 "path={},id=test0,pci_segment=1,iommu=on", 2833 test_disk_path.as_str() 2834 ) 2835 .as_str(), 2836 ), 2837 ); 2838 assert!(cmd_success); 2839 assert!(String::from_utf8_lossy(&cmd_output) 2840 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2841 2842 // Check IOMMU setup 2843 assert!(guest 2844 .does_device_vendor_pair_match("0x1057", "0x1af4") 2845 .unwrap_or_default()); 2846 assert_eq!( 2847 guest 2848 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2849 .unwrap() 2850 .trim(), 2851 "0001:00:01.0" 2852 ); 2853 }); 2854 2855 let _ = child.kill(); 2856 let output = child.wait_with_output().unwrap(); 2857 2858 handle_child_output(r, &output); 2859 } 2860 2861 #[test] 2862 fn test_pci_msi() { 2863 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2864 let guest = Guest::new(Box::new(focal)); 2865 let mut cmd = GuestCommand::new(&guest); 2866 cmd.args(["--cpus", "boot=1"]) 2867 .args(["--memory", "size=512M"]) 2868 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2869 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2870 .capture_output() 2871 .default_disks() 2872 .default_net(); 2873 2874 let mut child = cmd.spawn().unwrap(); 2875 2876 guest.wait_vm_boot(None).unwrap(); 2877 2878 #[cfg(target_arch = "x86_64")] 2879 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2880 #[cfg(target_arch = "aarch64")] 2881 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2882 2883 let r = std::panic::catch_unwind(|| { 2884 assert_eq!( 2885 guest 2886 .ssh_command(grep_cmd) 2887 .unwrap() 2888 .trim() 2889 .parse::<u32>() 2890 .unwrap_or_default(), 2891 12 2892 ); 2893 }); 2894 2895 let _ = child.kill(); 2896 let output = child.wait_with_output().unwrap(); 2897 2898 handle_child_output(r, &output); 2899 } 2900 2901 #[test] 2902 fn test_virtio_net_ctrl_queue() { 2903 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2904 let guest = Guest::new(Box::new(focal)); 2905 let mut cmd = GuestCommand::new(&guest); 2906 cmd.args(["--cpus", "boot=1"]) 2907 .args(["--memory", "size=512M"]) 2908 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2909 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2910 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2911 .capture_output() 2912 .default_disks(); 2913 2914 let mut child = cmd.spawn().unwrap(); 2915 2916 guest.wait_vm_boot(None).unwrap(); 2917 2918 #[cfg(target_arch = "aarch64")] 2919 let iface = "enp0s4"; 2920 #[cfg(target_arch = "x86_64")] 2921 let iface = "ens4"; 2922 2923 let r = std::panic::catch_unwind(|| { 2924 assert_eq!( 2925 guest 2926 .ssh_command( 2927 format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() 2928 ) 2929 .unwrap() 2930 .trim(), 2931 "success" 2932 ); 2933 assert_eq!( 2934 guest 2935 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 2936 .unwrap() 2937 .trim(), 2938 "3000" 2939 ); 2940 }); 2941 2942 let _ = child.kill(); 2943 let output = child.wait_with_output().unwrap(); 2944 2945 handle_child_output(r, &output); 2946 } 2947 2948 #[test] 2949 fn test_pci_multiple_segments() { 2950 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2951 let guest = Guest::new(Box::new(focal)); 2952 2953 // Prepare another disk file for the virtio-disk device 2954 let test_disk_path = String::from( 2955 guest 2956 .tmp_dir 2957 .as_path() 2958 .join("test-disk.raw") 2959 .to_str() 2960 .unwrap(), 2961 ); 2962 assert!( 2963 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2964 ); 2965 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2966 2967 let mut cmd = GuestCommand::new(&guest); 2968 cmd.args(["--cpus", "boot=1"]) 2969 .args(["--memory", "size=512M"]) 2970 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2971 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2972 .args([ 2973 "--platform", 2974 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 2975 ]) 2976 .args([ 2977 "--disk", 2978 format!( 2979 "path={}", 2980 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2981 ) 2982 .as_str(), 2983 format!( 2984 "path={}", 2985 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2986 ) 2987 .as_str(), 2988 format!("path={test_disk_path},pci_segment=15").as_str(), 2989 ]) 2990 .capture_output() 2991 .default_net(); 2992 2993 let mut child = cmd.spawn().unwrap(); 2994 2995 guest.wait_vm_boot(None).unwrap(); 2996 2997 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2998 2999 let r = std::panic::catch_unwind(|| { 3000 // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. 3001 assert_eq!( 3002 guest 3003 .ssh_command(grep_cmd) 3004 .unwrap() 3005 .trim() 3006 .parse::<u16>() 3007 .unwrap_or_default(), 3008 MAX_NUM_PCI_SEGMENTS 3009 ); 3010 3011 // Check both if /dev/vdc exists and if the block size is 4M. 3012 assert_eq!( 3013 guest 3014 .ssh_command("lsblk | grep vdc | grep -c 4M") 3015 .unwrap() 3016 .trim() 3017 .parse::<u32>() 3018 .unwrap_or_default(), 3019 1 3020 ); 3021 3022 // Mount the device. 3023 guest.ssh_command("mkdir mount_image").unwrap(); 3024 guest 3025 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 3026 .unwrap(); 3027 // Grant all users with write permission. 3028 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 3029 3030 // Write something to the device. 3031 guest 3032 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 3033 .unwrap(); 3034 3035 // Check the content of the block device. The file "foo" should 3036 // contain "bar". 3037 assert_eq!( 3038 guest 3039 .ssh_command("sudo cat mount_image/foo") 3040 .unwrap() 3041 .trim(), 3042 "bar" 3043 ); 3044 }); 3045 3046 let _ = child.kill(); 3047 let output = child.wait_with_output().unwrap(); 3048 3049 handle_child_output(r, &output); 3050 } 3051 3052 #[test] 3053 fn test_pci_multiple_segments_numa_node() { 3054 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3055 let guest = Guest::new(Box::new(focal)); 3056 let api_socket = temp_api_path(&guest.tmp_dir); 3057 #[cfg(target_arch = "x86_64")] 3058 let kernel_path = direct_kernel_boot_path(); 3059 #[cfg(target_arch = "aarch64")] 3060 let kernel_path = edk2_path(); 3061 3062 // Prepare another disk file for the virtio-disk device 3063 let test_disk_path = String::from( 3064 guest 3065 .tmp_dir 3066 .as_path() 3067 .join("test-disk.raw") 3068 .to_str() 3069 .unwrap(), 3070 ); 3071 assert!( 3072 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 3073 ); 3074 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 3075 const TEST_DISK_NODE: u16 = 1; 3076 3077 let mut child = GuestCommand::new(&guest) 3078 .args(["--platform", "num_pci_segments=2"]) 3079 .args(["--cpus", "boot=2"]) 3080 .args(["--memory", "size=0"]) 3081 .args(["--memory-zone", "id=mem0,size=256M", "id=mem1,size=256M"]) 3082 .args([ 3083 "--numa", 3084 "guest_numa_id=0,cpus=[0],distances=[1@20],memory_zones=mem0,pci_segments=[0]", 3085 "guest_numa_id=1,cpus=[1],distances=[0@20],memory_zones=mem1,pci_segments=[1]", 3086 ]) 3087 .args(["--kernel", kernel_path.to_str().unwrap()]) 3088 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3089 .args(["--api-socket", &api_socket]) 3090 .capture_output() 3091 .args([ 3092 "--disk", 3093 format!( 3094 "path={}", 3095 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3096 ) 3097 .as_str(), 3098 format!( 3099 "path={}", 3100 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3101 ) 3102 .as_str(), 3103 format!("path={test_disk_path},pci_segment={TEST_DISK_NODE}").as_str(), 3104 ]) 3105 .default_net() 3106 .spawn() 3107 .unwrap(); 3108 3109 let cmd = "cat /sys/block/vdc/device/../numa_node"; 3110 3111 let r = std::panic::catch_unwind(|| { 3112 guest.wait_vm_boot(None).unwrap(); 3113 3114 assert_eq!( 3115 guest 3116 .ssh_command(cmd) 3117 .unwrap() 3118 .trim() 3119 .parse::<u16>() 3120 .unwrap_or_default(), 3121 TEST_DISK_NODE 3122 ); 3123 }); 3124 3125 let _ = child.kill(); 3126 let output = child.wait_with_output().unwrap(); 3127 3128 handle_child_output(r, &output); 3129 } 3130 3131 #[test] 3132 fn test_direct_kernel_boot() { 3133 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3134 let guest = Guest::new(Box::new(focal)); 3135 3136 let kernel_path = direct_kernel_boot_path(); 3137 3138 let mut child = GuestCommand::new(&guest) 3139 .args(["--cpus", "boot=1"]) 3140 .args(["--memory", "size=512M"]) 3141 .args(["--kernel", kernel_path.to_str().unwrap()]) 3142 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3143 .default_disks() 3144 .default_net() 3145 .capture_output() 3146 .spawn() 3147 .unwrap(); 3148 3149 let r = std::panic::catch_unwind(|| { 3150 guest.wait_vm_boot(None).unwrap(); 3151 3152 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3153 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3154 3155 let grep_cmd = if cfg!(target_arch = "x86_64") { 3156 "grep -c PCI-MSI /proc/interrupts" 3157 } else { 3158 "grep -c ITS-MSI /proc/interrupts" 3159 }; 3160 assert_eq!( 3161 guest 3162 .ssh_command(grep_cmd) 3163 .unwrap() 3164 .trim() 3165 .parse::<u32>() 3166 .unwrap_or_default(), 3167 12 3168 ); 3169 }); 3170 3171 let _ = child.kill(); 3172 let output = child.wait_with_output().unwrap(); 3173 3174 handle_child_output(r, &output); 3175 } 3176 3177 #[test] 3178 #[cfg(target_arch = "x86_64")] 3179 fn test_direct_kernel_boot_bzimage() { 3180 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3181 let guest = Guest::new(Box::new(focal)); 3182 3183 let mut kernel_path = direct_kernel_boot_path(); 3184 // Replace the default kernel with the bzImage. 3185 kernel_path.pop(); 3186 kernel_path.push("bzImage"); 3187 3188 let mut child = GuestCommand::new(&guest) 3189 .args(["--cpus", "boot=1"]) 3190 .args(["--memory", "size=512M"]) 3191 .args(["--kernel", kernel_path.to_str().unwrap()]) 3192 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3193 .default_disks() 3194 .default_net() 3195 .capture_output() 3196 .spawn() 3197 .unwrap(); 3198 3199 let r = std::panic::catch_unwind(|| { 3200 guest.wait_vm_boot(None).unwrap(); 3201 3202 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3203 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3204 3205 let grep_cmd = if cfg!(target_arch = "x86_64") { 3206 "grep -c PCI-MSI /proc/interrupts" 3207 } else { 3208 "grep -c ITS-MSI /proc/interrupts" 3209 }; 3210 assert_eq!( 3211 guest 3212 .ssh_command(grep_cmd) 3213 .unwrap() 3214 .trim() 3215 .parse::<u32>() 3216 .unwrap_or_default(), 3217 12 3218 ); 3219 }); 3220 3221 let _ = child.kill(); 3222 let output = child.wait_with_output().unwrap(); 3223 3224 handle_child_output(r, &output); 3225 } 3226 3227 fn _test_virtio_block(image_name: &str, disable_io_uring: bool, disable_aio: bool) { 3228 let focal = UbuntuDiskConfig::new(image_name.to_string()); 3229 let guest = Guest::new(Box::new(focal)); 3230 3231 let mut workload_path = dirs::home_dir().unwrap(); 3232 workload_path.push("workloads"); 3233 3234 let mut blk_file_path = workload_path; 3235 blk_file_path.push("blk.img"); 3236 3237 let kernel_path = direct_kernel_boot_path(); 3238 3239 let mut cloud_child = GuestCommand::new(&guest) 3240 .args(["--cpus", "boot=4"]) 3241 .args(["--memory", "size=512M,shared=on"]) 3242 .args(["--kernel", kernel_path.to_str().unwrap()]) 3243 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3244 .args([ 3245 "--disk", 3246 format!( 3247 "path={}", 3248 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3249 ) 3250 .as_str(), 3251 format!( 3252 "path={}", 3253 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3254 ) 3255 .as_str(), 3256 format!( 3257 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", 3258 blk_file_path.to_str().unwrap(), 3259 disable_io_uring, 3260 disable_aio, 3261 ) 3262 .as_str(), 3263 ]) 3264 .default_net() 3265 .capture_output() 3266 .spawn() 3267 .unwrap(); 3268 3269 let r = std::panic::catch_unwind(|| { 3270 guest.wait_vm_boot(None).unwrap(); 3271 3272 // Check both if /dev/vdc exists and if the block size is 16M. 3273 assert_eq!( 3274 guest 3275 .ssh_command("lsblk | grep vdc | grep -c 16M") 3276 .unwrap() 3277 .trim() 3278 .parse::<u32>() 3279 .unwrap_or_default(), 3280 1 3281 ); 3282 3283 // Check both if /dev/vdc exists and if this block is RO. 3284 assert_eq!( 3285 guest 3286 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 3287 .unwrap() 3288 .trim() 3289 .parse::<u32>() 3290 .unwrap_or_default(), 3291 1 3292 ); 3293 3294 // Check if the number of queues is 4. 3295 assert_eq!( 3296 guest 3297 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 3298 .unwrap() 3299 .trim() 3300 .parse::<u32>() 3301 .unwrap_or_default(), 3302 4 3303 ); 3304 }); 3305 3306 let _ = cloud_child.kill(); 3307 let output = cloud_child.wait_with_output().unwrap(); 3308 3309 handle_child_output(r, &output); 3310 } 3311 3312 #[test] 3313 fn test_virtio_block_io_uring() { 3314 _test_virtio_block(FOCAL_IMAGE_NAME, false, true) 3315 } 3316 3317 #[test] 3318 fn test_virtio_block_aio() { 3319 _test_virtio_block(FOCAL_IMAGE_NAME, true, false) 3320 } 3321 3322 #[test] 3323 fn test_virtio_block_sync() { 3324 _test_virtio_block(FOCAL_IMAGE_NAME, true, true) 3325 } 3326 3327 #[test] 3328 fn test_virtio_block_qcow2() { 3329 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false, false) 3330 } 3331 3332 #[test] 3333 fn test_virtio_block_qcow2_backing_file() { 3334 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE, false, false) 3335 } 3336 3337 #[test] 3338 fn test_virtio_block_vhd() { 3339 let mut workload_path = dirs::home_dir().unwrap(); 3340 workload_path.push("workloads"); 3341 3342 let mut raw_file_path = workload_path.clone(); 3343 let mut vhd_file_path = workload_path; 3344 raw_file_path.push(FOCAL_IMAGE_NAME); 3345 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 3346 3347 // Generate VHD file from RAW file 3348 std::process::Command::new("qemu-img") 3349 .arg("convert") 3350 .arg("-p") 3351 .args(["-f", "raw"]) 3352 .args(["-O", "vpc"]) 3353 .args(["-o", "subformat=fixed"]) 3354 .arg(raw_file_path.to_str().unwrap()) 3355 .arg(vhd_file_path.to_str().unwrap()) 3356 .output() 3357 .expect("Expect generating VHD image from RAW image"); 3358 3359 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false, false) 3360 } 3361 3362 #[test] 3363 fn test_virtio_block_vhdx() { 3364 let mut workload_path = dirs::home_dir().unwrap(); 3365 workload_path.push("workloads"); 3366 3367 let mut raw_file_path = workload_path.clone(); 3368 let mut vhdx_file_path = workload_path; 3369 raw_file_path.push(FOCAL_IMAGE_NAME); 3370 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 3371 3372 // Generate dynamic VHDX file from RAW file 3373 std::process::Command::new("qemu-img") 3374 .arg("convert") 3375 .arg("-p") 3376 .args(["-f", "raw"]) 3377 .args(["-O", "vhdx"]) 3378 .arg(raw_file_path.to_str().unwrap()) 3379 .arg(vhdx_file_path.to_str().unwrap()) 3380 .output() 3381 .expect("Expect generating dynamic VHDx image from RAW image"); 3382 3383 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false, false) 3384 } 3385 3386 #[test] 3387 fn test_virtio_block_dynamic_vhdx_expand() { 3388 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 3389 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 3390 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 3391 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 3392 3393 let mut workload_path = dirs::home_dir().unwrap(); 3394 workload_path.push("workloads"); 3395 3396 let mut vhdx_file_path = workload_path; 3397 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 3398 let vhdx_path = vhdx_file_path.to_str().unwrap(); 3399 3400 // Generate a 100 MiB dynamic VHDX file 3401 std::process::Command::new("qemu-img") 3402 .arg("create") 3403 .args(["-f", "vhdx"]) 3404 .arg(vhdx_path) 3405 .arg(VIRTUAL_DISK_SIZE.to_string()) 3406 .output() 3407 .expect("Expect generating dynamic VHDx image from RAW image"); 3408 3409 // Check if the size matches with empty VHDx file size 3410 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 3411 3412 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3413 let guest = Guest::new(Box::new(focal)); 3414 let kernel_path = direct_kernel_boot_path(); 3415 3416 let mut cloud_child = GuestCommand::new(&guest) 3417 .args(["--cpus", "boot=1"]) 3418 .args(["--memory", "size=512M"]) 3419 .args(["--kernel", kernel_path.to_str().unwrap()]) 3420 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3421 .args([ 3422 "--disk", 3423 format!( 3424 "path={}", 3425 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3426 ) 3427 .as_str(), 3428 format!( 3429 "path={}", 3430 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3431 ) 3432 .as_str(), 3433 format!("path={vhdx_path}").as_str(), 3434 ]) 3435 .default_net() 3436 .capture_output() 3437 .spawn() 3438 .unwrap(); 3439 3440 let r = std::panic::catch_unwind(|| { 3441 guest.wait_vm_boot(None).unwrap(); 3442 3443 // Check both if /dev/vdc exists and if the block size is 100 MiB. 3444 assert_eq!( 3445 guest 3446 .ssh_command("lsblk | grep vdc | grep -c 100M") 3447 .unwrap() 3448 .trim() 3449 .parse::<u32>() 3450 .unwrap_or_default(), 3451 1 3452 ); 3453 3454 // Write 100 MB of data to the VHDx disk 3455 guest 3456 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 3457 .unwrap(); 3458 }); 3459 3460 // Check if the size matches with expected expanded VHDx file size 3461 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 3462 3463 let _ = cloud_child.kill(); 3464 let output = cloud_child.wait_with_output().unwrap(); 3465 3466 handle_child_output(r, &output); 3467 } 3468 3469 fn vhdx_image_size(disk_name: &str) -> u64 { 3470 std::fs::File::open(disk_name) 3471 .unwrap() 3472 .seek(SeekFrom::End(0)) 3473 .unwrap() 3474 } 3475 3476 #[test] 3477 fn test_virtio_block_direct_and_firmware() { 3478 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3479 let guest = Guest::new(Box::new(focal)); 3480 3481 // The OS disk must be copied to a location that is not backed by 3482 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 3483 // with EINVAL because tmpfs doesn't support this flag. 3484 let mut workloads_path = dirs::home_dir().unwrap(); 3485 workloads_path.push("workloads"); 3486 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 3487 let mut os_path = os_dir.as_path().to_path_buf(); 3488 os_path.push("osdisk.img"); 3489 rate_limited_copy( 3490 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3491 os_path.as_path(), 3492 ) 3493 .expect("copying of OS disk failed"); 3494 3495 let mut child = GuestCommand::new(&guest) 3496 .args(["--cpus", "boot=1"]) 3497 .args(["--memory", "size=512M"]) 3498 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 3499 .args([ 3500 "--disk", 3501 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 3502 format!( 3503 "path={}", 3504 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3505 ) 3506 .as_str(), 3507 ]) 3508 .default_net() 3509 .capture_output() 3510 .spawn() 3511 .unwrap(); 3512 3513 let r = std::panic::catch_unwind(|| { 3514 guest.wait_vm_boot(Some(120)).unwrap(); 3515 }); 3516 3517 let _ = child.kill(); 3518 let output = child.wait_with_output().unwrap(); 3519 3520 handle_child_output(r, &output); 3521 } 3522 3523 #[test] 3524 fn test_vhost_user_net_default() { 3525 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 3526 } 3527 3528 #[test] 3529 fn test_vhost_user_net_named_tap() { 3530 test_vhost_user_net( 3531 Some("mytap0"), 3532 2, 3533 &prepare_vhost_user_net_daemon, 3534 false, 3535 false, 3536 ) 3537 } 3538 3539 #[test] 3540 fn test_vhost_user_net_existing_tap() { 3541 test_vhost_user_net( 3542 Some("vunet-tap0"), 3543 2, 3544 &prepare_vhost_user_net_daemon, 3545 false, 3546 false, 3547 ) 3548 } 3549 3550 #[test] 3551 fn test_vhost_user_net_multiple_queues() { 3552 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 3553 } 3554 3555 #[test] 3556 fn test_vhost_user_net_tap_multiple_queues() { 3557 test_vhost_user_net( 3558 Some("vunet-tap1"), 3559 4, 3560 &prepare_vhost_user_net_daemon, 3561 false, 3562 false, 3563 ) 3564 } 3565 3566 #[test] 3567 fn test_vhost_user_net_host_mac() { 3568 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 3569 } 3570 3571 #[test] 3572 fn test_vhost_user_net_client_mode() { 3573 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 3574 } 3575 3576 #[test] 3577 #[cfg(not(target_arch = "aarch64"))] 3578 fn test_vhost_user_blk_default() { 3579 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 3580 } 3581 3582 #[test] 3583 #[cfg(not(target_arch = "aarch64"))] 3584 fn test_vhost_user_blk_readonly() { 3585 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3586 } 3587 3588 #[test] 3589 #[cfg(not(target_arch = "aarch64"))] 3590 fn test_vhost_user_blk_direct() { 3591 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3592 } 3593 3594 #[test] 3595 fn test_boot_from_vhost_user_blk_default() { 3596 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3597 } 3598 3599 #[test] 3600 #[cfg(target_arch = "x86_64")] 3601 fn test_split_irqchip() { 3602 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3603 let guest = Guest::new(Box::new(focal)); 3604 3605 let mut child = GuestCommand::new(&guest) 3606 .args(["--cpus", "boot=1"]) 3607 .args(["--memory", "size=512M"]) 3608 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3609 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3610 .default_disks() 3611 .default_net() 3612 .capture_output() 3613 .spawn() 3614 .unwrap(); 3615 3616 let r = std::panic::catch_unwind(|| { 3617 guest.wait_vm_boot(None).unwrap(); 3618 3619 assert_eq!( 3620 guest 3621 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3622 .unwrap() 3623 .trim() 3624 .parse::<u32>() 3625 .unwrap_or(1), 3626 0 3627 ); 3628 assert_eq!( 3629 guest 3630 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3631 .unwrap() 3632 .trim() 3633 .parse::<u32>() 3634 .unwrap_or(1), 3635 0 3636 ); 3637 }); 3638 3639 let _ = child.kill(); 3640 let output = child.wait_with_output().unwrap(); 3641 3642 handle_child_output(r, &output); 3643 } 3644 3645 #[test] 3646 #[cfg(target_arch = "x86_64")] 3647 fn test_dmi_serial_number() { 3648 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3649 let guest = Guest::new(Box::new(focal)); 3650 3651 let mut child = GuestCommand::new(&guest) 3652 .args(["--cpus", "boot=1"]) 3653 .args(["--memory", "size=512M"]) 3654 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3655 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3656 .args(["--platform", "serial_number=a=b;c=d"]) 3657 .default_disks() 3658 .default_net() 3659 .capture_output() 3660 .spawn() 3661 .unwrap(); 3662 3663 let r = std::panic::catch_unwind(|| { 3664 guest.wait_vm_boot(None).unwrap(); 3665 3666 assert_eq!( 3667 guest 3668 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3669 .unwrap() 3670 .trim(), 3671 "a=b;c=d" 3672 ); 3673 }); 3674 3675 let _ = child.kill(); 3676 let output = child.wait_with_output().unwrap(); 3677 3678 handle_child_output(r, &output); 3679 } 3680 3681 #[test] 3682 #[cfg(target_arch = "x86_64")] 3683 fn test_dmi_uuid() { 3684 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3685 let guest = Guest::new(Box::new(focal)); 3686 3687 let mut child = GuestCommand::new(&guest) 3688 .args(["--cpus", "boot=1"]) 3689 .args(["--memory", "size=512M"]) 3690 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3691 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3692 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3693 .default_disks() 3694 .default_net() 3695 .capture_output() 3696 .spawn() 3697 .unwrap(); 3698 3699 let r = std::panic::catch_unwind(|| { 3700 guest.wait_vm_boot(None).unwrap(); 3701 3702 assert_eq!( 3703 guest 3704 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3705 .unwrap() 3706 .trim(), 3707 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3708 ); 3709 }); 3710 3711 let _ = child.kill(); 3712 let output = child.wait_with_output().unwrap(); 3713 3714 handle_child_output(r, &output); 3715 } 3716 3717 #[test] 3718 #[cfg(target_arch = "x86_64")] 3719 fn test_dmi_oem_strings() { 3720 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3721 let guest = Guest::new(Box::new(focal)); 3722 3723 let s1 = "io.systemd.credential:xx=yy"; 3724 let s2 = "This is a test string"; 3725 3726 let oem_strings = format!("oem_strings=[{s1},{s2}]"); 3727 3728 let mut child = GuestCommand::new(&guest) 3729 .args(["--cpus", "boot=1"]) 3730 .args(["--memory", "size=512M"]) 3731 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3732 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3733 .args(["--platform", &oem_strings]) 3734 .default_disks() 3735 .default_net() 3736 .capture_output() 3737 .spawn() 3738 .unwrap(); 3739 3740 let r = std::panic::catch_unwind(|| { 3741 guest.wait_vm_boot(None).unwrap(); 3742 3743 assert_eq!( 3744 guest 3745 .ssh_command("sudo dmidecode --oem-string count") 3746 .unwrap() 3747 .trim(), 3748 "2" 3749 ); 3750 3751 assert_eq!( 3752 guest 3753 .ssh_command("sudo dmidecode --oem-string 1") 3754 .unwrap() 3755 .trim(), 3756 s1 3757 ); 3758 3759 assert_eq!( 3760 guest 3761 .ssh_command("sudo dmidecode --oem-string 2") 3762 .unwrap() 3763 .trim(), 3764 s2 3765 ); 3766 }); 3767 3768 let _ = child.kill(); 3769 let output = child.wait_with_output().unwrap(); 3770 3771 handle_child_output(r, &output); 3772 } 3773 3774 #[test] 3775 fn test_virtio_fs() { 3776 _test_virtio_fs(&prepare_virtiofsd, false, None) 3777 } 3778 3779 #[test] 3780 fn test_virtio_fs_hotplug() { 3781 _test_virtio_fs(&prepare_virtiofsd, true, None) 3782 } 3783 3784 #[test] 3785 #[cfg(not(feature = "mshv"))] 3786 fn test_virtio_fs_multi_segment_hotplug() { 3787 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3788 } 3789 3790 #[test] 3791 #[cfg(not(feature = "mshv"))] 3792 fn test_virtio_fs_multi_segment() { 3793 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3794 } 3795 3796 #[test] 3797 fn test_virtio_pmem_persist_writes() { 3798 test_virtio_pmem(false, false) 3799 } 3800 3801 #[test] 3802 fn test_virtio_pmem_discard_writes() { 3803 test_virtio_pmem(true, false) 3804 } 3805 3806 #[test] 3807 fn test_virtio_pmem_with_size() { 3808 test_virtio_pmem(true, true) 3809 } 3810 3811 #[test] 3812 fn test_boot_from_virtio_pmem() { 3813 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3814 let guest = Guest::new(Box::new(focal)); 3815 3816 let kernel_path = direct_kernel_boot_path(); 3817 3818 let mut child = GuestCommand::new(&guest) 3819 .args(["--cpus", "boot=1"]) 3820 .args(["--memory", "size=512M"]) 3821 .args(["--kernel", kernel_path.to_str().unwrap()]) 3822 .args([ 3823 "--disk", 3824 format!( 3825 "path={}", 3826 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3827 ) 3828 .as_str(), 3829 ]) 3830 .default_net() 3831 .args([ 3832 "--pmem", 3833 format!( 3834 "file={},size={}", 3835 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3836 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3837 .unwrap() 3838 .len() 3839 ) 3840 .as_str(), 3841 ]) 3842 .args([ 3843 "--cmdline", 3844 DIRECT_KERNEL_BOOT_CMDLINE 3845 .replace("vda1", "pmem0p1") 3846 .as_str(), 3847 ]) 3848 .capture_output() 3849 .spawn() 3850 .unwrap(); 3851 3852 let r = std::panic::catch_unwind(|| { 3853 guest.wait_vm_boot(None).unwrap(); 3854 3855 // Simple checks to validate the VM booted properly 3856 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3857 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3858 }); 3859 3860 let _ = child.kill(); 3861 let output = child.wait_with_output().unwrap(); 3862 3863 handle_child_output(r, &output); 3864 } 3865 3866 #[test] 3867 fn test_multiple_network_interfaces() { 3868 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3869 let guest = Guest::new(Box::new(focal)); 3870 3871 let kernel_path = direct_kernel_boot_path(); 3872 3873 let mut child = GuestCommand::new(&guest) 3874 .args(["--cpus", "boot=1"]) 3875 .args(["--memory", "size=512M"]) 3876 .args(["--kernel", kernel_path.to_str().unwrap()]) 3877 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3878 .default_disks() 3879 .args([ 3880 "--net", 3881 guest.default_net_string().as_str(), 3882 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3883 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3884 ]) 3885 .capture_output() 3886 .spawn() 3887 .unwrap(); 3888 3889 let r = std::panic::catch_unwind(|| { 3890 guest.wait_vm_boot(None).unwrap(); 3891 3892 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3893 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3894 3895 // 3 network interfaces + default localhost ==> 4 interfaces 3896 assert_eq!( 3897 guest 3898 .ssh_command("ip -o link | wc -l") 3899 .unwrap() 3900 .trim() 3901 .parse::<u32>() 3902 .unwrap_or_default(), 3903 4 3904 ); 3905 }); 3906 3907 let _ = child.kill(); 3908 let output = child.wait_with_output().unwrap(); 3909 3910 handle_child_output(r, &output); 3911 } 3912 3913 #[test] 3914 #[cfg(target_arch = "aarch64")] 3915 fn test_pmu_on() { 3916 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3917 let guest = Guest::new(Box::new(focal)); 3918 let mut child = GuestCommand::new(&guest) 3919 .args(["--cpus", "boot=1"]) 3920 .args(["--memory", "size=512M"]) 3921 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3922 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3923 .default_disks() 3924 .default_net() 3925 .capture_output() 3926 .spawn() 3927 .unwrap(); 3928 3929 let r = std::panic::catch_unwind(|| { 3930 guest.wait_vm_boot(None).unwrap(); 3931 3932 // Test that PMU exists. 3933 assert_eq!( 3934 guest 3935 .ssh_command(GREP_PMU_IRQ_CMD) 3936 .unwrap() 3937 .trim() 3938 .parse::<u32>() 3939 .unwrap_or_default(), 3940 1 3941 ); 3942 }); 3943 3944 let _ = child.kill(); 3945 let output = child.wait_with_output().unwrap(); 3946 3947 handle_child_output(r, &output); 3948 } 3949 3950 #[test] 3951 fn test_serial_off() { 3952 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3953 let guest = Guest::new(Box::new(focal)); 3954 let mut child = GuestCommand::new(&guest) 3955 .args(["--cpus", "boot=1"]) 3956 .args(["--memory", "size=512M"]) 3957 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3958 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3959 .default_disks() 3960 .default_net() 3961 .args(["--serial", "off"]) 3962 .capture_output() 3963 .spawn() 3964 .unwrap(); 3965 3966 let r = std::panic::catch_unwind(|| { 3967 guest.wait_vm_boot(None).unwrap(); 3968 3969 // Test that there is no ttyS0 3970 assert_eq!( 3971 guest 3972 .ssh_command(GREP_SERIAL_IRQ_CMD) 3973 .unwrap() 3974 .trim() 3975 .parse::<u32>() 3976 .unwrap_or(1), 3977 0 3978 ); 3979 }); 3980 3981 let _ = child.kill(); 3982 let output = child.wait_with_output().unwrap(); 3983 3984 handle_child_output(r, &output); 3985 } 3986 3987 #[test] 3988 fn test_serial_null() { 3989 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3990 let guest = Guest::new(Box::new(focal)); 3991 let mut cmd = GuestCommand::new(&guest); 3992 #[cfg(target_arch = "x86_64")] 3993 let console_str: &str = "console=ttyS0"; 3994 #[cfg(target_arch = "aarch64")] 3995 let console_str: &str = "console=ttyAMA0"; 3996 3997 cmd.args(["--cpus", "boot=1"]) 3998 .args(["--memory", "size=512M"]) 3999 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4000 .args([ 4001 "--cmdline", 4002 DIRECT_KERNEL_BOOT_CMDLINE 4003 .replace("console=hvc0 ", console_str) 4004 .as_str(), 4005 ]) 4006 .default_disks() 4007 .default_net() 4008 .args(["--serial", "null"]) 4009 .args(["--console", "off"]) 4010 .capture_output(); 4011 4012 let mut child = cmd.spawn().unwrap(); 4013 4014 let r = std::panic::catch_unwind(|| { 4015 guest.wait_vm_boot(None).unwrap(); 4016 4017 // Test that there is a ttyS0 4018 assert_eq!( 4019 guest 4020 .ssh_command(GREP_SERIAL_IRQ_CMD) 4021 .unwrap() 4022 .trim() 4023 .parse::<u32>() 4024 .unwrap_or_default(), 4025 1 4026 ); 4027 }); 4028 4029 let _ = child.kill(); 4030 let output = child.wait_with_output().unwrap(); 4031 handle_child_output(r, &output); 4032 4033 let r = std::panic::catch_unwind(|| { 4034 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4035 }); 4036 4037 handle_child_output(r, &output); 4038 } 4039 4040 #[test] 4041 fn test_serial_tty() { 4042 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4043 let guest = Guest::new(Box::new(focal)); 4044 4045 let kernel_path = direct_kernel_boot_path(); 4046 4047 #[cfg(target_arch = "x86_64")] 4048 let console_str: &str = "console=ttyS0"; 4049 #[cfg(target_arch = "aarch64")] 4050 let console_str: &str = "console=ttyAMA0"; 4051 4052 let mut child = GuestCommand::new(&guest) 4053 .args(["--cpus", "boot=1"]) 4054 .args(["--memory", "size=512M"]) 4055 .args(["--kernel", kernel_path.to_str().unwrap()]) 4056 .args([ 4057 "--cmdline", 4058 DIRECT_KERNEL_BOOT_CMDLINE 4059 .replace("console=hvc0 ", console_str) 4060 .as_str(), 4061 ]) 4062 .default_disks() 4063 .default_net() 4064 .args(["--serial", "tty"]) 4065 .args(["--console", "off"]) 4066 .capture_output() 4067 .spawn() 4068 .unwrap(); 4069 4070 let r = std::panic::catch_unwind(|| { 4071 guest.wait_vm_boot(None).unwrap(); 4072 4073 // Test that there is a ttyS0 4074 assert_eq!( 4075 guest 4076 .ssh_command(GREP_SERIAL_IRQ_CMD) 4077 .unwrap() 4078 .trim() 4079 .parse::<u32>() 4080 .unwrap_or_default(), 4081 1 4082 ); 4083 }); 4084 4085 // This sleep is needed to wait for the login prompt 4086 thread::sleep(std::time::Duration::new(2, 0)); 4087 4088 let _ = child.kill(); 4089 let output = child.wait_with_output().unwrap(); 4090 handle_child_output(r, &output); 4091 4092 let r = std::panic::catch_unwind(|| { 4093 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4094 }); 4095 4096 handle_child_output(r, &output); 4097 } 4098 4099 #[test] 4100 fn test_serial_file() { 4101 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4102 let guest = Guest::new(Box::new(focal)); 4103 4104 let serial_path = guest.tmp_dir.as_path().join("serial-output"); 4105 #[cfg(target_arch = "x86_64")] 4106 let console_str: &str = "console=ttyS0"; 4107 #[cfg(target_arch = "aarch64")] 4108 let console_str: &str = "console=ttyAMA0"; 4109 4110 let mut child = GuestCommand::new(&guest) 4111 .args(["--cpus", "boot=1"]) 4112 .args(["--memory", "size=512M"]) 4113 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4114 .args([ 4115 "--cmdline", 4116 DIRECT_KERNEL_BOOT_CMDLINE 4117 .replace("console=hvc0 ", console_str) 4118 .as_str(), 4119 ]) 4120 .default_disks() 4121 .default_net() 4122 .args([ 4123 "--serial", 4124 format!("file={}", serial_path.to_str().unwrap()).as_str(), 4125 ]) 4126 .capture_output() 4127 .spawn() 4128 .unwrap(); 4129 4130 let r = std::panic::catch_unwind(|| { 4131 guest.wait_vm_boot(None).unwrap(); 4132 4133 // Test that there is a ttyS0 4134 assert_eq!( 4135 guest 4136 .ssh_command(GREP_SERIAL_IRQ_CMD) 4137 .unwrap() 4138 .trim() 4139 .parse::<u32>() 4140 .unwrap_or_default(), 4141 1 4142 ); 4143 4144 guest.ssh_command("sudo shutdown -h now").unwrap(); 4145 }); 4146 4147 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4148 let _ = child.kill(); 4149 let output = child.wait_with_output().unwrap(); 4150 handle_child_output(r, &output); 4151 4152 let r = std::panic::catch_unwind(|| { 4153 // Check that the cloud-hypervisor binary actually terminated 4154 assert!(output.status.success()); 4155 4156 // Do this check after shutdown of the VM as an easy way to ensure 4157 // all writes are flushed to disk 4158 let mut f = std::fs::File::open(serial_path).unwrap(); 4159 let mut buf = String::new(); 4160 f.read_to_string(&mut buf).unwrap(); 4161 assert!(buf.contains(CONSOLE_TEST_STRING)); 4162 }); 4163 4164 handle_child_output(r, &output); 4165 } 4166 4167 #[test] 4168 fn test_pty_interaction() { 4169 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4170 let guest = Guest::new(Box::new(focal)); 4171 let api_socket = temp_api_path(&guest.tmp_dir); 4172 let serial_option = if cfg!(target_arch = "x86_64") { 4173 " console=ttyS0" 4174 } else { 4175 " console=ttyAMA0" 4176 }; 4177 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4178 4179 let mut child = GuestCommand::new(&guest) 4180 .args(["--cpus", "boot=1"]) 4181 .args(["--memory", "size=512M"]) 4182 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4183 .args(["--cmdline", &cmdline]) 4184 .default_disks() 4185 .default_net() 4186 .args(["--serial", "null"]) 4187 .args(["--console", "pty"]) 4188 .args(["--api-socket", &api_socket]) 4189 .spawn() 4190 .unwrap(); 4191 4192 let r = std::panic::catch_unwind(|| { 4193 guest.wait_vm_boot(None).unwrap(); 4194 // Get pty fd for console 4195 let console_path = get_pty_path(&api_socket, "console"); 4196 _test_pty_interaction(console_path); 4197 4198 guest.ssh_command("sudo shutdown -h now").unwrap(); 4199 }); 4200 4201 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4202 let _ = child.kill(); 4203 let output = child.wait_with_output().unwrap(); 4204 handle_child_output(r, &output); 4205 4206 let r = std::panic::catch_unwind(|| { 4207 // Check that the cloud-hypervisor binary actually terminated 4208 assert!(output.status.success()) 4209 }); 4210 handle_child_output(r, &output); 4211 } 4212 4213 #[test] 4214 fn test_serial_socket_interaction() { 4215 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4216 let guest = Guest::new(Box::new(focal)); 4217 let serial_socket = guest.tmp_dir.as_path().join("serial.socket"); 4218 let serial_socket_pty = guest.tmp_dir.as_path().join("serial.pty"); 4219 let serial_option = if cfg!(target_arch = "x86_64") { 4220 " console=ttyS0" 4221 } else { 4222 " console=ttyAMA0" 4223 }; 4224 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4225 4226 let mut child = GuestCommand::new(&guest) 4227 .args(["--cpus", "boot=1"]) 4228 .args(["--memory", "size=512M"]) 4229 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4230 .args(["--cmdline", &cmdline]) 4231 .default_disks() 4232 .default_net() 4233 .args(["--console", "null"]) 4234 .args([ 4235 "--serial", 4236 format!("socket={}", serial_socket.to_str().unwrap()).as_str(), 4237 ]) 4238 .spawn() 4239 .unwrap(); 4240 4241 let _ = std::panic::catch_unwind(|| { 4242 guest.wait_vm_boot(None).unwrap(); 4243 }); 4244 4245 let mut socat_command = Command::new("socat"); 4246 let socat_args = [ 4247 &format!("pty,link={},raw", serial_socket_pty.display()), 4248 &format!("UNIX-CONNECT:{}", serial_socket.display()), 4249 ]; 4250 socat_command.args(socat_args); 4251 4252 let mut socat_child = socat_command.spawn().unwrap(); 4253 thread::sleep(std::time::Duration::new(1, 0)); 4254 4255 let _ = std::panic::catch_unwind(|| { 4256 _test_pty_interaction(serial_socket_pty); 4257 }); 4258 4259 let _ = socat_child.kill(); 4260 4261 let r = std::panic::catch_unwind(|| { 4262 guest.ssh_command("sudo shutdown -h now").unwrap(); 4263 }); 4264 4265 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4266 let _ = child.kill(); 4267 let output = child.wait_with_output().unwrap(); 4268 handle_child_output(r, &output); 4269 4270 let r = std::panic::catch_unwind(|| { 4271 // Check that the cloud-hypervisor binary actually terminated 4272 if !output.status.success() { 4273 panic!( 4274 "Cloud Hypervisor process failed to terminate gracefully: {:?}", 4275 output.status 4276 ); 4277 } 4278 }); 4279 handle_child_output(r, &output); 4280 } 4281 4282 #[test] 4283 fn test_virtio_console() { 4284 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4285 let guest = Guest::new(Box::new(focal)); 4286 4287 let kernel_path = direct_kernel_boot_path(); 4288 4289 let mut child = GuestCommand::new(&guest) 4290 .args(["--cpus", "boot=1"]) 4291 .args(["--memory", "size=512M"]) 4292 .args(["--kernel", kernel_path.to_str().unwrap()]) 4293 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4294 .default_disks() 4295 .default_net() 4296 .args(["--console", "tty"]) 4297 .args(["--serial", "null"]) 4298 .capture_output() 4299 .spawn() 4300 .unwrap(); 4301 4302 let text = String::from("On a branch floating down river a cricket, singing."); 4303 let cmd = format!("echo {text} | sudo tee /dev/hvc0"); 4304 4305 let r = std::panic::catch_unwind(|| { 4306 guest.wait_vm_boot(None).unwrap(); 4307 4308 assert!(guest 4309 .does_device_vendor_pair_match("0x1043", "0x1af4") 4310 .unwrap_or_default()); 4311 4312 guest.ssh_command(&cmd).unwrap(); 4313 }); 4314 4315 let _ = child.kill(); 4316 let output = child.wait_with_output().unwrap(); 4317 handle_child_output(r, &output); 4318 4319 let r = std::panic::catch_unwind(|| { 4320 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 4321 }); 4322 4323 handle_child_output(r, &output); 4324 } 4325 4326 #[test] 4327 fn test_console_file() { 4328 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4329 let guest = Guest::new(Box::new(focal)); 4330 4331 let console_path = guest.tmp_dir.as_path().join("console-output"); 4332 let mut child = GuestCommand::new(&guest) 4333 .args(["--cpus", "boot=1"]) 4334 .args(["--memory", "size=512M"]) 4335 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4336 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4337 .default_disks() 4338 .default_net() 4339 .args([ 4340 "--console", 4341 format!("file={}", console_path.to_str().unwrap()).as_str(), 4342 ]) 4343 .capture_output() 4344 .spawn() 4345 .unwrap(); 4346 4347 guest.wait_vm_boot(None).unwrap(); 4348 4349 guest.ssh_command("sudo shutdown -h now").unwrap(); 4350 4351 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4352 let _ = child.kill(); 4353 let output = child.wait_with_output().unwrap(); 4354 4355 let r = std::panic::catch_unwind(|| { 4356 // Check that the cloud-hypervisor binary actually terminated 4357 assert!(output.status.success()); 4358 4359 // Do this check after shutdown of the VM as an easy way to ensure 4360 // all writes are flushed to disk 4361 let mut f = std::fs::File::open(console_path).unwrap(); 4362 let mut buf = String::new(); 4363 f.read_to_string(&mut buf).unwrap(); 4364 4365 if !buf.contains(CONSOLE_TEST_STRING) { 4366 eprintln!( 4367 "\n\n==== Console file output ====\n\n{buf}\n\n==== End console file output ====" 4368 ); 4369 } 4370 assert!(buf.contains(CONSOLE_TEST_STRING)); 4371 }); 4372 4373 handle_child_output(r, &output); 4374 } 4375 4376 #[test] 4377 #[cfg(target_arch = "x86_64")] 4378 #[cfg(not(feature = "mshv"))] 4379 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 4380 // backed networking interfaces, bound through a simple bridge on the host. 4381 // So if the nested cloud-hypervisor succeeds in getting a directly 4382 // assigned interface from its cloud-hypervisor host, we should be able to 4383 // ssh into it, and verify that it's running with the right kernel command 4384 // line (We tag the command line from cloud-hypervisor for that purpose). 4385 // The third device is added to validate that hotplug works correctly since 4386 // it is being added to the L2 VM through hotplugging mechanism. 4387 // Also, we pass-through a virtio-blk device to the L2 VM to test the 32-bit 4388 // vfio device support 4389 fn test_vfio() { 4390 setup_vfio_network_interfaces(); 4391 4392 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 4393 let guest = Guest::new_from_ip_range(Box::new(jammy), "172.18", 0); 4394 4395 let mut workload_path = dirs::home_dir().unwrap(); 4396 workload_path.push("workloads"); 4397 4398 let kernel_path = direct_kernel_boot_path(); 4399 4400 let mut vfio_path = workload_path.clone(); 4401 vfio_path.push("vfio"); 4402 4403 let mut cloud_init_vfio_base_path = vfio_path.clone(); 4404 cloud_init_vfio_base_path.push("cloudinit.img"); 4405 4406 // We copy our cloudinit into the vfio mount point, for the nested 4407 // cloud-hypervisor guest to use. 4408 rate_limited_copy( 4409 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 4410 &cloud_init_vfio_base_path, 4411 ) 4412 .expect("copying of cloud-init disk failed"); 4413 4414 let mut vfio_disk_path = workload_path.clone(); 4415 vfio_disk_path.push("vfio.img"); 4416 4417 // Create the vfio disk image 4418 let output = Command::new("mkfs.ext4") 4419 .arg("-d") 4420 .arg(vfio_path.to_str().unwrap()) 4421 .arg(vfio_disk_path.to_str().unwrap()) 4422 .arg("2g") 4423 .output() 4424 .unwrap(); 4425 if !output.status.success() { 4426 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 4427 panic!("mkfs.ext4 command generated an error"); 4428 } 4429 4430 let mut blk_file_path = workload_path; 4431 blk_file_path.push("blk.img"); 4432 4433 let vfio_tap0 = "vfio-tap0"; 4434 let vfio_tap1 = "vfio-tap1"; 4435 let vfio_tap2 = "vfio-tap2"; 4436 let vfio_tap3 = "vfio-tap3"; 4437 4438 let mut child = GuestCommand::new(&guest) 4439 .args(["--cpus", "boot=4"]) 4440 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 4441 .args(["--kernel", kernel_path.to_str().unwrap()]) 4442 .args([ 4443 "--disk", 4444 format!( 4445 "path={}", 4446 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4447 ) 4448 .as_str(), 4449 format!( 4450 "path={}", 4451 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4452 ) 4453 .as_str(), 4454 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 4455 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 4456 ]) 4457 .args([ 4458 "--cmdline", 4459 format!( 4460 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 4461 ) 4462 .as_str(), 4463 ]) 4464 .args([ 4465 "--net", 4466 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 4467 format!( 4468 "tap={},mac={},iommu=on", 4469 vfio_tap1, guest.network.l2_guest_mac1 4470 ) 4471 .as_str(), 4472 format!( 4473 "tap={},mac={},iommu=on", 4474 vfio_tap2, guest.network.l2_guest_mac2 4475 ) 4476 .as_str(), 4477 format!( 4478 "tap={},mac={},iommu=on", 4479 vfio_tap3, guest.network.l2_guest_mac3 4480 ) 4481 .as_str(), 4482 ]) 4483 .capture_output() 4484 .spawn() 4485 .unwrap(); 4486 4487 thread::sleep(std::time::Duration::new(30, 0)); 4488 4489 let r = std::panic::catch_unwind(|| { 4490 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 4491 thread::sleep(std::time::Duration::new(120, 0)); 4492 4493 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 4494 // added to its kernel command line. 4495 // Let's ssh into it and verify that it's there. If it is it means 4496 // we're in the right guest (The L2 one) because the QEMU L1 guest 4497 // does not have this command line tag. 4498 assert!(check_matched_lines_count( 4499 guest.ssh_command_l2_1("cat /proc/cmdline").unwrap().trim(), 4500 vec!["VFIOTAG"], 4501 1 4502 )); 4503 4504 // Let's also verify from the second virtio-net device passed to 4505 // the L2 VM. 4506 assert!(check_matched_lines_count( 4507 guest.ssh_command_l2_2("cat /proc/cmdline").unwrap().trim(), 4508 vec!["VFIOTAG"], 4509 1 4510 )); 4511 4512 // Check the amount of PCI devices appearing in L2 VM. 4513 assert!(check_lines_count( 4514 guest 4515 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4516 .unwrap() 4517 .trim(), 4518 8 4519 )); 4520 4521 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 4522 assert!(check_matched_lines_count( 4523 guest.ssh_command_l2_1("lsblk").unwrap().trim(), 4524 vec!["vdc", "16M"], 4525 1 4526 )); 4527 4528 // Hotplug an extra virtio-net device through L2 VM. 4529 guest 4530 .ssh_command_l1( 4531 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 4532 ) 4533 .unwrap(); 4534 guest 4535 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 4536 .unwrap(); 4537 let vfio_hotplug_output = guest 4538 .ssh_command_l1( 4539 "sudo /mnt/ch-remote \ 4540 --api-socket=/tmp/ch_api.sock \ 4541 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 4542 ) 4543 .unwrap(); 4544 assert!(check_matched_lines_count( 4545 vfio_hotplug_output.trim(), 4546 vec!["{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}"], 4547 1 4548 )); 4549 4550 thread::sleep(std::time::Duration::new(10, 0)); 4551 4552 // Let's also verify from the third virtio-net device passed to 4553 // the L2 VM. This third device has been hotplugged through the L2 4554 // VM, so this is our way to validate hotplug works for VFIO PCI. 4555 assert!(check_matched_lines_count( 4556 guest.ssh_command_l2_3("cat /proc/cmdline").unwrap().trim(), 4557 vec!["VFIOTAG"], 4558 1 4559 )); 4560 4561 // Check the amount of PCI devices appearing in L2 VM. 4562 // There should be one more device than before, raising the count 4563 // up to 9 PCI devices. 4564 assert!(check_lines_count( 4565 guest 4566 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4567 .unwrap() 4568 .trim(), 4569 9 4570 )); 4571 4572 // Let's now verify that we can correctly remove the virtio-net 4573 // device through the "remove-device" command responsible for 4574 // unplugging VFIO devices. 4575 guest 4576 .ssh_command_l1( 4577 "sudo /mnt/ch-remote \ 4578 --api-socket=/tmp/ch_api.sock \ 4579 remove-device vfio123", 4580 ) 4581 .unwrap(); 4582 thread::sleep(std::time::Duration::new(10, 0)); 4583 4584 // Check the amount of PCI devices appearing in L2 VM is back down 4585 // to 8 devices. 4586 assert!(check_lines_count( 4587 guest 4588 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4589 .unwrap() 4590 .trim(), 4591 8 4592 )); 4593 4594 // Perform memory hotplug in L2 and validate the memory is showing 4595 // up as expected. In order to check, we will use the virtio-net 4596 // device already passed through L2 as a VFIO device, this will 4597 // verify that VFIO devices are functional with memory hotplug. 4598 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 4599 guest 4600 .ssh_command_l2_1( 4601 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 4602 ) 4603 .unwrap(); 4604 guest 4605 .ssh_command_l1( 4606 "sudo /mnt/ch-remote \ 4607 --api-socket=/tmp/ch_api.sock \ 4608 resize --memory=1073741824", 4609 ) 4610 .unwrap(); 4611 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 4612 }); 4613 4614 let _ = child.kill(); 4615 let output = child.wait_with_output().unwrap(); 4616 4617 cleanup_vfio_network_interfaces(); 4618 4619 handle_child_output(r, &output); 4620 } 4621 4622 #[test] 4623 fn test_direct_kernel_boot_noacpi() { 4624 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4625 let guest = Guest::new(Box::new(focal)); 4626 4627 let kernel_path = direct_kernel_boot_path(); 4628 4629 let mut child = GuestCommand::new(&guest) 4630 .args(["--cpus", "boot=1"]) 4631 .args(["--memory", "size=512M"]) 4632 .args(["--kernel", kernel_path.to_str().unwrap()]) 4633 .args([ 4634 "--cmdline", 4635 format!("{DIRECT_KERNEL_BOOT_CMDLINE} acpi=off").as_str(), 4636 ]) 4637 .default_disks() 4638 .default_net() 4639 .capture_output() 4640 .spawn() 4641 .unwrap(); 4642 4643 let r = std::panic::catch_unwind(|| { 4644 guest.wait_vm_boot(None).unwrap(); 4645 4646 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4647 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4648 }); 4649 4650 let _ = child.kill(); 4651 let output = child.wait_with_output().unwrap(); 4652 4653 handle_child_output(r, &output); 4654 } 4655 4656 #[test] 4657 fn test_virtio_vsock() { 4658 _test_virtio_vsock(false) 4659 } 4660 4661 #[test] 4662 fn test_virtio_vsock_hotplug() { 4663 _test_virtio_vsock(true); 4664 } 4665 4666 #[test] 4667 fn test_api_http_shutdown() { 4668 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4669 let guest = Guest::new(Box::new(focal)); 4670 4671 _test_api_shutdown(TargetApi::new_http_api(&guest.tmp_dir), guest) 4672 } 4673 4674 #[test] 4675 fn test_api_http_delete() { 4676 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4677 let guest = Guest::new(Box::new(focal)); 4678 4679 _test_api_delete(TargetApi::new_http_api(&guest.tmp_dir), guest); 4680 } 4681 4682 #[test] 4683 fn test_api_http_pause_resume() { 4684 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4685 let guest = Guest::new(Box::new(focal)); 4686 4687 _test_api_pause_resume(TargetApi::new_http_api(&guest.tmp_dir), guest) 4688 } 4689 4690 #[test] 4691 fn test_api_http_create_boot() { 4692 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4693 let guest = Guest::new(Box::new(focal)); 4694 4695 _test_api_create_boot(TargetApi::new_http_api(&guest.tmp_dir), guest) 4696 } 4697 4698 #[test] 4699 fn test_virtio_iommu() { 4700 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4701 } 4702 4703 #[test] 4704 // We cannot force the software running in the guest to reprogram the BAR 4705 // with some different addresses, but we have a reliable way of testing it 4706 // with a standard Linux kernel. 4707 // By removing a device from the PCI tree, and then rescanning the tree, 4708 // Linux consistently chooses to reorganize the PCI device BARs to other 4709 // locations in the guest address space. 4710 // This test creates a dedicated PCI network device to be checked as being 4711 // properly probed first, then removing it, and adding it again by doing a 4712 // rescan. 4713 fn test_pci_bar_reprogramming() { 4714 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4715 let guest = Guest::new(Box::new(focal)); 4716 4717 #[cfg(target_arch = "x86_64")] 4718 let kernel_path = direct_kernel_boot_path(); 4719 #[cfg(target_arch = "aarch64")] 4720 let kernel_path = edk2_path(); 4721 4722 let mut child = GuestCommand::new(&guest) 4723 .args(["--cpus", "boot=1"]) 4724 .args(["--memory", "size=512M"]) 4725 .args(["--kernel", kernel_path.to_str().unwrap()]) 4726 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4727 .default_disks() 4728 .args([ 4729 "--net", 4730 guest.default_net_string().as_str(), 4731 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4732 ]) 4733 .capture_output() 4734 .spawn() 4735 .unwrap(); 4736 4737 let r = std::panic::catch_unwind(|| { 4738 guest.wait_vm_boot(None).unwrap(); 4739 4740 // 2 network interfaces + default localhost ==> 3 interfaces 4741 assert_eq!( 4742 guest 4743 .ssh_command("ip -o link | wc -l") 4744 .unwrap() 4745 .trim() 4746 .parse::<u32>() 4747 .unwrap_or_default(), 4748 3 4749 ); 4750 4751 let init_bar_addr = guest 4752 .ssh_command( 4753 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4754 ) 4755 .unwrap(); 4756 4757 // Remove the PCI device 4758 guest 4759 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4760 .unwrap(); 4761 4762 // Only 1 network interface left + default localhost ==> 2 interfaces 4763 assert_eq!( 4764 guest 4765 .ssh_command("ip -o link | wc -l") 4766 .unwrap() 4767 .trim() 4768 .parse::<u32>() 4769 .unwrap_or_default(), 4770 2 4771 ); 4772 4773 // Remove the PCI device 4774 guest 4775 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4776 .unwrap(); 4777 4778 // Back to 2 network interface + default localhost ==> 3 interfaces 4779 assert_eq!( 4780 guest 4781 .ssh_command("ip -o link | wc -l") 4782 .unwrap() 4783 .trim() 4784 .parse::<u32>() 4785 .unwrap_or_default(), 4786 3 4787 ); 4788 4789 let new_bar_addr = guest 4790 .ssh_command( 4791 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4792 ) 4793 .unwrap(); 4794 4795 // Let's compare the BAR addresses for our virtio-net device. 4796 // They should be different as we expect the BAR reprogramming 4797 // to have happened. 4798 assert_ne!(init_bar_addr, new_bar_addr); 4799 }); 4800 4801 let _ = child.kill(); 4802 let output = child.wait_with_output().unwrap(); 4803 4804 handle_child_output(r, &output); 4805 } 4806 4807 #[test] 4808 fn test_memory_mergeable_off() { 4809 test_memory_mergeable(false) 4810 } 4811 4812 #[test] 4813 #[cfg(target_arch = "x86_64")] 4814 fn test_cpu_hotplug() { 4815 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4816 let guest = Guest::new(Box::new(focal)); 4817 let api_socket = temp_api_path(&guest.tmp_dir); 4818 4819 let kernel_path = direct_kernel_boot_path(); 4820 4821 let mut child = GuestCommand::new(&guest) 4822 .args(["--cpus", "boot=2,max=4"]) 4823 .args(["--memory", "size=512M"]) 4824 .args(["--kernel", kernel_path.to_str().unwrap()]) 4825 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4826 .default_disks() 4827 .default_net() 4828 .args(["--api-socket", &api_socket]) 4829 .capture_output() 4830 .spawn() 4831 .unwrap(); 4832 4833 let r = std::panic::catch_unwind(|| { 4834 guest.wait_vm_boot(None).unwrap(); 4835 4836 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4837 4838 // Resize the VM 4839 let desired_vcpus = 4; 4840 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4841 4842 guest 4843 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4844 .unwrap(); 4845 guest 4846 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4847 .unwrap(); 4848 thread::sleep(std::time::Duration::new(10, 0)); 4849 assert_eq!( 4850 guest.get_cpu_count().unwrap_or_default(), 4851 u32::from(desired_vcpus) 4852 ); 4853 4854 guest.reboot_linux(0, None); 4855 4856 assert_eq!( 4857 guest.get_cpu_count().unwrap_or_default(), 4858 u32::from(desired_vcpus) 4859 ); 4860 4861 // Resize the VM 4862 let desired_vcpus = 2; 4863 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4864 4865 thread::sleep(std::time::Duration::new(10, 0)); 4866 assert_eq!( 4867 guest.get_cpu_count().unwrap_or_default(), 4868 u32::from(desired_vcpus) 4869 ); 4870 4871 // Resize the VM back up to 4 4872 let desired_vcpus = 4; 4873 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4874 4875 guest 4876 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4877 .unwrap(); 4878 guest 4879 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4880 .unwrap(); 4881 thread::sleep(std::time::Duration::new(10, 0)); 4882 assert_eq!( 4883 guest.get_cpu_count().unwrap_or_default(), 4884 u32::from(desired_vcpus) 4885 ); 4886 }); 4887 4888 let _ = child.kill(); 4889 let output = child.wait_with_output().unwrap(); 4890 4891 handle_child_output(r, &output); 4892 } 4893 4894 #[test] 4895 fn test_memory_hotplug() { 4896 #[cfg(target_arch = "aarch64")] 4897 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4898 #[cfg(target_arch = "x86_64")] 4899 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4900 let focal = UbuntuDiskConfig::new(focal_image); 4901 let guest = Guest::new(Box::new(focal)); 4902 let api_socket = temp_api_path(&guest.tmp_dir); 4903 4904 #[cfg(target_arch = "aarch64")] 4905 let kernel_path = edk2_path(); 4906 #[cfg(target_arch = "x86_64")] 4907 let kernel_path = direct_kernel_boot_path(); 4908 4909 let mut child = GuestCommand::new(&guest) 4910 .args(["--cpus", "boot=2,max=4"]) 4911 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4912 .args(["--kernel", kernel_path.to_str().unwrap()]) 4913 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4914 .default_disks() 4915 .default_net() 4916 .args(["--balloon", "size=0"]) 4917 .args(["--api-socket", &api_socket]) 4918 .capture_output() 4919 .spawn() 4920 .unwrap(); 4921 4922 let r = std::panic::catch_unwind(|| { 4923 guest.wait_vm_boot(None).unwrap(); 4924 4925 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4926 4927 guest.enable_memory_hotplug(); 4928 4929 // Add RAM to the VM 4930 let desired_ram = 1024 << 20; 4931 resize_command(&api_socket, None, Some(desired_ram), None, None); 4932 4933 thread::sleep(std::time::Duration::new(10, 0)); 4934 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4935 4936 // Use balloon to remove RAM from the VM 4937 let desired_balloon = 512 << 20; 4938 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4939 4940 thread::sleep(std::time::Duration::new(10, 0)); 4941 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4942 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4943 4944 guest.reboot_linux(0, None); 4945 4946 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4947 4948 // Use balloon add RAM to the VM 4949 let desired_balloon = 0; 4950 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4951 4952 thread::sleep(std::time::Duration::new(10, 0)); 4953 4954 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4955 4956 guest.enable_memory_hotplug(); 4957 4958 // Add RAM to the VM 4959 let desired_ram = 2048 << 20; 4960 resize_command(&api_socket, None, Some(desired_ram), None, None); 4961 4962 thread::sleep(std::time::Duration::new(10, 0)); 4963 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4964 4965 // Remove RAM to the VM (only applies after reboot) 4966 let desired_ram = 1024 << 20; 4967 resize_command(&api_socket, None, Some(desired_ram), None, None); 4968 4969 guest.reboot_linux(1, None); 4970 4971 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4972 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4973 }); 4974 4975 let _ = child.kill(); 4976 let output = child.wait_with_output().unwrap(); 4977 4978 handle_child_output(r, &output); 4979 } 4980 4981 #[test] 4982 #[cfg(not(feature = "mshv"))] 4983 fn test_virtio_mem() { 4984 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4985 let guest = Guest::new(Box::new(focal)); 4986 let api_socket = temp_api_path(&guest.tmp_dir); 4987 4988 let kernel_path = direct_kernel_boot_path(); 4989 4990 let mut child = GuestCommand::new(&guest) 4991 .args(["--cpus", "boot=2,max=4"]) 4992 .args([ 4993 "--memory", 4994 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4995 ]) 4996 .args(["--kernel", kernel_path.to_str().unwrap()]) 4997 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4998 .default_disks() 4999 .default_net() 5000 .args(["--api-socket", &api_socket]) 5001 .capture_output() 5002 .spawn() 5003 .unwrap(); 5004 5005 let r = std::panic::catch_unwind(|| { 5006 guest.wait_vm_boot(None).unwrap(); 5007 5008 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5009 5010 guest.enable_memory_hotplug(); 5011 5012 // Add RAM to the VM 5013 let desired_ram = 1024 << 20; 5014 resize_command(&api_socket, None, Some(desired_ram), None, None); 5015 5016 thread::sleep(std::time::Duration::new(10, 0)); 5017 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5018 5019 // Add RAM to the VM 5020 let desired_ram = 2048 << 20; 5021 resize_command(&api_socket, None, Some(desired_ram), None, None); 5022 5023 thread::sleep(std::time::Duration::new(10, 0)); 5024 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 5025 5026 // Remove RAM from the VM 5027 let desired_ram = 1024 << 20; 5028 resize_command(&api_socket, None, Some(desired_ram), None, None); 5029 5030 thread::sleep(std::time::Duration::new(10, 0)); 5031 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5032 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5033 5034 guest.reboot_linux(0, None); 5035 5036 // Check the amount of memory after reboot is 1GiB 5037 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5038 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5039 5040 // Check we can still resize to 512MiB 5041 let desired_ram = 512 << 20; 5042 resize_command(&api_socket, None, Some(desired_ram), None, None); 5043 thread::sleep(std::time::Duration::new(10, 0)); 5044 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5045 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 5046 }); 5047 5048 let _ = child.kill(); 5049 let output = child.wait_with_output().unwrap(); 5050 5051 handle_child_output(r, &output); 5052 } 5053 5054 #[test] 5055 #[cfg(target_arch = "x86_64")] 5056 #[cfg(not(feature = "mshv"))] 5057 // Test both vCPU and memory resizing together 5058 fn test_resize() { 5059 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5060 let guest = Guest::new(Box::new(focal)); 5061 let api_socket = temp_api_path(&guest.tmp_dir); 5062 5063 let kernel_path = direct_kernel_boot_path(); 5064 5065 let mut child = GuestCommand::new(&guest) 5066 .args(["--cpus", "boot=2,max=4"]) 5067 .args(["--memory", "size=512M,hotplug_size=8192M"]) 5068 .args(["--kernel", kernel_path.to_str().unwrap()]) 5069 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5070 .default_disks() 5071 .default_net() 5072 .args(["--api-socket", &api_socket]) 5073 .capture_output() 5074 .spawn() 5075 .unwrap(); 5076 5077 let r = std::panic::catch_unwind(|| { 5078 guest.wait_vm_boot(None).unwrap(); 5079 5080 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 5081 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5082 5083 guest.enable_memory_hotplug(); 5084 5085 // Resize the VM 5086 let desired_vcpus = 4; 5087 let desired_ram = 1024 << 20; 5088 resize_command( 5089 &api_socket, 5090 Some(desired_vcpus), 5091 Some(desired_ram), 5092 None, 5093 None, 5094 ); 5095 5096 guest 5097 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 5098 .unwrap(); 5099 guest 5100 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 5101 .unwrap(); 5102 thread::sleep(std::time::Duration::new(10, 0)); 5103 assert_eq!( 5104 guest.get_cpu_count().unwrap_or_default(), 5105 u32::from(desired_vcpus) 5106 ); 5107 5108 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5109 }); 5110 5111 let _ = child.kill(); 5112 let output = child.wait_with_output().unwrap(); 5113 5114 handle_child_output(r, &output); 5115 } 5116 5117 #[test] 5118 fn test_memory_overhead() { 5119 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5120 let guest = Guest::new(Box::new(focal)); 5121 5122 let kernel_path = direct_kernel_boot_path(); 5123 5124 let guest_memory_size_kb = 512 * 1024; 5125 5126 let mut child = GuestCommand::new(&guest) 5127 .args(["--cpus", "boot=1"]) 5128 .args(["--memory", format!("size={guest_memory_size_kb}K").as_str()]) 5129 .args(["--kernel", kernel_path.to_str().unwrap()]) 5130 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5131 .default_net() 5132 .default_disks() 5133 .capture_output() 5134 .spawn() 5135 .unwrap(); 5136 5137 guest.wait_vm_boot(None).unwrap(); 5138 5139 let r = std::panic::catch_unwind(|| { 5140 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 5141 eprintln!("Guest memory overhead: {overhead} vs {MAXIMUM_VMM_OVERHEAD_KB}"); 5142 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 5143 }); 5144 5145 let _ = child.kill(); 5146 let output = child.wait_with_output().unwrap(); 5147 5148 handle_child_output(r, &output); 5149 } 5150 5151 #[test] 5152 fn test_disk_hotplug() { 5153 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5154 let guest = Guest::new(Box::new(focal)); 5155 5156 #[cfg(target_arch = "x86_64")] 5157 let kernel_path = direct_kernel_boot_path(); 5158 #[cfg(target_arch = "aarch64")] 5159 let kernel_path = edk2_path(); 5160 5161 let api_socket = temp_api_path(&guest.tmp_dir); 5162 5163 let mut child = GuestCommand::new(&guest) 5164 .args(["--api-socket", &api_socket]) 5165 .args(["--cpus", "boot=1"]) 5166 .args(["--memory", "size=512M"]) 5167 .args(["--kernel", kernel_path.to_str().unwrap()]) 5168 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5169 .default_disks() 5170 .default_net() 5171 .capture_output() 5172 .spawn() 5173 .unwrap(); 5174 5175 let r = std::panic::catch_unwind(|| { 5176 guest.wait_vm_boot(None).unwrap(); 5177 5178 // Check /dev/vdc is not there 5179 assert_eq!( 5180 guest 5181 .ssh_command("lsblk | grep -c vdc.*16M || true") 5182 .unwrap() 5183 .trim() 5184 .parse::<u32>() 5185 .unwrap_or(1), 5186 0 5187 ); 5188 5189 // Now let's add the extra disk. 5190 let mut blk_file_path = dirs::home_dir().unwrap(); 5191 blk_file_path.push("workloads"); 5192 blk_file_path.push("blk.img"); 5193 let (cmd_success, cmd_output) = remote_command_w_output( 5194 &api_socket, 5195 "add-disk", 5196 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5197 ); 5198 assert!(cmd_success); 5199 assert!(String::from_utf8_lossy(&cmd_output) 5200 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5201 5202 thread::sleep(std::time::Duration::new(10, 0)); 5203 5204 // Check that /dev/vdc exists and the block size is 16M. 5205 assert_eq!( 5206 guest 5207 .ssh_command("lsblk | grep vdc | grep -c 16M") 5208 .unwrap() 5209 .trim() 5210 .parse::<u32>() 5211 .unwrap_or_default(), 5212 1 5213 ); 5214 // And check the block device can be read. 5215 guest 5216 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5217 .unwrap(); 5218 5219 // Let's remove it the extra disk. 5220 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5221 thread::sleep(std::time::Duration::new(5, 0)); 5222 // And check /dev/vdc is not there 5223 assert_eq!( 5224 guest 5225 .ssh_command("lsblk | grep -c vdc.*16M || true") 5226 .unwrap() 5227 .trim() 5228 .parse::<u32>() 5229 .unwrap_or(1), 5230 0 5231 ); 5232 5233 // And add it back to validate unplug did work correctly. 5234 let (cmd_success, cmd_output) = remote_command_w_output( 5235 &api_socket, 5236 "add-disk", 5237 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5238 ); 5239 assert!(cmd_success); 5240 assert!(String::from_utf8_lossy(&cmd_output) 5241 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5242 5243 thread::sleep(std::time::Duration::new(10, 0)); 5244 5245 // Check that /dev/vdc exists and the block size is 16M. 5246 assert_eq!( 5247 guest 5248 .ssh_command("lsblk | grep vdc | grep -c 16M") 5249 .unwrap() 5250 .trim() 5251 .parse::<u32>() 5252 .unwrap_or_default(), 5253 1 5254 ); 5255 // And check the block device can be read. 5256 guest 5257 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5258 .unwrap(); 5259 5260 // Reboot the VM. 5261 guest.reboot_linux(0, None); 5262 5263 // Check still there after reboot 5264 assert_eq!( 5265 guest 5266 .ssh_command("lsblk | grep vdc | grep -c 16M") 5267 .unwrap() 5268 .trim() 5269 .parse::<u32>() 5270 .unwrap_or_default(), 5271 1 5272 ); 5273 5274 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5275 5276 thread::sleep(std::time::Duration::new(20, 0)); 5277 5278 // Check device has gone away 5279 assert_eq!( 5280 guest 5281 .ssh_command("lsblk | grep -c vdc.*16M || true") 5282 .unwrap() 5283 .trim() 5284 .parse::<u32>() 5285 .unwrap_or(1), 5286 0 5287 ); 5288 5289 guest.reboot_linux(1, None); 5290 5291 // Check device still absent 5292 assert_eq!( 5293 guest 5294 .ssh_command("lsblk | grep -c vdc.*16M || true") 5295 .unwrap() 5296 .trim() 5297 .parse::<u32>() 5298 .unwrap_or(1), 5299 0 5300 ); 5301 }); 5302 5303 let _ = child.kill(); 5304 let output = child.wait_with_output().unwrap(); 5305 5306 handle_child_output(r, &output); 5307 } 5308 5309 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 5310 const LOOP_CONFIGURE: u64 = 0x4c0a; 5311 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 5312 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 5313 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 5314 5315 #[repr(C)] 5316 struct LoopInfo64 { 5317 lo_device: u64, 5318 lo_inode: u64, 5319 lo_rdevice: u64, 5320 lo_offset: u64, 5321 lo_sizelimit: u64, 5322 lo_number: u32, 5323 lo_encrypt_type: u32, 5324 lo_encrypt_key_size: u32, 5325 lo_flags: u32, 5326 lo_file_name: [u8; 64], 5327 lo_crypt_name: [u8; 64], 5328 lo_encrypt_key: [u8; 32], 5329 lo_init: [u64; 2], 5330 } 5331 5332 impl Default for LoopInfo64 { 5333 fn default() -> Self { 5334 LoopInfo64 { 5335 lo_device: 0, 5336 lo_inode: 0, 5337 lo_rdevice: 0, 5338 lo_offset: 0, 5339 lo_sizelimit: 0, 5340 lo_number: 0, 5341 lo_encrypt_type: 0, 5342 lo_encrypt_key_size: 0, 5343 lo_flags: 0, 5344 lo_file_name: [0; 64], 5345 lo_crypt_name: [0; 64], 5346 lo_encrypt_key: [0; 32], 5347 lo_init: [0; 2], 5348 } 5349 } 5350 } 5351 5352 #[derive(Default)] 5353 #[repr(C)] 5354 struct LoopConfig { 5355 fd: u32, 5356 block_size: u32, 5357 info: LoopInfo64, 5358 _reserved: [u64; 8], 5359 } 5360 5361 // Open loop-control device 5362 let loop_ctl_file = OpenOptions::new() 5363 .read(true) 5364 .write(true) 5365 .open(LOOP_CTL_PATH) 5366 .unwrap(); 5367 5368 // Request a free loop device 5369 let loop_device_number = 5370 unsafe { libc::ioctl(loop_ctl_file.as_raw_fd(), LOOP_CTL_GET_FREE as _) }; 5371 5372 if loop_device_number < 0 { 5373 panic!("Couldn't find a free loop device"); 5374 } 5375 5376 // Create loop device path 5377 let loop_device_path = format!("{LOOP_DEVICE_PREFIX}{loop_device_number}"); 5378 5379 // Open loop device 5380 let loop_device_file = OpenOptions::new() 5381 .read(true) 5382 .write(true) 5383 .open(&loop_device_path) 5384 .unwrap(); 5385 5386 // Open backing file 5387 let backing_file = OpenOptions::new() 5388 .read(true) 5389 .write(true) 5390 .open(backing_file_path) 5391 .unwrap(); 5392 5393 let loop_config = LoopConfig { 5394 fd: backing_file.as_raw_fd() as u32, 5395 block_size, 5396 ..Default::default() 5397 }; 5398 5399 for i in 0..num_retries { 5400 let ret = unsafe { 5401 libc::ioctl( 5402 loop_device_file.as_raw_fd(), 5403 LOOP_CONFIGURE as _, 5404 &loop_config, 5405 ) 5406 }; 5407 if ret != 0 { 5408 if i < num_retries - 1 { 5409 println!( 5410 "Iteration {}: Failed to configure the loop device {}: {}", 5411 i, 5412 loop_device_path, 5413 std::io::Error::last_os_error() 5414 ); 5415 } else { 5416 panic!( 5417 "Failed {} times trying to configure the loop device {}: {}", 5418 num_retries, 5419 loop_device_path, 5420 std::io::Error::last_os_error() 5421 ); 5422 } 5423 } else { 5424 break; 5425 } 5426 5427 // Wait for a bit before retrying 5428 thread::sleep(std::time::Duration::new(5, 0)); 5429 } 5430 5431 loop_device_path 5432 } 5433 5434 #[test] 5435 fn test_virtio_block_topology() { 5436 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5437 let guest = Guest::new(Box::new(focal)); 5438 5439 let kernel_path = direct_kernel_boot_path(); 5440 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5441 5442 let output = exec_host_command_output( 5443 format!( 5444 "qemu-img create -f raw {} 16M", 5445 test_disk_path.to_str().unwrap() 5446 ) 5447 .as_str(), 5448 ); 5449 if !output.status.success() { 5450 let stdout = String::from_utf8_lossy(&output.stdout); 5451 let stderr = String::from_utf8_lossy(&output.stderr); 5452 panic!("qemu-img command failed\nstdout\n{stdout}\nstderr\n{stderr}"); 5453 } 5454 5455 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5456 5457 let mut child = GuestCommand::new(&guest) 5458 .args(["--cpus", "boot=1"]) 5459 .args(["--memory", "size=512M"]) 5460 .args(["--kernel", kernel_path.to_str().unwrap()]) 5461 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5462 .args([ 5463 "--disk", 5464 format!( 5465 "path={}", 5466 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5467 ) 5468 .as_str(), 5469 format!( 5470 "path={}", 5471 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5472 ) 5473 .as_str(), 5474 format!("path={}", &loop_dev).as_str(), 5475 ]) 5476 .default_net() 5477 .capture_output() 5478 .spawn() 5479 .unwrap(); 5480 5481 let r = std::panic::catch_unwind(|| { 5482 guest.wait_vm_boot(None).unwrap(); 5483 5484 // MIN-IO column 5485 assert_eq!( 5486 guest 5487 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5488 .unwrap() 5489 .trim() 5490 .parse::<u32>() 5491 .unwrap_or_default(), 5492 4096 5493 ); 5494 // PHY-SEC column 5495 assert_eq!( 5496 guest 5497 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5498 .unwrap() 5499 .trim() 5500 .parse::<u32>() 5501 .unwrap_or_default(), 5502 4096 5503 ); 5504 // LOG-SEC column 5505 assert_eq!( 5506 guest 5507 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5508 .unwrap() 5509 .trim() 5510 .parse::<u32>() 5511 .unwrap_or_default(), 5512 4096 5513 ); 5514 }); 5515 5516 let _ = child.kill(); 5517 let output = child.wait_with_output().unwrap(); 5518 5519 handle_child_output(r, &output); 5520 5521 Command::new("losetup") 5522 .args(["-d", &loop_dev]) 5523 .output() 5524 .expect("loop device not found"); 5525 } 5526 5527 #[test] 5528 fn test_virtio_balloon_deflate_on_oom() { 5529 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5530 let guest = Guest::new(Box::new(focal)); 5531 5532 let kernel_path = direct_kernel_boot_path(); 5533 5534 let api_socket = temp_api_path(&guest.tmp_dir); 5535 5536 //Let's start a 4G guest with balloon occupied 2G memory 5537 let mut child = GuestCommand::new(&guest) 5538 .args(["--api-socket", &api_socket]) 5539 .args(["--cpus", "boot=1"]) 5540 .args(["--memory", "size=4G"]) 5541 .args(["--kernel", kernel_path.to_str().unwrap()]) 5542 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5543 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5544 .default_disks() 5545 .default_net() 5546 .capture_output() 5547 .spawn() 5548 .unwrap(); 5549 5550 let r = std::panic::catch_unwind(|| { 5551 guest.wait_vm_boot(None).unwrap(); 5552 5553 // Wait for balloon memory's initialization and check its size. 5554 // The virtio-balloon driver might take a few seconds to report the 5555 // balloon effective size back to the VMM. 5556 thread::sleep(std::time::Duration::new(20, 0)); 5557 5558 let orig_balloon = balloon_size(&api_socket); 5559 println!("The original balloon memory size is {orig_balloon} bytes"); 5560 assert!(orig_balloon == 2147483648); 5561 5562 // Two steps to verify if the 'deflate_on_oom' parameter works. 5563 // 1st: run a command to trigger an OOM in the guest. 5564 guest 5565 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5566 .unwrap(); 5567 5568 // Give some time for the OOM to happen in the guest and be reported 5569 // back to the host. 5570 thread::sleep(std::time::Duration::new(20, 0)); 5571 5572 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5573 let deflated_balloon = balloon_size(&api_socket); 5574 println!("After deflating, balloon memory size is {deflated_balloon} bytes"); 5575 // Verify the balloon size deflated 5576 assert!(deflated_balloon < 2147483648); 5577 }); 5578 5579 let _ = child.kill(); 5580 let output = child.wait_with_output().unwrap(); 5581 5582 handle_child_output(r, &output); 5583 } 5584 5585 #[test] 5586 #[cfg(not(feature = "mshv"))] 5587 fn test_virtio_balloon_free_page_reporting() { 5588 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5589 let guest = Guest::new(Box::new(focal)); 5590 5591 //Let's start a 4G guest with balloon occupied 2G memory 5592 let mut child = GuestCommand::new(&guest) 5593 .args(["--cpus", "boot=1"]) 5594 .args(["--memory", "size=4G"]) 5595 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5596 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5597 .args(["--balloon", "size=0,free_page_reporting=on"]) 5598 .default_disks() 5599 .default_net() 5600 .capture_output() 5601 .spawn() 5602 .unwrap(); 5603 5604 let pid = child.id(); 5605 let r = std::panic::catch_unwind(|| { 5606 guest.wait_vm_boot(None).unwrap(); 5607 5608 // Check the initial RSS is less than 1GiB 5609 let rss = process_rss_kib(pid); 5610 println!("RSS {rss} < 1048576"); 5611 assert!(rss < 1048576); 5612 5613 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5614 // seconds 5615 let guest_ip = guest.network.guest_ip.clone(); 5616 thread::spawn(move || { 5617 ssh_command_ip( 5618 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5619 &guest_ip, 5620 DEFAULT_SSH_RETRIES, 5621 DEFAULT_SSH_TIMEOUT, 5622 ) 5623 .unwrap(); 5624 }); 5625 5626 // Wait for 50 seconds to make sure the stress command is consuming 5627 // the expected amount of memory. 5628 thread::sleep(std::time::Duration::new(50, 0)); 5629 let rss = process_rss_kib(pid); 5630 println!("RSS {rss} >= 2097152"); 5631 assert!(rss >= 2097152); 5632 5633 // Wait for an extra minute to make sure the stress command has 5634 // completed and that the guest reported the free pages to the VMM 5635 // through the virtio-balloon device. We expect the RSS to be under 5636 // 2GiB. 5637 thread::sleep(std::time::Duration::new(60, 0)); 5638 let rss = process_rss_kib(pid); 5639 println!("RSS {rss} < 2097152"); 5640 assert!(rss < 2097152); 5641 }); 5642 5643 let _ = child.kill(); 5644 let output = child.wait_with_output().unwrap(); 5645 5646 handle_child_output(r, &output); 5647 } 5648 5649 #[test] 5650 fn test_pmem_hotplug() { 5651 _test_pmem_hotplug(None) 5652 } 5653 5654 #[test] 5655 fn test_pmem_multi_segment_hotplug() { 5656 _test_pmem_hotplug(Some(15)) 5657 } 5658 5659 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5660 #[cfg(target_arch = "aarch64")] 5661 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5662 #[cfg(target_arch = "x86_64")] 5663 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5664 let focal = UbuntuDiskConfig::new(focal_image); 5665 let guest = Guest::new(Box::new(focal)); 5666 5667 #[cfg(target_arch = "x86_64")] 5668 let kernel_path = direct_kernel_boot_path(); 5669 #[cfg(target_arch = "aarch64")] 5670 let kernel_path = edk2_path(); 5671 5672 let api_socket = temp_api_path(&guest.tmp_dir); 5673 5674 let mut cmd = GuestCommand::new(&guest); 5675 5676 cmd.args(["--api-socket", &api_socket]) 5677 .args(["--cpus", "boot=1"]) 5678 .args(["--memory", "size=512M"]) 5679 .args(["--kernel", kernel_path.to_str().unwrap()]) 5680 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5681 .default_disks() 5682 .default_net() 5683 .capture_output(); 5684 5685 if pci_segment.is_some() { 5686 cmd.args([ 5687 "--platform", 5688 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5689 ]); 5690 } 5691 5692 let mut child = cmd.spawn().unwrap(); 5693 5694 let r = std::panic::catch_unwind(|| { 5695 guest.wait_vm_boot(None).unwrap(); 5696 5697 // Check /dev/pmem0 is not there 5698 assert_eq!( 5699 guest 5700 .ssh_command("lsblk | grep -c pmem0 || true") 5701 .unwrap() 5702 .trim() 5703 .parse::<u32>() 5704 .unwrap_or(1), 5705 0 5706 ); 5707 5708 let pmem_temp_file = TempFile::new().unwrap(); 5709 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5710 let (cmd_success, cmd_output) = remote_command_w_output( 5711 &api_socket, 5712 "add-pmem", 5713 Some(&format!( 5714 "file={},id=test0{}", 5715 pmem_temp_file.as_path().to_str().unwrap(), 5716 if let Some(pci_segment) = pci_segment { 5717 format!(",pci_segment={pci_segment}") 5718 } else { 5719 "".to_owned() 5720 } 5721 )), 5722 ); 5723 assert!(cmd_success); 5724 if let Some(pci_segment) = pci_segment { 5725 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5726 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5727 ))); 5728 } else { 5729 assert!(String::from_utf8_lossy(&cmd_output) 5730 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5731 } 5732 5733 // Check that /dev/pmem0 exists and the block size is 128M 5734 assert_eq!( 5735 guest 5736 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5737 .unwrap() 5738 .trim() 5739 .parse::<u32>() 5740 .unwrap_or_default(), 5741 1 5742 ); 5743 5744 guest.reboot_linux(0, None); 5745 5746 // Check still there after reboot 5747 assert_eq!( 5748 guest 5749 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5750 .unwrap() 5751 .trim() 5752 .parse::<u32>() 5753 .unwrap_or_default(), 5754 1 5755 ); 5756 5757 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5758 5759 thread::sleep(std::time::Duration::new(20, 0)); 5760 5761 // Check device has gone away 5762 assert_eq!( 5763 guest 5764 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5765 .unwrap() 5766 .trim() 5767 .parse::<u32>() 5768 .unwrap_or(1), 5769 0 5770 ); 5771 5772 guest.reboot_linux(1, None); 5773 5774 // Check still absent after reboot 5775 assert_eq!( 5776 guest 5777 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5778 .unwrap() 5779 .trim() 5780 .parse::<u32>() 5781 .unwrap_or(1), 5782 0 5783 ); 5784 }); 5785 5786 let _ = child.kill(); 5787 let output = child.wait_with_output().unwrap(); 5788 5789 handle_child_output(r, &output); 5790 } 5791 5792 #[test] 5793 fn test_net_hotplug() { 5794 _test_net_hotplug(None) 5795 } 5796 5797 #[test] 5798 fn test_net_multi_segment_hotplug() { 5799 _test_net_hotplug(Some(15)) 5800 } 5801 5802 fn _test_net_hotplug(pci_segment: Option<u16>) { 5803 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5804 let guest = Guest::new(Box::new(focal)); 5805 5806 #[cfg(target_arch = "x86_64")] 5807 let kernel_path = direct_kernel_boot_path(); 5808 #[cfg(target_arch = "aarch64")] 5809 let kernel_path = edk2_path(); 5810 5811 let api_socket = temp_api_path(&guest.tmp_dir); 5812 5813 // Boot without network 5814 let mut cmd = GuestCommand::new(&guest); 5815 5816 cmd.args(["--api-socket", &api_socket]) 5817 .args(["--cpus", "boot=1"]) 5818 .args(["--memory", "size=512M"]) 5819 .args(["--kernel", kernel_path.to_str().unwrap()]) 5820 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5821 .default_disks() 5822 .capture_output(); 5823 5824 if pci_segment.is_some() { 5825 cmd.args([ 5826 "--platform", 5827 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5828 ]); 5829 } 5830 5831 let mut child = cmd.spawn().unwrap(); 5832 5833 thread::sleep(std::time::Duration::new(20, 0)); 5834 5835 let r = std::panic::catch_unwind(|| { 5836 // Add network 5837 let (cmd_success, cmd_output) = remote_command_w_output( 5838 &api_socket, 5839 "add-net", 5840 Some( 5841 format!( 5842 "{}{},id=test0", 5843 guest.default_net_string(), 5844 if let Some(pci_segment) = pci_segment { 5845 format!(",pci_segment={pci_segment}") 5846 } else { 5847 "".to_owned() 5848 } 5849 ) 5850 .as_str(), 5851 ), 5852 ); 5853 assert!(cmd_success); 5854 5855 if let Some(pci_segment) = pci_segment { 5856 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5857 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5858 ))); 5859 } else { 5860 assert!(String::from_utf8_lossy(&cmd_output) 5861 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5862 } 5863 5864 thread::sleep(std::time::Duration::new(5, 0)); 5865 5866 // 1 network interfaces + default localhost ==> 2 interfaces 5867 assert_eq!( 5868 guest 5869 .ssh_command("ip -o link | wc -l") 5870 .unwrap() 5871 .trim() 5872 .parse::<u32>() 5873 .unwrap_or_default(), 5874 2 5875 ); 5876 5877 // Remove network 5878 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5879 thread::sleep(std::time::Duration::new(5, 0)); 5880 5881 let (cmd_success, cmd_output) = remote_command_w_output( 5882 &api_socket, 5883 "add-net", 5884 Some( 5885 format!( 5886 "{}{},id=test1", 5887 guest.default_net_string(), 5888 if let Some(pci_segment) = pci_segment { 5889 format!(",pci_segment={pci_segment}") 5890 } else { 5891 "".to_owned() 5892 } 5893 ) 5894 .as_str(), 5895 ), 5896 ); 5897 assert!(cmd_success); 5898 5899 if let Some(pci_segment) = pci_segment { 5900 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5901 "{{\"id\":\"test1\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5902 ))); 5903 } else { 5904 assert!(String::from_utf8_lossy(&cmd_output) 5905 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5906 } 5907 5908 thread::sleep(std::time::Duration::new(5, 0)); 5909 5910 // 1 network interfaces + default localhost ==> 2 interfaces 5911 assert_eq!( 5912 guest 5913 .ssh_command("ip -o link | wc -l") 5914 .unwrap() 5915 .trim() 5916 .parse::<u32>() 5917 .unwrap_or_default(), 5918 2 5919 ); 5920 5921 guest.reboot_linux(0, None); 5922 5923 // Check still there after reboot 5924 // 1 network interfaces + default localhost ==> 2 interfaces 5925 assert_eq!( 5926 guest 5927 .ssh_command("ip -o link | wc -l") 5928 .unwrap() 5929 .trim() 5930 .parse::<u32>() 5931 .unwrap_or_default(), 5932 2 5933 ); 5934 }); 5935 5936 let _ = child.kill(); 5937 let output = child.wait_with_output().unwrap(); 5938 5939 handle_child_output(r, &output); 5940 } 5941 5942 #[test] 5943 fn test_initramfs() { 5944 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5945 let guest = Guest::new(Box::new(focal)); 5946 let mut workload_path = dirs::home_dir().unwrap(); 5947 workload_path.push("workloads"); 5948 5949 #[cfg(target_arch = "x86_64")] 5950 let mut kernels = vec![direct_kernel_boot_path()]; 5951 #[cfg(target_arch = "aarch64")] 5952 let kernels = [direct_kernel_boot_path()]; 5953 5954 #[cfg(target_arch = "x86_64")] 5955 { 5956 let mut pvh_kernel_path = workload_path.clone(); 5957 pvh_kernel_path.push("vmlinux"); 5958 kernels.push(pvh_kernel_path); 5959 } 5960 5961 let mut initramfs_path = workload_path; 5962 initramfs_path.push("alpine_initramfs.img"); 5963 5964 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5965 let cmdline = format!("console=hvc0 quiet TEST_STRING={test_string}"); 5966 5967 kernels.iter().for_each(|k_path| { 5968 let mut child = GuestCommand::new(&guest) 5969 .args(["--kernel", k_path.to_str().unwrap()]) 5970 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 5971 .args(["--cmdline", &cmdline]) 5972 .capture_output() 5973 .spawn() 5974 .unwrap(); 5975 5976 thread::sleep(std::time::Duration::new(20, 0)); 5977 5978 let _ = child.kill(); 5979 let output = child.wait_with_output().unwrap(); 5980 5981 let r = std::panic::catch_unwind(|| { 5982 let s = String::from_utf8_lossy(&output.stdout); 5983 5984 assert_ne!(s.lines().position(|line| line == test_string), None); 5985 }); 5986 5987 handle_child_output(r, &output); 5988 }); 5989 } 5990 5991 #[test] 5992 fn test_counters() { 5993 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5994 let guest = Guest::new(Box::new(focal)); 5995 let api_socket = temp_api_path(&guest.tmp_dir); 5996 5997 let mut cmd = GuestCommand::new(&guest); 5998 cmd.args(["--cpus", "boot=1"]) 5999 .args(["--memory", "size=512M"]) 6000 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 6001 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6002 .default_disks() 6003 .args(["--net", guest.default_net_string().as_str()]) 6004 .args(["--api-socket", &api_socket]) 6005 .capture_output(); 6006 6007 let mut child = cmd.spawn().unwrap(); 6008 6009 let r = std::panic::catch_unwind(|| { 6010 guest.wait_vm_boot(None).unwrap(); 6011 6012 let orig_counters = get_counters(&api_socket); 6013 guest 6014 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 6015 .unwrap(); 6016 6017 let new_counters = get_counters(&api_socket); 6018 6019 // Check that all the counters have increased 6020 assert!(new_counters > orig_counters); 6021 }); 6022 6023 let _ = child.kill(); 6024 let output = child.wait_with_output().unwrap(); 6025 6026 handle_child_output(r, &output); 6027 } 6028 6029 #[test] 6030 #[cfg(feature = "guest_debug")] 6031 fn test_coredump() { 6032 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6033 let guest = Guest::new(Box::new(focal)); 6034 let api_socket = temp_api_path(&guest.tmp_dir); 6035 6036 let mut cmd = GuestCommand::new(&guest); 6037 cmd.args(["--cpus", "boot=4"]) 6038 .args(["--memory", "size=4G"]) 6039 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6040 .default_disks() 6041 .args(["--net", guest.default_net_string().as_str()]) 6042 .args(["--api-socket", &api_socket]) 6043 .capture_output(); 6044 6045 let mut child = cmd.spawn().unwrap(); 6046 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6047 6048 let r = std::panic::catch_unwind(|| { 6049 guest.wait_vm_boot(None).unwrap(); 6050 6051 assert!(remote_command(&api_socket, "pause", None)); 6052 6053 assert!(remote_command( 6054 &api_socket, 6055 "coredump", 6056 Some(format!("file://{vmcore_file}").as_str()), 6057 )); 6058 6059 // the num of CORE notes should equals to vcpu 6060 let readelf_core_num_cmd = 6061 format!("readelf --all {vmcore_file} |grep CORE |grep -v Type |wc -l"); 6062 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 6063 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 6064 6065 // the num of QEMU notes should equals to vcpu 6066 let readelf_vmm_num_cmd = format!("readelf --all {vmcore_file} |grep QEMU |wc -l"); 6067 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 6068 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 6069 }); 6070 6071 let _ = child.kill(); 6072 let output = child.wait_with_output().unwrap(); 6073 6074 handle_child_output(r, &output); 6075 } 6076 6077 #[test] 6078 #[cfg(feature = "guest_debug")] 6079 fn test_coredump_no_pause() { 6080 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6081 let guest = Guest::new(Box::new(focal)); 6082 let api_socket = temp_api_path(&guest.tmp_dir); 6083 6084 let mut cmd = GuestCommand::new(&guest); 6085 cmd.args(["--cpus", "boot=4"]) 6086 .args(["--memory", "size=4G"]) 6087 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6088 .default_disks() 6089 .args(["--net", guest.default_net_string().as_str()]) 6090 .args(["--api-socket", &api_socket]) 6091 .capture_output(); 6092 6093 let mut child = cmd.spawn().unwrap(); 6094 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6095 6096 let r = std::panic::catch_unwind(|| { 6097 guest.wait_vm_boot(None).unwrap(); 6098 6099 assert!(remote_command( 6100 &api_socket, 6101 "coredump", 6102 Some(format!("file://{vmcore_file}").as_str()), 6103 )); 6104 6105 assert_eq!(vm_state(&api_socket), "Running"); 6106 }); 6107 6108 let _ = child.kill(); 6109 let output = child.wait_with_output().unwrap(); 6110 6111 handle_child_output(r, &output); 6112 } 6113 6114 #[test] 6115 fn test_watchdog() { 6116 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6117 let guest = Guest::new(Box::new(focal)); 6118 let api_socket = temp_api_path(&guest.tmp_dir); 6119 6120 let kernel_path = direct_kernel_boot_path(); 6121 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6122 6123 let mut cmd = GuestCommand::new(&guest); 6124 cmd.args(["--cpus", "boot=1"]) 6125 .args(["--memory", "size=512M"]) 6126 .args(["--kernel", kernel_path.to_str().unwrap()]) 6127 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6128 .default_disks() 6129 .args(["--net", guest.default_net_string().as_str()]) 6130 .args(["--watchdog"]) 6131 .args(["--api-socket", &api_socket]) 6132 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6133 .capture_output(); 6134 6135 let mut child = cmd.spawn().unwrap(); 6136 6137 let r = std::panic::catch_unwind(|| { 6138 guest.wait_vm_boot(None).unwrap(); 6139 6140 let mut expected_reboot_count = 1; 6141 6142 // Enable the watchdog with a 15s timeout 6143 enable_guest_watchdog(&guest, 15); 6144 6145 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6146 assert_eq!( 6147 guest 6148 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6149 .unwrap() 6150 .trim() 6151 .parse::<u32>() 6152 .unwrap_or_default(), 6153 1 6154 ); 6155 6156 // Allow some normal time to elapse to check we don't get spurious reboots 6157 thread::sleep(std::time::Duration::new(40, 0)); 6158 // Check no reboot 6159 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6160 6161 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6162 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6163 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6164 guest.wait_vm_boot(Some(50)).unwrap(); 6165 // Check a reboot is triggered by the watchdog 6166 expected_reboot_count += 1; 6167 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6168 6169 #[cfg(target_arch = "x86_64")] 6170 { 6171 // Now pause the VM and remain offline for 30s 6172 assert!(remote_command(&api_socket, "pause", None)); 6173 let latest_events = [ 6174 &MetaEvent { 6175 event: "pausing".to_string(), 6176 device_id: None, 6177 }, 6178 &MetaEvent { 6179 event: "paused".to_string(), 6180 device_id: None, 6181 }, 6182 ]; 6183 assert!(check_latest_events_exact(&latest_events, &event_path)); 6184 assert!(remote_command(&api_socket, "resume", None)); 6185 6186 // Check no reboot 6187 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6188 } 6189 }); 6190 6191 let _ = child.kill(); 6192 let output = child.wait_with_output().unwrap(); 6193 6194 handle_child_output(r, &output); 6195 } 6196 6197 #[test] 6198 fn test_pvpanic() { 6199 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 6200 let guest = Guest::new(Box::new(jammy)); 6201 let api_socket = temp_api_path(&guest.tmp_dir); 6202 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6203 6204 let kernel_path = direct_kernel_boot_path(); 6205 6206 let mut cmd = GuestCommand::new(&guest); 6207 cmd.args(["--cpus", "boot=1"]) 6208 .args(["--memory", "size=512M"]) 6209 .args(["--kernel", kernel_path.to_str().unwrap()]) 6210 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6211 .default_disks() 6212 .args(["--net", guest.default_net_string().as_str()]) 6213 .args(["--pvpanic"]) 6214 .args(["--api-socket", &api_socket]) 6215 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6216 .capture_output(); 6217 6218 let mut child = cmd.spawn().unwrap(); 6219 6220 let r = std::panic::catch_unwind(|| { 6221 guest.wait_vm_boot(None).unwrap(); 6222 6223 // Trigger guest a panic 6224 make_guest_panic(&guest); 6225 6226 // Wait a while for guest 6227 thread::sleep(std::time::Duration::new(10, 0)); 6228 6229 let expected_sequential_events = [&MetaEvent { 6230 event: "panic".to_string(), 6231 device_id: None, 6232 }]; 6233 assert!(check_latest_events_exact( 6234 &expected_sequential_events, 6235 &event_path 6236 )); 6237 }); 6238 6239 let _ = child.kill(); 6240 let output = child.wait_with_output().unwrap(); 6241 6242 handle_child_output(r, &output); 6243 } 6244 6245 #[test] 6246 fn test_tap_from_fd() { 6247 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6248 let guest = Guest::new(Box::new(focal)); 6249 let kernel_path = direct_kernel_boot_path(); 6250 6251 // Create a TAP interface with multi-queue enabled 6252 let num_queue_pairs: usize = 2; 6253 6254 use std::str::FromStr; 6255 let taps = net_util::open_tap( 6256 Some("chtap0"), 6257 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6258 None, 6259 &mut None, 6260 None, 6261 num_queue_pairs, 6262 Some(libc::O_RDWR | libc::O_NONBLOCK), 6263 ) 6264 .unwrap(); 6265 6266 let mut child = GuestCommand::new(&guest) 6267 .args(["--cpus", &format!("boot={num_queue_pairs}")]) 6268 .args(["--memory", "size=512M"]) 6269 .args(["--kernel", kernel_path.to_str().unwrap()]) 6270 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6271 .default_disks() 6272 .args([ 6273 "--net", 6274 &format!( 6275 "fd=[{},{}],mac={},num_queues={}", 6276 taps[0].as_raw_fd(), 6277 taps[1].as_raw_fd(), 6278 guest.network.guest_mac, 6279 num_queue_pairs * 2 6280 ), 6281 ]) 6282 .capture_output() 6283 .spawn() 6284 .unwrap(); 6285 6286 let r = std::panic::catch_unwind(|| { 6287 guest.wait_vm_boot(None).unwrap(); 6288 6289 assert_eq!( 6290 guest 6291 .ssh_command("ip -o link | wc -l") 6292 .unwrap() 6293 .trim() 6294 .parse::<u32>() 6295 .unwrap_or_default(), 6296 2 6297 ); 6298 6299 guest.reboot_linux(0, None); 6300 6301 assert_eq!( 6302 guest 6303 .ssh_command("ip -o link | wc -l") 6304 .unwrap() 6305 .trim() 6306 .parse::<u32>() 6307 .unwrap_or_default(), 6308 2 6309 ); 6310 }); 6311 6312 let _ = child.kill(); 6313 let output = child.wait_with_output().unwrap(); 6314 6315 handle_child_output(r, &output); 6316 } 6317 6318 // By design, a guest VM won't be able to connect to the host 6319 // machine when using a macvtap network interface (while it can 6320 // communicate externally). As a workaround, this integration 6321 // test creates two macvtap interfaces in 'bridge' mode on the 6322 // same physical net interface, one for the guest and one for 6323 // the host. With additional setup on the IP address and the 6324 // routing table, it enables the communications between the 6325 // guest VM and the host machine. 6326 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6327 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6328 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6329 let guest = Guest::new(Box::new(focal)); 6330 let api_socket = temp_api_path(&guest.tmp_dir); 6331 6332 #[cfg(target_arch = "x86_64")] 6333 let kernel_path = direct_kernel_boot_path(); 6334 #[cfg(target_arch = "aarch64")] 6335 let kernel_path = edk2_path(); 6336 6337 let phy_net = "eth0"; 6338 6339 // Create a macvtap interface for the guest VM to use 6340 assert!(exec_host_command_status(&format!( 6341 "sudo ip link add link {phy_net} name {guest_macvtap_name} type macvtap mod bridge" 6342 )) 6343 .success()); 6344 assert!(exec_host_command_status(&format!( 6345 "sudo ip link set {} address {} up", 6346 guest_macvtap_name, guest.network.guest_mac 6347 )) 6348 .success()); 6349 assert!( 6350 exec_host_command_status(&format!("sudo ip link show {guest_macvtap_name}")).success() 6351 ); 6352 6353 let tap_index = 6354 fs::read_to_string(format!("/sys/class/net/{guest_macvtap_name}/ifindex")).unwrap(); 6355 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6356 6357 assert!(exec_host_command_status(&format!("sudo chown $UID.$UID {tap_device}")).success()); 6358 6359 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6360 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6361 assert!(tap_fd1 > 0); 6362 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6363 assert!(tap_fd2 > 0); 6364 6365 // Create a macvtap on the same physical net interface for 6366 // the host machine to use 6367 assert!(exec_host_command_status(&format!( 6368 "sudo ip link add link {phy_net} name {host_macvtap_name} type macvtap mod bridge" 6369 )) 6370 .success()); 6371 // Use default mask "255.255.255.0" 6372 assert!(exec_host_command_status(&format!( 6373 "sudo ip address add {}/24 dev {}", 6374 guest.network.host_ip, host_macvtap_name 6375 )) 6376 .success()); 6377 assert!( 6378 exec_host_command_status(&format!("sudo ip link set dev {host_macvtap_name} up")) 6379 .success() 6380 ); 6381 6382 let mut guest_command = GuestCommand::new(&guest); 6383 guest_command 6384 .args(["--cpus", "boot=2"]) 6385 .args(["--memory", "size=512M"]) 6386 .args(["--kernel", kernel_path.to_str().unwrap()]) 6387 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6388 .default_disks() 6389 .args(["--api-socket", &api_socket]); 6390 6391 let net_params = format!( 6392 "fd=[{},{}],mac={},num_queues=4", 6393 tap_fd1, tap_fd2, guest.network.guest_mac 6394 ); 6395 6396 if !hotplug { 6397 guest_command.args(["--net", &net_params]); 6398 } 6399 6400 let mut child = guest_command.capture_output().spawn().unwrap(); 6401 6402 if hotplug { 6403 // Give some time to the VMM process to listen to the API 6404 // socket. This is the only requirement to avoid the following 6405 // call to ch-remote from failing. 6406 thread::sleep(std::time::Duration::new(10, 0)); 6407 // Hotplug the virtio-net device 6408 let (cmd_success, cmd_output) = 6409 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6410 assert!(cmd_success); 6411 #[cfg(target_arch = "x86_64")] 6412 assert!(String::from_utf8_lossy(&cmd_output) 6413 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6414 #[cfg(target_arch = "aarch64")] 6415 assert!(String::from_utf8_lossy(&cmd_output) 6416 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6417 } 6418 6419 // The functional connectivity provided by the virtio-net device 6420 // gets tested through wait_vm_boot() as it expects to receive a 6421 // HTTP request, and through the SSH command as well. 6422 let r = std::panic::catch_unwind(|| { 6423 guest.wait_vm_boot(None).unwrap(); 6424 6425 assert_eq!( 6426 guest 6427 .ssh_command("ip -o link | wc -l") 6428 .unwrap() 6429 .trim() 6430 .parse::<u32>() 6431 .unwrap_or_default(), 6432 2 6433 ); 6434 6435 guest.reboot_linux(0, None); 6436 6437 assert_eq!( 6438 guest 6439 .ssh_command("ip -o link | wc -l") 6440 .unwrap() 6441 .trim() 6442 .parse::<u32>() 6443 .unwrap_or_default(), 6444 2 6445 ); 6446 }); 6447 6448 let _ = child.kill(); 6449 6450 exec_host_command_status(&format!("sudo ip link del {guest_macvtap_name}")); 6451 exec_host_command_status(&format!("sudo ip link del {host_macvtap_name}")); 6452 6453 let output = child.wait_with_output().unwrap(); 6454 6455 handle_child_output(r, &output); 6456 } 6457 6458 #[test] 6459 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6460 fn test_macvtap() { 6461 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6462 } 6463 6464 #[test] 6465 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6466 fn test_macvtap_hotplug() { 6467 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6468 } 6469 6470 #[test] 6471 #[cfg(not(feature = "mshv"))] 6472 fn test_ovs_dpdk() { 6473 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6474 let guest1 = Guest::new(Box::new(focal1)); 6475 6476 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6477 let guest2 = Guest::new(Box::new(focal2)); 6478 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6479 6480 let (mut child1, mut child2) = 6481 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6482 6483 // Create the snapshot directory 6484 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6485 6486 let r = std::panic::catch_unwind(|| { 6487 // Remove one of the two ports from the OVS bridge 6488 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6489 6490 // Spawn a new netcat listener in the first VM 6491 let guest_ip = guest1.network.guest_ip.clone(); 6492 thread::spawn(move || { 6493 ssh_command_ip( 6494 "nc -l 12345", 6495 &guest_ip, 6496 DEFAULT_SSH_RETRIES, 6497 DEFAULT_SSH_TIMEOUT, 6498 ) 6499 .unwrap(); 6500 }); 6501 6502 // Wait for the server to be listening 6503 thread::sleep(std::time::Duration::new(5, 0)); 6504 6505 // Check the connection fails this time 6506 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6507 6508 // Add the OVS port back 6509 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6510 6511 // And finally check the connection is functional again 6512 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6513 6514 // Pause the VM 6515 assert!(remote_command(&api_socket_source, "pause", None)); 6516 6517 // Take a snapshot from the VM 6518 assert!(remote_command( 6519 &api_socket_source, 6520 "snapshot", 6521 Some(format!("file://{snapshot_dir}").as_str()), 6522 )); 6523 6524 // Wait to make sure the snapshot is completed 6525 thread::sleep(std::time::Duration::new(10, 0)); 6526 }); 6527 6528 // Shutdown the source VM 6529 let _ = child2.kill(); 6530 let output = child2.wait_with_output().unwrap(); 6531 handle_child_output(r, &output); 6532 6533 // Remove the vhost-user socket file. 6534 Command::new("rm") 6535 .arg("-f") 6536 .arg("/tmp/dpdkvhostclient2") 6537 .output() 6538 .unwrap(); 6539 6540 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6541 // Restore the VM from the snapshot 6542 let mut child2 = GuestCommand::new(&guest2) 6543 .args(["--api-socket", &api_socket_restored]) 6544 .args([ 6545 "--restore", 6546 format!("source_url=file://{snapshot_dir}").as_str(), 6547 ]) 6548 .capture_output() 6549 .spawn() 6550 .unwrap(); 6551 6552 // Wait for the VM to be restored 6553 thread::sleep(std::time::Duration::new(10, 0)); 6554 6555 let r = std::panic::catch_unwind(|| { 6556 // Resume the VM 6557 assert!(remote_command(&api_socket_restored, "resume", None)); 6558 6559 // Spawn a new netcat listener in the first VM 6560 let guest_ip = guest1.network.guest_ip.clone(); 6561 thread::spawn(move || { 6562 ssh_command_ip( 6563 "nc -l 12345", 6564 &guest_ip, 6565 DEFAULT_SSH_RETRIES, 6566 DEFAULT_SSH_TIMEOUT, 6567 ) 6568 .unwrap(); 6569 }); 6570 6571 // Wait for the server to be listening 6572 thread::sleep(std::time::Duration::new(5, 0)); 6573 6574 // And check the connection is still functional after restore 6575 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6576 }); 6577 6578 let _ = child1.kill(); 6579 let _ = child2.kill(); 6580 6581 let output = child1.wait_with_output().unwrap(); 6582 child2.wait().unwrap(); 6583 6584 cleanup_ovs_dpdk(); 6585 6586 handle_child_output(r, &output); 6587 } 6588 6589 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6590 cleanup_spdk_nvme(); 6591 6592 assert!(exec_host_command_status(&format!( 6593 "mkdir -p {}", 6594 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6595 )) 6596 .success()); 6597 assert!(exec_host_command_status(&format!( 6598 "truncate {} -s 128M", 6599 nvme_dir.join("test-disk.raw").to_str().unwrap() 6600 )) 6601 .success()); 6602 assert!(exec_host_command_status(&format!( 6603 "mkfs.ext4 {}", 6604 nvme_dir.join("test-disk.raw").to_str().unwrap() 6605 )) 6606 .success()); 6607 6608 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6609 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6610 .args(["-i", "0", "-m", "0x1"]) 6611 .spawn() 6612 .unwrap(); 6613 thread::sleep(std::time::Duration::new(2, 0)); 6614 6615 assert!(exec_host_command_with_retries( 6616 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER", 6617 3, 6618 std::time::Duration::new(5, 0), 6619 )); 6620 assert!(exec_host_command_status(&format!( 6621 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6622 nvme_dir.join("test-disk.raw").to_str().unwrap() 6623 )) 6624 .success()); 6625 assert!(exec_host_command_status( 6626 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6627 ) 6628 .success()); 6629 assert!(exec_host_command_status( 6630 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6631 ) 6632 .success()); 6633 assert!(exec_host_command_status(&format!( 6634 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6635 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6636 )) 6637 .success()); 6638 } 6639 6640 fn cleanup_spdk_nvme() { 6641 exec_host_command_status("pkill -f nvmf_tgt"); 6642 } 6643 6644 #[test] 6645 fn test_vfio_user() { 6646 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6647 let jammy = UbuntuDiskConfig::new(jammy_image); 6648 let guest = Guest::new(Box::new(jammy)); 6649 6650 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6651 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6652 6653 let api_socket = temp_api_path(&guest.tmp_dir); 6654 let mut child = GuestCommand::new(&guest) 6655 .args(["--api-socket", &api_socket]) 6656 .args(["--cpus", "boot=1"]) 6657 .args(["--memory", "size=512M,shared=on,hugepages=on"]) 6658 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6659 .args(["--serial", "tty", "--console", "off"]) 6660 .default_disks() 6661 .default_net() 6662 .capture_output() 6663 .spawn() 6664 .unwrap(); 6665 6666 let r = std::panic::catch_unwind(|| { 6667 guest.wait_vm_boot(None).unwrap(); 6668 6669 // Hotplug the SPDK-NVMe device to the VM 6670 let (cmd_success, cmd_output) = remote_command_w_output( 6671 &api_socket, 6672 "add-user-device", 6673 Some(&format!( 6674 "socket={},id=vfio_user0", 6675 spdk_nvme_dir 6676 .as_path() 6677 .join("nvme-vfio-user/cntrl") 6678 .to_str() 6679 .unwrap(), 6680 )), 6681 ); 6682 assert!(cmd_success); 6683 assert!(String::from_utf8_lossy(&cmd_output) 6684 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6685 6686 thread::sleep(std::time::Duration::new(10, 0)); 6687 6688 // Check both if /dev/nvme exists and if the block size is 128M. 6689 assert_eq!( 6690 guest 6691 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6692 .unwrap() 6693 .trim() 6694 .parse::<u32>() 6695 .unwrap_or_default(), 6696 1 6697 ); 6698 6699 // Check changes persist after reboot 6700 assert_eq!( 6701 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6702 "" 6703 ); 6704 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6705 guest 6706 .ssh_command("echo test123 | sudo tee /mnt/test") 6707 .unwrap(); 6708 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6709 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6710 6711 guest.reboot_linux(0, None); 6712 assert_eq!( 6713 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6714 "" 6715 ); 6716 assert_eq!( 6717 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6718 "test123" 6719 ); 6720 }); 6721 6722 cleanup_spdk_nvme(); 6723 6724 let _ = child.kill(); 6725 let output = child.wait_with_output().unwrap(); 6726 6727 handle_child_output(r, &output); 6728 } 6729 6730 #[test] 6731 #[cfg(target_arch = "x86_64")] 6732 fn test_vdpa_block() { 6733 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6734 assert!(exec_host_command_status("lsmod | grep vdpa_sim_blk").success()); 6735 6736 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6737 let guest = Guest::new(Box::new(focal)); 6738 let api_socket = temp_api_path(&guest.tmp_dir); 6739 6740 let kernel_path = direct_kernel_boot_path(); 6741 6742 let mut child = GuestCommand::new(&guest) 6743 .args(["--cpus", "boot=2"]) 6744 .args(["--memory", "size=512M,hugepages=on"]) 6745 .args(["--kernel", kernel_path.to_str().unwrap()]) 6746 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6747 .default_disks() 6748 .default_net() 6749 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6750 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 6751 .args(["--api-socket", &api_socket]) 6752 .capture_output() 6753 .spawn() 6754 .unwrap(); 6755 6756 let r = std::panic::catch_unwind(|| { 6757 guest.wait_vm_boot(None).unwrap(); 6758 6759 // Check both if /dev/vdc exists and if the block size is 128M. 6760 assert_eq!( 6761 guest 6762 .ssh_command("lsblk | grep vdc | grep -c 128M") 6763 .unwrap() 6764 .trim() 6765 .parse::<u32>() 6766 .unwrap_or_default(), 6767 1 6768 ); 6769 6770 // Check the content of the block device after we wrote to it. 6771 // The vpda-sim-blk should let us read what we previously wrote. 6772 guest 6773 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6774 .unwrap(); 6775 assert_eq!( 6776 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6777 "foobar" 6778 ); 6779 6780 // Hotplug an extra vDPA block device behind the vIOMMU 6781 // Add a new vDPA device to the VM 6782 let (cmd_success, cmd_output) = remote_command_w_output( 6783 &api_socket, 6784 "add-vdpa", 6785 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6786 ); 6787 assert!(cmd_success); 6788 assert!(String::from_utf8_lossy(&cmd_output) 6789 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6790 6791 thread::sleep(std::time::Duration::new(10, 0)); 6792 6793 // Check IOMMU setup 6794 assert!(guest 6795 .does_device_vendor_pair_match("0x1057", "0x1af4") 6796 .unwrap_or_default()); 6797 assert_eq!( 6798 guest 6799 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6800 .unwrap() 6801 .trim(), 6802 "0001:00:01.0" 6803 ); 6804 6805 // Check both if /dev/vdd exists and if the block size is 128M. 6806 assert_eq!( 6807 guest 6808 .ssh_command("lsblk | grep vdd | grep -c 128M") 6809 .unwrap() 6810 .trim() 6811 .parse::<u32>() 6812 .unwrap_or_default(), 6813 1 6814 ); 6815 6816 // Write some content to the block device we've just plugged. 6817 guest 6818 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6819 .unwrap(); 6820 6821 // Check we can read the content back. 6822 assert_eq!( 6823 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6824 "foobar" 6825 ); 6826 6827 // Unplug the device 6828 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6829 assert!(cmd_success); 6830 thread::sleep(std::time::Duration::new(10, 0)); 6831 6832 // Check /dev/vdd doesn't exist anymore 6833 assert_eq!( 6834 guest 6835 .ssh_command("lsblk | grep -c vdd || true") 6836 .unwrap() 6837 .trim() 6838 .parse::<u32>() 6839 .unwrap_or(1), 6840 0 6841 ); 6842 }); 6843 6844 let _ = child.kill(); 6845 let output = child.wait_with_output().unwrap(); 6846 6847 handle_child_output(r, &output); 6848 } 6849 6850 #[test] 6851 #[cfg(target_arch = "x86_64")] 6852 #[ignore = "See #5756"] 6853 fn test_vdpa_net() { 6854 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6855 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6856 return; 6857 } 6858 6859 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6860 let guest = Guest::new(Box::new(focal)); 6861 6862 let kernel_path = direct_kernel_boot_path(); 6863 6864 let mut child = GuestCommand::new(&guest) 6865 .args(["--cpus", "boot=2"]) 6866 .args(["--memory", "size=512M,hugepages=on"]) 6867 .args(["--kernel", kernel_path.to_str().unwrap()]) 6868 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6869 .default_disks() 6870 .default_net() 6871 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 6872 .capture_output() 6873 .spawn() 6874 .unwrap(); 6875 6876 let r = std::panic::catch_unwind(|| { 6877 guest.wait_vm_boot(None).unwrap(); 6878 6879 // Check we can find network interface related to vDPA device 6880 assert_eq!( 6881 guest 6882 .ssh_command("ip -o link | grep -c ens6") 6883 .unwrap() 6884 .trim() 6885 .parse::<u32>() 6886 .unwrap_or(0), 6887 1 6888 ); 6889 6890 guest 6891 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 6892 .unwrap(); 6893 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 6894 6895 // Check there is no packet yet on both TX/RX of the network interface 6896 assert_eq!( 6897 guest 6898 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 6899 .unwrap() 6900 .trim() 6901 .parse::<u32>() 6902 .unwrap_or(0), 6903 2 6904 ); 6905 6906 // Send 6 packets with ping command 6907 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 6908 6909 // Check we can find 6 packets on both TX/RX of the network interface 6910 assert_eq!( 6911 guest 6912 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 6913 .unwrap() 6914 .trim() 6915 .parse::<u32>() 6916 .unwrap_or(0), 6917 2 6918 ); 6919 6920 // No need to check for hotplug as we already tested it through 6921 // test_vdpa_block() 6922 }); 6923 6924 let _ = child.kill(); 6925 let output = child.wait_with_output().unwrap(); 6926 6927 handle_child_output(r, &output); 6928 } 6929 6930 #[test] 6931 #[cfg(target_arch = "x86_64")] 6932 fn test_tpm() { 6933 let focal = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 6934 let guest = Guest::new(Box::new(focal)); 6935 6936 let (mut swtpm_command, swtpm_socket_path) = prepare_swtpm_daemon(&guest.tmp_dir); 6937 6938 let mut guest_cmd = GuestCommand::new(&guest); 6939 guest_cmd 6940 .args(["--cpus", "boot=1"]) 6941 .args(["--memory", "size=512M"]) 6942 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6943 .args(["--tpm", &format!("socket={swtpm_socket_path}")]) 6944 .capture_output() 6945 .default_disks() 6946 .default_net(); 6947 6948 // Start swtpm daemon 6949 let mut swtpm_child = swtpm_command.spawn().unwrap(); 6950 thread::sleep(std::time::Duration::new(10, 0)); 6951 let mut child = guest_cmd.spawn().unwrap(); 6952 let r = std::panic::catch_unwind(|| { 6953 guest.wait_vm_boot(None).unwrap(); 6954 assert_eq!( 6955 guest.ssh_command("ls /dev/tpm0").unwrap().trim(), 6956 "/dev/tpm0" 6957 ); 6958 guest.ssh_command("sudo tpm2_selftest -f").unwrap(); 6959 guest 6960 .ssh_command("echo 'hello' > /tmp/checksum_test; ") 6961 .unwrap(); 6962 guest.ssh_command("cmp <(sudo tpm2_pcrevent /tmp/checksum_test | grep sha256 | awk '{print $2}') <(sha256sum /tmp/checksum_test| awk '{print $1}')").unwrap(); 6963 }); 6964 6965 let _ = swtpm_child.kill(); 6966 let _d_out = swtpm_child.wait_with_output().unwrap(); 6967 6968 let _ = child.kill(); 6969 let output = child.wait_with_output().unwrap(); 6970 6971 handle_child_output(r, &output); 6972 } 6973 6974 #[test] 6975 #[cfg(target_arch = "x86_64")] 6976 fn test_double_tty() { 6977 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6978 let guest = Guest::new(Box::new(focal)); 6979 let mut cmd = GuestCommand::new(&guest); 6980 let api_socket = temp_api_path(&guest.tmp_dir); 6981 let tty_str: &str = "console=hvc0 earlyprintk=ttyS0 "; 6982 // linux printk module enable console log. 6983 let con_dis_str: &str = "console [hvc0] enabled"; 6984 // linux printk module disable console log. 6985 let con_enb_str: &str = "bootconsole [earlyser0] disabled"; 6986 6987 let kernel_path = direct_kernel_boot_path(); 6988 6989 cmd.args(["--cpus", "boot=1"]) 6990 .args(["--memory", "size=512M"]) 6991 .args(["--kernel", kernel_path.to_str().unwrap()]) 6992 .args([ 6993 "--cmdline", 6994 DIRECT_KERNEL_BOOT_CMDLINE 6995 .replace("console=hvc0 ", tty_str) 6996 .as_str(), 6997 ]) 6998 .capture_output() 6999 .default_disks() 7000 .default_net() 7001 .args(["--serial", "tty"]) 7002 .args(["--console", "tty"]) 7003 .args(["--api-socket", &api_socket]); 7004 7005 let mut child = cmd.spawn().unwrap(); 7006 7007 let mut r = std::panic::catch_unwind(|| { 7008 guest.wait_vm_boot(None).unwrap(); 7009 }); 7010 7011 let _ = child.kill(); 7012 let output = child.wait_with_output().unwrap(); 7013 7014 if r.is_ok() { 7015 r = std::panic::catch_unwind(|| { 7016 let s = String::from_utf8_lossy(&output.stdout); 7017 assert!(s.contains(tty_str)); 7018 assert!(s.contains(con_dis_str)); 7019 assert!(s.contains(con_enb_str)); 7020 }); 7021 } 7022 7023 handle_child_output(r, &output); 7024 } 7025 7026 #[test] 7027 #[cfg(target_arch = "x86_64")] 7028 fn test_nmi() { 7029 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7030 let guest = Guest::new(Box::new(jammy)); 7031 let api_socket = temp_api_path(&guest.tmp_dir); 7032 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7033 7034 let kernel_path = direct_kernel_boot_path(); 7035 let cmd_line = format!("{} {}", DIRECT_KERNEL_BOOT_CMDLINE, "unknown_nmi_panic=1"); 7036 7037 let mut cmd = GuestCommand::new(&guest); 7038 cmd.args(["--cpus", "boot=4"]) 7039 .args(["--memory", "size=512M"]) 7040 .args(["--kernel", kernel_path.to_str().unwrap()]) 7041 .args(["--cmdline", cmd_line.as_str()]) 7042 .default_disks() 7043 .args(["--net", guest.default_net_string().as_str()]) 7044 .args(["--pvpanic"]) 7045 .args(["--api-socket", &api_socket]) 7046 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7047 .capture_output(); 7048 7049 let mut child = cmd.spawn().unwrap(); 7050 7051 let r = std::panic::catch_unwind(|| { 7052 guest.wait_vm_boot(None).unwrap(); 7053 7054 assert!(remote_command(&api_socket, "nmi", None)); 7055 7056 // Wait a while for guest 7057 thread::sleep(std::time::Duration::new(3, 0)); 7058 7059 let expected_sequential_events = [&MetaEvent { 7060 event: "panic".to_string(), 7061 device_id: None, 7062 }]; 7063 assert!(check_latest_events_exact( 7064 &expected_sequential_events, 7065 &event_path 7066 )); 7067 }); 7068 7069 let _ = child.kill(); 7070 let output = child.wait_with_output().unwrap(); 7071 7072 handle_child_output(r, &output); 7073 } 7074 } 7075 7076 mod dbus_api { 7077 use crate::*; 7078 7079 // Start cloud-hypervisor with no VM parameters, running both the HTTP 7080 // and DBus APIs. Alternate calls to the external APIs (HTTP and DBus) 7081 // to create a VM, boot it, and verify that it can be shut down and then 7082 // booted again. 7083 #[test] 7084 fn test_api_dbus_and_http_interleaved() { 7085 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7086 let guest = Guest::new(Box::new(focal)); 7087 let dbus_api = TargetApi::new_dbus_api(&guest.tmp_dir); 7088 let http_api = TargetApi::new_http_api(&guest.tmp_dir); 7089 7090 let mut child = GuestCommand::new(&guest) 7091 .args(dbus_api.guest_args()) 7092 .args(http_api.guest_args()) 7093 .capture_output() 7094 .spawn() 7095 .unwrap(); 7096 7097 thread::sleep(std::time::Duration::new(1, 0)); 7098 7099 // Verify API servers are running 7100 assert!(dbus_api.remote_command("ping", None)); 7101 assert!(http_api.remote_command("ping", None)); 7102 7103 // Create the VM first 7104 let cpu_count: u8 = 4; 7105 let request_body = guest.api_create_body( 7106 cpu_count, 7107 direct_kernel_boot_path().to_str().unwrap(), 7108 DIRECT_KERNEL_BOOT_CMDLINE, 7109 ); 7110 7111 let temp_config_path = guest.tmp_dir.as_path().join("config"); 7112 std::fs::write(&temp_config_path, request_body).unwrap(); 7113 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 7114 7115 let r = std::panic::catch_unwind(|| { 7116 // Create the VM 7117 assert!(dbus_api.remote_command("create", Some(create_config),)); 7118 7119 // Then boot it 7120 assert!(http_api.remote_command("boot", None)); 7121 guest.wait_vm_boot(None).unwrap(); 7122 7123 // Check that the VM booted as expected 7124 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7125 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7126 7127 // Sync and shutdown without powering off to prevent filesystem 7128 // corruption. 7129 guest.ssh_command("sync").unwrap(); 7130 guest.ssh_command("sudo shutdown -H now").unwrap(); 7131 7132 // Wait for the guest to be fully shutdown 7133 thread::sleep(std::time::Duration::new(20, 0)); 7134 7135 // Then shutdown the VM 7136 assert!(dbus_api.remote_command("shutdown", None)); 7137 7138 // Then boot it again 7139 assert!(http_api.remote_command("boot", None)); 7140 guest.wait_vm_boot(None).unwrap(); 7141 7142 // Check that the VM booted as expected 7143 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7144 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7145 }); 7146 7147 let _ = child.kill(); 7148 let output = child.wait_with_output().unwrap(); 7149 7150 handle_child_output(r, &output); 7151 } 7152 7153 #[test] 7154 fn test_api_dbus_create_boot() { 7155 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7156 let guest = Guest::new(Box::new(focal)); 7157 7158 _test_api_create_boot(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7159 } 7160 7161 #[test] 7162 fn test_api_dbus_shutdown() { 7163 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7164 let guest = Guest::new(Box::new(focal)); 7165 7166 _test_api_shutdown(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7167 } 7168 7169 #[test] 7170 fn test_api_dbus_delete() { 7171 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7172 let guest = Guest::new(Box::new(focal)); 7173 7174 _test_api_delete(TargetApi::new_dbus_api(&guest.tmp_dir), guest); 7175 } 7176 7177 #[test] 7178 fn test_api_dbus_pause_resume() { 7179 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7180 let guest = Guest::new(Box::new(focal)); 7181 7182 _test_api_pause_resume(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7183 } 7184 } 7185 7186 mod common_sequential { 7187 use std::fs::remove_dir_all; 7188 7189 use crate::*; 7190 7191 #[test] 7192 #[cfg(not(feature = "mshv"))] 7193 fn test_memory_mergeable_on() { 7194 test_memory_mergeable(true) 7195 } 7196 7197 fn snapshot_and_check_events(api_socket: &str, snapshot_dir: &str, event_path: &str) { 7198 // Pause the VM 7199 assert!(remote_command(api_socket, "pause", None)); 7200 let latest_events: [&MetaEvent; 2] = [ 7201 &MetaEvent { 7202 event: "pausing".to_string(), 7203 device_id: None, 7204 }, 7205 &MetaEvent { 7206 event: "paused".to_string(), 7207 device_id: None, 7208 }, 7209 ]; 7210 // See: #5938 7211 thread::sleep(std::time::Duration::new(1, 0)); 7212 assert!(check_latest_events_exact(&latest_events, event_path)); 7213 7214 // Take a snapshot from the VM 7215 assert!(remote_command( 7216 api_socket, 7217 "snapshot", 7218 Some(format!("file://{snapshot_dir}").as_str()), 7219 )); 7220 7221 // Wait to make sure the snapshot is completed 7222 thread::sleep(std::time::Duration::new(10, 0)); 7223 7224 let latest_events = [ 7225 &MetaEvent { 7226 event: "snapshotting".to_string(), 7227 device_id: None, 7228 }, 7229 &MetaEvent { 7230 event: "snapshotted".to_string(), 7231 device_id: None, 7232 }, 7233 ]; 7234 // See: #5938 7235 thread::sleep(std::time::Duration::new(1, 0)); 7236 assert!(check_latest_events_exact(&latest_events, event_path)); 7237 } 7238 7239 // One thing to note about this test. The virtio-net device is heavily used 7240 // through each ssh command. There's no need to perform a dedicated test to 7241 // verify the migration went well for virtio-net. 7242 #[test] 7243 #[cfg(not(feature = "mshv"))] 7244 fn test_snapshot_restore_hotplug_virtiomem() { 7245 _test_snapshot_restore(true); 7246 } 7247 7248 #[test] 7249 fn test_snapshot_restore_basic() { 7250 _test_snapshot_restore(false); 7251 } 7252 7253 fn _test_snapshot_restore(use_hotplug: bool) { 7254 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7255 let guest = Guest::new(Box::new(focal)); 7256 let kernel_path = direct_kernel_boot_path(); 7257 7258 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7259 7260 let net_id = "net123"; 7261 let net_params = format!( 7262 "id={},tap=,mac={},ip={},mask=255.255.255.0", 7263 net_id, guest.network.guest_mac, guest.network.host_ip 7264 ); 7265 let mut mem_params = "size=2G"; 7266 7267 if use_hotplug { 7268 mem_params = "size=2G,hotplug_method=virtio-mem,hotplug_size=32G" 7269 } 7270 7271 let cloudinit_params = format!( 7272 "path={},iommu=on", 7273 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7274 ); 7275 7276 let socket = temp_vsock_path(&guest.tmp_dir); 7277 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7278 7279 let mut child = GuestCommand::new(&guest) 7280 .args(["--api-socket", &api_socket_source]) 7281 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7282 .args(["--cpus", "boot=4"]) 7283 .args(["--memory", mem_params]) 7284 .args(["--balloon", "size=0"]) 7285 .args(["--kernel", kernel_path.to_str().unwrap()]) 7286 .args([ 7287 "--disk", 7288 format!( 7289 "path={}", 7290 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7291 ) 7292 .as_str(), 7293 cloudinit_params.as_str(), 7294 ]) 7295 .args(["--net", net_params.as_str()]) 7296 .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) 7297 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7298 .capture_output() 7299 .spawn() 7300 .unwrap(); 7301 7302 let console_text = String::from("On a branch floating down river a cricket, singing."); 7303 // Create the snapshot directory 7304 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7305 7306 let r = std::panic::catch_unwind(|| { 7307 guest.wait_vm_boot(None).unwrap(); 7308 7309 // Check the number of vCPUs 7310 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 7311 // Check the guest RAM 7312 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 7313 if use_hotplug { 7314 // Increase guest RAM with virtio-mem 7315 resize_command( 7316 &api_socket_source, 7317 None, 7318 Some(6 << 30), 7319 None, 7320 Some(&event_path), 7321 ); 7322 thread::sleep(std::time::Duration::new(5, 0)); 7323 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7324 // Use balloon to remove RAM from the VM 7325 resize_command( 7326 &api_socket_source, 7327 None, 7328 None, 7329 Some(1 << 30), 7330 Some(&event_path), 7331 ); 7332 thread::sleep(std::time::Duration::new(5, 0)); 7333 let total_memory = guest.get_total_memory().unwrap_or_default(); 7334 assert!(total_memory > 4_800_000); 7335 assert!(total_memory < 5_760_000); 7336 } 7337 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 7338 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7339 7340 // x86_64: We check that removing and adding back the virtio-net device 7341 // does not break the snapshot/restore support for virtio-pci. 7342 // This is an important thing to test as the hotplug will 7343 // trigger a PCI BAR reprogramming, which is a good way of 7344 // checking if the stored resources are correctly restored. 7345 // Unplug the virtio-net device 7346 // AArch64: Device hotplug is currently not supported, skipping here. 7347 #[cfg(target_arch = "x86_64")] 7348 { 7349 assert!(remote_command( 7350 &api_socket_source, 7351 "remove-device", 7352 Some(net_id), 7353 )); 7354 thread::sleep(std::time::Duration::new(10, 0)); 7355 let latest_events = [&MetaEvent { 7356 event: "device-removed".to_string(), 7357 device_id: Some(net_id.to_string()), 7358 }]; 7359 // See: #5938 7360 thread::sleep(std::time::Duration::new(1, 0)); 7361 assert!(check_latest_events_exact(&latest_events, &event_path)); 7362 7363 // Plug the virtio-net device again 7364 assert!(remote_command( 7365 &api_socket_source, 7366 "add-net", 7367 Some(net_params.as_str()), 7368 )); 7369 thread::sleep(std::time::Duration::new(10, 0)); 7370 } 7371 7372 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7373 }); 7374 7375 // Shutdown the source VM and check console output 7376 let _ = child.kill(); 7377 let output = child.wait_with_output().unwrap(); 7378 handle_child_output(r, &output); 7379 7380 let r = std::panic::catch_unwind(|| { 7381 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7382 }); 7383 7384 handle_child_output(r, &output); 7385 7386 // Remove the vsock socket file. 7387 Command::new("rm") 7388 .arg("-f") 7389 .arg(socket.as_str()) 7390 .output() 7391 .unwrap(); 7392 7393 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7394 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7395 7396 // Restore the VM from the snapshot 7397 let mut child = GuestCommand::new(&guest) 7398 .args(["--api-socket", &api_socket_restored]) 7399 .args([ 7400 "--event-monitor", 7401 format!("path={event_path_restored}").as_str(), 7402 ]) 7403 .args([ 7404 "--restore", 7405 format!("source_url=file://{snapshot_dir}").as_str(), 7406 ]) 7407 .capture_output() 7408 .spawn() 7409 .unwrap(); 7410 7411 // Wait for the VM to be restored 7412 thread::sleep(std::time::Duration::new(20, 0)); 7413 let expected_events = [ 7414 &MetaEvent { 7415 event: "starting".to_string(), 7416 device_id: None, 7417 }, 7418 &MetaEvent { 7419 event: "activated".to_string(), 7420 device_id: Some("__console".to_string()), 7421 }, 7422 &MetaEvent { 7423 event: "activated".to_string(), 7424 device_id: Some("__rng".to_string()), 7425 }, 7426 &MetaEvent { 7427 event: "restoring".to_string(), 7428 device_id: None, 7429 }, 7430 ]; 7431 assert!(check_sequential_events( 7432 &expected_events, 7433 &event_path_restored 7434 )); 7435 let latest_events = [&MetaEvent { 7436 event: "restored".to_string(), 7437 device_id: None, 7438 }]; 7439 assert!(check_latest_events_exact( 7440 &latest_events, 7441 &event_path_restored 7442 )); 7443 7444 // Remove the snapshot dir 7445 let _ = remove_dir_all(snapshot_dir.as_str()); 7446 7447 let r = std::panic::catch_unwind(|| { 7448 // Resume the VM 7449 assert!(remote_command(&api_socket_restored, "resume", None)); 7450 // There is no way that we can ensure the 'write()' to the 7451 // event file is completed when the 'resume' request is 7452 // returned successfully, because the 'write()' was done 7453 // asynchronously from a different thread of Cloud 7454 // Hypervisor (e.g. the event-monitor thread). 7455 thread::sleep(std::time::Duration::new(1, 0)); 7456 let latest_events = [ 7457 &MetaEvent { 7458 event: "resuming".to_string(), 7459 device_id: None, 7460 }, 7461 &MetaEvent { 7462 event: "resumed".to_string(), 7463 device_id: None, 7464 }, 7465 ]; 7466 assert!(check_latest_events_exact( 7467 &latest_events, 7468 &event_path_restored 7469 )); 7470 7471 // Perform same checks to validate VM has been properly restored 7472 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 7473 let total_memory = guest.get_total_memory().unwrap_or_default(); 7474 if !use_hotplug { 7475 assert!(total_memory > 1_920_000); 7476 } else { 7477 assert!(total_memory > 4_800_000); 7478 assert!(total_memory < 5_760_000); 7479 // Deflate balloon to restore entire RAM to the VM 7480 resize_command(&api_socket_restored, None, None, Some(0), None); 7481 thread::sleep(std::time::Duration::new(5, 0)); 7482 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7483 // Decrease guest RAM with virtio-mem 7484 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 7485 thread::sleep(std::time::Duration::new(5, 0)); 7486 let total_memory = guest.get_total_memory().unwrap_or_default(); 7487 assert!(total_memory > 4_800_000); 7488 assert!(total_memory < 5_760_000); 7489 } 7490 7491 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7492 }); 7493 // Shutdown the target VM and check console output 7494 let _ = child.kill(); 7495 let output = child.wait_with_output().unwrap(); 7496 handle_child_output(r, &output); 7497 7498 let r = std::panic::catch_unwind(|| { 7499 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7500 }); 7501 7502 handle_child_output(r, &output); 7503 } 7504 7505 #[test] 7506 fn test_snapshot_restore_with_fd() { 7507 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7508 let guest = Guest::new(Box::new(focal)); 7509 let kernel_path = direct_kernel_boot_path(); 7510 7511 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7512 7513 let net_id = "net123"; 7514 let num_queue_pairs: usize = 2; 7515 // use a name that does not conflict with tap dev created from other tests 7516 let tap_name = "chtap999"; 7517 use std::str::FromStr; 7518 let taps = net_util::open_tap( 7519 Some(tap_name), 7520 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 7521 None, 7522 &mut None, 7523 None, 7524 num_queue_pairs, 7525 Some(libc::O_RDWR | libc::O_NONBLOCK), 7526 ) 7527 .unwrap(); 7528 let net_params = format!( 7529 "id={},fd=[{},{}],mac={},ip={},mask=255.255.255.0,num_queues={}", 7530 net_id, 7531 taps[0].as_raw_fd(), 7532 taps[1].as_raw_fd(), 7533 guest.network.guest_mac, 7534 guest.network.host_ip, 7535 num_queue_pairs * 2 7536 ); 7537 7538 let cloudinit_params = format!( 7539 "path={},iommu=on", 7540 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7541 ); 7542 7543 let n_cpu = 2; 7544 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7545 7546 let mut child = GuestCommand::new(&guest) 7547 .args(["--api-socket", &api_socket_source]) 7548 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7549 .args(["--cpus", format!("boot={}", n_cpu).as_str()]) 7550 .args(["--memory", "size=1G"]) 7551 .args(["--kernel", kernel_path.to_str().unwrap()]) 7552 .args([ 7553 "--disk", 7554 format!( 7555 "path={}", 7556 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7557 ) 7558 .as_str(), 7559 cloudinit_params.as_str(), 7560 ]) 7561 .args(["--net", net_params.as_str()]) 7562 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7563 .capture_output() 7564 .spawn() 7565 .unwrap(); 7566 7567 let console_text = String::from("On a branch floating down river a cricket, singing."); 7568 // Create the snapshot directory 7569 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7570 7571 let r = std::panic::catch_unwind(|| { 7572 guest.wait_vm_boot(None).unwrap(); 7573 7574 // close the fds after VM boots, as CH duplicates them before using 7575 for tap in taps.iter() { 7576 unsafe { libc::close(tap.as_raw_fd()) }; 7577 } 7578 7579 // Check the number of vCPUs 7580 assert_eq!(guest.get_cpu_count().unwrap_or_default(), n_cpu); 7581 // Check the guest RAM 7582 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 7583 7584 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 7585 guest.check_devices_common(None, Some(&console_text), None); 7586 7587 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7588 }); 7589 7590 // Shutdown the source VM and check console output 7591 let _ = child.kill(); 7592 let output = child.wait_with_output().unwrap(); 7593 handle_child_output(r, &output); 7594 7595 let r = std::panic::catch_unwind(|| { 7596 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7597 }); 7598 7599 handle_child_output(r, &output); 7600 7601 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7602 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7603 7604 // Restore the VM from the snapshot 7605 let mut child = GuestCommand::new(&guest) 7606 .args(["--api-socket", &api_socket_restored]) 7607 .args([ 7608 "--event-monitor", 7609 format!("path={event_path_restored}").as_str(), 7610 ]) 7611 .capture_output() 7612 .spawn() 7613 .unwrap(); 7614 thread::sleep(std::time::Duration::new(2, 0)); 7615 7616 let taps = net_util::open_tap( 7617 Some(tap_name), 7618 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 7619 None, 7620 &mut None, 7621 None, 7622 num_queue_pairs, 7623 Some(libc::O_RDWR | libc::O_NONBLOCK), 7624 ) 7625 .unwrap(); 7626 let restore_params = format!( 7627 "source_url=file://{},net_fds=[{}@[{},{}]]", 7628 snapshot_dir, 7629 net_id, 7630 taps[0].as_raw_fd(), 7631 taps[1].as_raw_fd() 7632 ); 7633 assert!(remote_command( 7634 &api_socket_restored, 7635 "restore", 7636 Some(restore_params.as_str()) 7637 )); 7638 7639 // Wait for the VM to be restored 7640 thread::sleep(std::time::Duration::new(20, 0)); 7641 7642 // close the fds as CH duplicates them before using 7643 for tap in taps.iter() { 7644 unsafe { libc::close(tap.as_raw_fd()) }; 7645 } 7646 7647 let expected_events = [ 7648 &MetaEvent { 7649 event: "starting".to_string(), 7650 device_id: None, 7651 }, 7652 &MetaEvent { 7653 event: "activated".to_string(), 7654 device_id: Some("__console".to_string()), 7655 }, 7656 &MetaEvent { 7657 event: "activated".to_string(), 7658 device_id: Some("__rng".to_string()), 7659 }, 7660 &MetaEvent { 7661 event: "restoring".to_string(), 7662 device_id: None, 7663 }, 7664 ]; 7665 assert!(check_sequential_events( 7666 &expected_events, 7667 &event_path_restored 7668 )); 7669 let latest_events = [&MetaEvent { 7670 event: "restored".to_string(), 7671 device_id: None, 7672 }]; 7673 assert!(check_latest_events_exact( 7674 &latest_events, 7675 &event_path_restored 7676 )); 7677 7678 // Remove the snapshot dir 7679 let _ = remove_dir_all(snapshot_dir.as_str()); 7680 7681 let r = std::panic::catch_unwind(|| { 7682 // Resume the VM 7683 assert!(remote_command(&api_socket_restored, "resume", None)); 7684 // There is no way that we can ensure the 'write()' to the 7685 // event file is completed when the 'resume' request is 7686 // returned successfully, because the 'write()' was done 7687 // asynchronously from a different thread of Cloud 7688 // Hypervisor (e.g. the event-monitor thread). 7689 thread::sleep(std::time::Duration::new(1, 0)); 7690 let latest_events = [ 7691 &MetaEvent { 7692 event: "resuming".to_string(), 7693 device_id: None, 7694 }, 7695 &MetaEvent { 7696 event: "resumed".to_string(), 7697 device_id: None, 7698 }, 7699 ]; 7700 assert!(check_latest_events_exact( 7701 &latest_events, 7702 &event_path_restored 7703 )); 7704 7705 // Perform same checks to validate VM has been properly restored 7706 assert_eq!(guest.get_cpu_count().unwrap_or_default(), n_cpu); 7707 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 7708 7709 guest.check_devices_common(None, Some(&console_text), None); 7710 }); 7711 // Shutdown the target VM and check console output 7712 let _ = child.kill(); 7713 let output = child.wait_with_output().unwrap(); 7714 handle_child_output(r, &output); 7715 7716 let r = std::panic::catch_unwind(|| { 7717 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7718 }); 7719 7720 handle_child_output(r, &output); 7721 } 7722 } 7723 7724 mod windows { 7725 use crate::*; 7726 use once_cell::sync::Lazy; 7727 7728 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 7729 7730 struct WindowsGuest { 7731 guest: Guest, 7732 auth: PasswordAuth, 7733 } 7734 7735 trait FsType { 7736 const FS_FAT: u8; 7737 const FS_NTFS: u8; 7738 } 7739 impl FsType for WindowsGuest { 7740 const FS_FAT: u8 = 0; 7741 const FS_NTFS: u8 = 1; 7742 } 7743 7744 impl WindowsGuest { 7745 fn new() -> Self { 7746 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 7747 let guest = Guest::new(Box::new(disk)); 7748 let auth = PasswordAuth { 7749 username: String::from("administrator"), 7750 password: String::from("Admin123"), 7751 }; 7752 7753 WindowsGuest { guest, auth } 7754 } 7755 7756 fn guest(&self) -> &Guest { 7757 &self.guest 7758 } 7759 7760 fn ssh_cmd(&self, cmd: &str) -> String { 7761 ssh_command_ip_with_auth( 7762 cmd, 7763 &self.auth, 7764 &self.guest.network.guest_ip, 7765 DEFAULT_SSH_RETRIES, 7766 DEFAULT_SSH_TIMEOUT, 7767 ) 7768 .unwrap() 7769 } 7770 7771 fn cpu_count(&self) -> u8 { 7772 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 7773 .trim() 7774 .parse::<u8>() 7775 .unwrap_or(0) 7776 } 7777 7778 fn ram_size(&self) -> usize { 7779 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 7780 .trim() 7781 .parse::<usize>() 7782 .unwrap_or(0) 7783 } 7784 7785 fn netdev_count(&self) -> u8 { 7786 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7787 .trim() 7788 .parse::<u8>() 7789 .unwrap_or(0) 7790 } 7791 7792 fn disk_count(&self) -> u8 { 7793 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7794 .trim() 7795 .parse::<u8>() 7796 .unwrap_or(0) 7797 } 7798 7799 fn reboot(&self) { 7800 let _ = self.ssh_cmd("shutdown /r /t 0"); 7801 } 7802 7803 fn shutdown(&self) { 7804 let _ = self.ssh_cmd("shutdown /s /t 0"); 7805 } 7806 7807 fn run_dnsmasq(&self) -> std::process::Child { 7808 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 7809 let dhcp_host = format!( 7810 "--dhcp-host={},{}", 7811 self.guest.network.guest_mac, self.guest.network.guest_ip 7812 ); 7813 let dhcp_range = format!( 7814 "--dhcp-range=eth,{},{}", 7815 self.guest.network.guest_ip, self.guest.network.guest_ip 7816 ); 7817 7818 Command::new("dnsmasq") 7819 .arg("--no-daemon") 7820 .arg("--log-queries") 7821 .arg(listen_address.as_str()) 7822 .arg("--except-interface=lo") 7823 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 7824 .arg("--conf-file=/dev/null") 7825 .arg(dhcp_host.as_str()) 7826 .arg(dhcp_range.as_str()) 7827 .spawn() 7828 .unwrap() 7829 } 7830 7831 // TODO Cleanup image file explicitly after test, if there's some space issues. 7832 fn disk_new(&self, fs: u8, sz: usize) -> String { 7833 let mut guard = NEXT_DISK_ID.lock().unwrap(); 7834 let id = *guard; 7835 *guard = id + 1; 7836 7837 let img = PathBuf::from(format!("/tmp/test-hotplug-{id}.raw")); 7838 let _ = fs::remove_file(&img); 7839 7840 // Create an image file 7841 let out = Command::new("qemu-img") 7842 .args([ 7843 "create", 7844 "-f", 7845 "raw", 7846 img.to_str().unwrap(), 7847 format!("{sz}m").as_str(), 7848 ]) 7849 .output() 7850 .expect("qemu-img command failed") 7851 .stdout; 7852 println!("{out:?}"); 7853 7854 // Associate image to a loop device 7855 let out = Command::new("losetup") 7856 .args(["--show", "-f", img.to_str().unwrap()]) 7857 .output() 7858 .expect("failed to create loop device") 7859 .stdout; 7860 let _tmp = String::from_utf8_lossy(&out); 7861 let loop_dev = _tmp.trim(); 7862 println!("{out:?}"); 7863 7864 // Create a partition table 7865 // echo 'type=7' | sudo sfdisk "${LOOP}" 7866 let mut child = Command::new("sfdisk") 7867 .args([loop_dev]) 7868 .stdin(Stdio::piped()) 7869 .spawn() 7870 .unwrap(); 7871 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 7872 stdin 7873 .write_all("type=7".as_bytes()) 7874 .expect("failed to write stdin"); 7875 let out = child.wait_with_output().expect("sfdisk failed").stdout; 7876 println!("{out:?}"); 7877 7878 // Disengage the loop device 7879 let out = Command::new("losetup") 7880 .args(["-d", loop_dev]) 7881 .output() 7882 .expect("loop device not found") 7883 .stdout; 7884 println!("{out:?}"); 7885 7886 // Re-associate loop device pointing to the partition only 7887 let out = Command::new("losetup") 7888 .args([ 7889 "--show", 7890 "--offset", 7891 (512 * 2048).to_string().as_str(), 7892 "-f", 7893 img.to_str().unwrap(), 7894 ]) 7895 .output() 7896 .expect("failed to create loop device") 7897 .stdout; 7898 let _tmp = String::from_utf8_lossy(&out); 7899 let loop_dev = _tmp.trim(); 7900 println!("{out:?}"); 7901 7902 // Create filesystem. 7903 let fs_cmd = match fs { 7904 WindowsGuest::FS_FAT => "mkfs.msdos", 7905 WindowsGuest::FS_NTFS => "mkfs.ntfs", 7906 _ => panic!("Unknown filesystem type '{fs}'"), 7907 }; 7908 let out = Command::new(fs_cmd) 7909 .args([&loop_dev]) 7910 .output() 7911 .unwrap_or_else(|_| panic!("{fs_cmd} failed")) 7912 .stdout; 7913 println!("{out:?}"); 7914 7915 // Disengage the loop device 7916 let out = Command::new("losetup") 7917 .args(["-d", loop_dev]) 7918 .output() 7919 .unwrap_or_else(|_| panic!("loop device '{loop_dev}' not found")) 7920 .stdout; 7921 println!("{out:?}"); 7922 7923 img.to_str().unwrap().to_string() 7924 } 7925 7926 fn disks_set_rw(&self) { 7927 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 7928 } 7929 7930 fn disks_online(&self) { 7931 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 7932 } 7933 7934 fn disk_file_put(&self, fname: &str, data: &str) { 7935 let _ = self.ssh_cmd(&format!( 7936 "powershell -Command \"'{data}' | Set-Content -Path {fname}\"" 7937 )); 7938 } 7939 7940 fn disk_file_read(&self, fname: &str) -> String { 7941 self.ssh_cmd(&format!( 7942 "powershell -Command \"Get-Content -Path {fname}\"" 7943 )) 7944 } 7945 7946 fn wait_for_boot(&self) -> bool { 7947 let cmd = "dir /b c:\\ | find \"Windows\""; 7948 let tmo_max = 180; 7949 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 7950 // interval must be small. 7951 let tmo_int = 2; 7952 let out = ssh_command_ip_with_auth( 7953 cmd, 7954 &self.auth, 7955 &self.guest.network.guest_ip, 7956 { 7957 let mut ret = 1; 7958 let mut tmo_acc = 0; 7959 loop { 7960 tmo_acc += tmo_int * ret; 7961 if tmo_acc >= tmo_max { 7962 break; 7963 } 7964 ret += 1; 7965 } 7966 ret 7967 }, 7968 tmo_int, 7969 ) 7970 .unwrap(); 7971 7972 if "Windows" == out.trim() { 7973 return true; 7974 } 7975 7976 false 7977 } 7978 } 7979 7980 fn vcpu_threads_count(pid: u32) -> u8 { 7981 // ps -T -p 12345 | grep vcpu | wc -l 7982 let out = Command::new("ps") 7983 .args(["-T", "-p", format!("{pid}").as_str()]) 7984 .output() 7985 .expect("ps command failed") 7986 .stdout; 7987 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 7988 } 7989 7990 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 7991 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 7992 let out = Command::new("ps") 7993 .args(["-T", "-p", format!("{pid}").as_str()]) 7994 .output() 7995 .expect("ps command failed") 7996 .stdout; 7997 let mut n = 0; 7998 String::from_utf8_lossy(&out) 7999 .split_whitespace() 8000 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 8001 n 8002 } 8003 8004 fn disk_ctrl_threads_count(pid: u32) -> u8 { 8005 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 8006 let out = Command::new("ps") 8007 .args(["-T", "-p", format!("{pid}").as_str()]) 8008 .output() 8009 .expect("ps command failed") 8010 .stdout; 8011 let mut n = 0; 8012 String::from_utf8_lossy(&out) 8013 .split_whitespace() 8014 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 8015 n 8016 } 8017 8018 #[test] 8019 fn test_windows_guest() { 8020 let windows_guest = WindowsGuest::new(); 8021 8022 let mut child = GuestCommand::new(windows_guest.guest()) 8023 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8024 .args(["--memory", "size=4G"]) 8025 .args(["--kernel", edk2_path().to_str().unwrap()]) 8026 .args(["--serial", "tty"]) 8027 .args(["--console", "off"]) 8028 .default_disks() 8029 .default_net() 8030 .capture_output() 8031 .spawn() 8032 .unwrap(); 8033 8034 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8035 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8036 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8037 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8038 8039 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8040 8041 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8042 8043 let r = std::panic::catch_unwind(|| { 8044 // Wait to make sure Windows boots up 8045 assert!(windows_guest.wait_for_boot()); 8046 8047 windows_guest.shutdown(); 8048 }); 8049 8050 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8051 let _ = child.kill(); 8052 let output = child.wait_with_output().unwrap(); 8053 8054 let _ = child_dnsmasq.kill(); 8055 let _ = child_dnsmasq.wait(); 8056 8057 handle_child_output(r, &output); 8058 } 8059 8060 #[test] 8061 fn test_windows_guest_multiple_queues() { 8062 let windows_guest = WindowsGuest::new(); 8063 8064 let mut ovmf_path = dirs::home_dir().unwrap(); 8065 ovmf_path.push("workloads"); 8066 ovmf_path.push(OVMF_NAME); 8067 8068 let mut child = GuestCommand::new(windows_guest.guest()) 8069 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 8070 .args(["--memory", "size=4G"]) 8071 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8072 .args(["--serial", "tty"]) 8073 .args(["--console", "off"]) 8074 .args([ 8075 "--disk", 8076 format!( 8077 "path={},num_queues=4", 8078 windows_guest 8079 .guest() 8080 .disk_config 8081 .disk(DiskType::OperatingSystem) 8082 .unwrap() 8083 ) 8084 .as_str(), 8085 ]) 8086 .args([ 8087 "--net", 8088 format!( 8089 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 8090 windows_guest.guest().network.guest_mac, 8091 windows_guest.guest().network.host_ip 8092 ) 8093 .as_str(), 8094 ]) 8095 .capture_output() 8096 .spawn() 8097 .unwrap(); 8098 8099 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8100 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8101 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8102 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8103 8104 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8105 8106 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8107 8108 let r = std::panic::catch_unwind(|| { 8109 // Wait to make sure Windows boots up 8110 assert!(windows_guest.wait_for_boot()); 8111 8112 windows_guest.shutdown(); 8113 }); 8114 8115 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8116 let _ = child.kill(); 8117 let output = child.wait_with_output().unwrap(); 8118 8119 let _ = child_dnsmasq.kill(); 8120 let _ = child_dnsmasq.wait(); 8121 8122 handle_child_output(r, &output); 8123 } 8124 8125 #[test] 8126 #[cfg(not(feature = "mshv"))] 8127 #[ignore = "See #4327"] 8128 fn test_windows_guest_snapshot_restore() { 8129 let windows_guest = WindowsGuest::new(); 8130 8131 let mut ovmf_path = dirs::home_dir().unwrap(); 8132 ovmf_path.push("workloads"); 8133 ovmf_path.push(OVMF_NAME); 8134 8135 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8136 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 8137 8138 let mut child = GuestCommand::new(windows_guest.guest()) 8139 .args(["--api-socket", &api_socket_source]) 8140 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8141 .args(["--memory", "size=4G"]) 8142 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8143 .args(["--serial", "tty"]) 8144 .args(["--console", "off"]) 8145 .default_disks() 8146 .default_net() 8147 .capture_output() 8148 .spawn() 8149 .unwrap(); 8150 8151 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8152 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8153 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8154 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8155 8156 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8157 8158 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8159 8160 // Wait to make sure Windows boots up 8161 assert!(windows_guest.wait_for_boot()); 8162 8163 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 8164 8165 // Pause the VM 8166 assert!(remote_command(&api_socket_source, "pause", None)); 8167 8168 // Take a snapshot from the VM 8169 assert!(remote_command( 8170 &api_socket_source, 8171 "snapshot", 8172 Some(format!("file://{snapshot_dir}").as_str()), 8173 )); 8174 8175 // Wait to make sure the snapshot is completed 8176 thread::sleep(std::time::Duration::new(30, 0)); 8177 8178 let _ = child.kill(); 8179 child.wait().unwrap(); 8180 8181 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 8182 8183 // Restore the VM from the snapshot 8184 let mut child = GuestCommand::new(windows_guest.guest()) 8185 .args(["--api-socket", &api_socket_restored]) 8186 .args([ 8187 "--restore", 8188 format!("source_url=file://{snapshot_dir}").as_str(), 8189 ]) 8190 .capture_output() 8191 .spawn() 8192 .unwrap(); 8193 8194 // Wait for the VM to be restored 8195 thread::sleep(std::time::Duration::new(20, 0)); 8196 8197 let r = std::panic::catch_unwind(|| { 8198 // Resume the VM 8199 assert!(remote_command(&api_socket_restored, "resume", None)); 8200 8201 windows_guest.shutdown(); 8202 }); 8203 8204 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8205 let _ = child.kill(); 8206 let output = child.wait_with_output().unwrap(); 8207 8208 let _ = child_dnsmasq.kill(); 8209 let _ = child_dnsmasq.wait(); 8210 8211 handle_child_output(r, &output); 8212 } 8213 8214 #[test] 8215 #[cfg(not(feature = "mshv"))] 8216 #[cfg(not(target_arch = "aarch64"))] 8217 fn test_windows_guest_cpu_hotplug() { 8218 let windows_guest = WindowsGuest::new(); 8219 8220 let mut ovmf_path = dirs::home_dir().unwrap(); 8221 ovmf_path.push("workloads"); 8222 ovmf_path.push(OVMF_NAME); 8223 8224 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8225 let api_socket = temp_api_path(&tmp_dir); 8226 8227 let mut child = GuestCommand::new(windows_guest.guest()) 8228 .args(["--api-socket", &api_socket]) 8229 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 8230 .args(["--memory", "size=4G"]) 8231 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8232 .args(["--serial", "tty"]) 8233 .args(["--console", "off"]) 8234 .default_disks() 8235 .default_net() 8236 .capture_output() 8237 .spawn() 8238 .unwrap(); 8239 8240 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8241 8242 let r = std::panic::catch_unwind(|| { 8243 // Wait to make sure Windows boots up 8244 assert!(windows_guest.wait_for_boot()); 8245 8246 let vcpu_num = 2; 8247 // Check the initial number of CPUs the guest sees 8248 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8249 // Check the initial number of vcpu threads in the CH process 8250 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8251 8252 let vcpu_num = 6; 8253 // Hotplug some CPUs 8254 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8255 // Wait to make sure CPUs are added 8256 thread::sleep(std::time::Duration::new(10, 0)); 8257 // Check the guest sees the correct number 8258 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8259 // Check the CH process has the correct number of vcpu threads 8260 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8261 8262 let vcpu_num = 4; 8263 // Remove some CPUs. Note that Windows doesn't support hot-remove. 8264 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8265 // Wait to make sure CPUs are removed 8266 thread::sleep(std::time::Duration::new(10, 0)); 8267 // Reboot to let Windows catch up 8268 windows_guest.reboot(); 8269 // Wait to make sure Windows completely rebooted 8270 thread::sleep(std::time::Duration::new(60, 0)); 8271 // Check the guest sees the correct number 8272 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8273 // Check the CH process has the correct number of vcpu threads 8274 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8275 8276 windows_guest.shutdown(); 8277 }); 8278 8279 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8280 let _ = child.kill(); 8281 let output = child.wait_with_output().unwrap(); 8282 8283 let _ = child_dnsmasq.kill(); 8284 let _ = child_dnsmasq.wait(); 8285 8286 handle_child_output(r, &output); 8287 } 8288 8289 #[test] 8290 #[cfg(not(feature = "mshv"))] 8291 #[cfg(not(target_arch = "aarch64"))] 8292 fn test_windows_guest_ram_hotplug() { 8293 let windows_guest = WindowsGuest::new(); 8294 8295 let mut ovmf_path = dirs::home_dir().unwrap(); 8296 ovmf_path.push("workloads"); 8297 ovmf_path.push(OVMF_NAME); 8298 8299 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8300 let api_socket = temp_api_path(&tmp_dir); 8301 8302 let mut child = GuestCommand::new(windows_guest.guest()) 8303 .args(["--api-socket", &api_socket]) 8304 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8305 .args(["--memory", "size=2G,hotplug_size=5G"]) 8306 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8307 .args(["--serial", "tty"]) 8308 .args(["--console", "off"]) 8309 .default_disks() 8310 .default_net() 8311 .capture_output() 8312 .spawn() 8313 .unwrap(); 8314 8315 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8316 8317 let r = std::panic::catch_unwind(|| { 8318 // Wait to make sure Windows boots up 8319 assert!(windows_guest.wait_for_boot()); 8320 8321 let ram_size = 2 * 1024 * 1024 * 1024; 8322 // Check the initial number of RAM the guest sees 8323 let current_ram_size = windows_guest.ram_size(); 8324 // This size seems to be reserved by the system and thus the 8325 // reported amount differs by this constant value. 8326 let reserved_ram_size = ram_size - current_ram_size; 8327 // Verify that there's not more than 4mb constant diff wasted 8328 // by the reserved ram. 8329 assert!(reserved_ram_size < 4 * 1024 * 1024); 8330 8331 let ram_size = 4 * 1024 * 1024 * 1024; 8332 // Hotplug some RAM 8333 resize_command(&api_socket, None, Some(ram_size), None, None); 8334 // Wait to make sure RAM has been added 8335 thread::sleep(std::time::Duration::new(10, 0)); 8336 // Check the guest sees the correct number 8337 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8338 8339 let ram_size = 3 * 1024 * 1024 * 1024; 8340 // Unplug some RAM. Note that hot-remove most likely won't work. 8341 resize_command(&api_socket, None, Some(ram_size), None, None); 8342 // Wait to make sure RAM has been added 8343 thread::sleep(std::time::Duration::new(10, 0)); 8344 // Reboot to let Windows catch up 8345 windows_guest.reboot(); 8346 // Wait to make sure guest completely rebooted 8347 thread::sleep(std::time::Duration::new(60, 0)); 8348 // Check the guest sees the correct number 8349 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8350 8351 windows_guest.shutdown(); 8352 }); 8353 8354 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8355 let _ = child.kill(); 8356 let output = child.wait_with_output().unwrap(); 8357 8358 let _ = child_dnsmasq.kill(); 8359 let _ = child_dnsmasq.wait(); 8360 8361 handle_child_output(r, &output); 8362 } 8363 8364 #[test] 8365 #[cfg(not(feature = "mshv"))] 8366 fn test_windows_guest_netdev_hotplug() { 8367 let windows_guest = WindowsGuest::new(); 8368 8369 let mut ovmf_path = dirs::home_dir().unwrap(); 8370 ovmf_path.push("workloads"); 8371 ovmf_path.push(OVMF_NAME); 8372 8373 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8374 let api_socket = temp_api_path(&tmp_dir); 8375 8376 let mut child = GuestCommand::new(windows_guest.guest()) 8377 .args(["--api-socket", &api_socket]) 8378 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8379 .args(["--memory", "size=4G"]) 8380 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8381 .args(["--serial", "tty"]) 8382 .args(["--console", "off"]) 8383 .default_disks() 8384 .default_net() 8385 .capture_output() 8386 .spawn() 8387 .unwrap(); 8388 8389 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8390 8391 let r = std::panic::catch_unwind(|| { 8392 // Wait to make sure Windows boots up 8393 assert!(windows_guest.wait_for_boot()); 8394 8395 // Initially present network device 8396 let netdev_num = 1; 8397 assert_eq!(windows_guest.netdev_count(), netdev_num); 8398 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8399 8400 // Hotplug network device 8401 let (cmd_success, cmd_output) = remote_command_w_output( 8402 &api_socket, 8403 "add-net", 8404 Some(windows_guest.guest().default_net_string().as_str()), 8405 ); 8406 assert!(cmd_success); 8407 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 8408 thread::sleep(std::time::Duration::new(5, 0)); 8409 // Verify the device is on the system 8410 let netdev_num = 2; 8411 assert_eq!(windows_guest.netdev_count(), netdev_num); 8412 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8413 8414 // Remove network device 8415 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 8416 assert!(cmd_success); 8417 thread::sleep(std::time::Duration::new(5, 0)); 8418 // Verify the device has been removed 8419 let netdev_num = 1; 8420 assert_eq!(windows_guest.netdev_count(), netdev_num); 8421 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8422 8423 windows_guest.shutdown(); 8424 }); 8425 8426 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8427 let _ = child.kill(); 8428 let output = child.wait_with_output().unwrap(); 8429 8430 let _ = child_dnsmasq.kill(); 8431 let _ = child_dnsmasq.wait(); 8432 8433 handle_child_output(r, &output); 8434 } 8435 8436 #[test] 8437 #[ignore = "See #6037"] 8438 #[cfg(not(feature = "mshv"))] 8439 #[cfg(not(target_arch = "aarch64"))] 8440 fn test_windows_guest_disk_hotplug() { 8441 let windows_guest = WindowsGuest::new(); 8442 8443 let mut ovmf_path = dirs::home_dir().unwrap(); 8444 ovmf_path.push("workloads"); 8445 ovmf_path.push(OVMF_NAME); 8446 8447 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8448 let api_socket = temp_api_path(&tmp_dir); 8449 8450 let mut child = GuestCommand::new(windows_guest.guest()) 8451 .args(["--api-socket", &api_socket]) 8452 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8453 .args(["--memory", "size=4G"]) 8454 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8455 .args(["--serial", "tty"]) 8456 .args(["--console", "off"]) 8457 .default_disks() 8458 .default_net() 8459 .capture_output() 8460 .spawn() 8461 .unwrap(); 8462 8463 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8464 8465 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 8466 8467 let r = std::panic::catch_unwind(|| { 8468 // Wait to make sure Windows boots up 8469 assert!(windows_guest.wait_for_boot()); 8470 8471 // Initially present disk device 8472 let disk_num = 1; 8473 assert_eq!(windows_guest.disk_count(), disk_num); 8474 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8475 8476 // Hotplug disk device 8477 let (cmd_success, cmd_output) = remote_command_w_output( 8478 &api_socket, 8479 "add-disk", 8480 Some(format!("path={disk},readonly=off").as_str()), 8481 ); 8482 assert!(cmd_success); 8483 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 8484 thread::sleep(std::time::Duration::new(5, 0)); 8485 // Online disk device 8486 windows_guest.disks_set_rw(); 8487 windows_guest.disks_online(); 8488 // Verify the device is on the system 8489 let disk_num = 2; 8490 assert_eq!(windows_guest.disk_count(), disk_num); 8491 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8492 8493 let data = "hello"; 8494 let fname = "d:\\world"; 8495 windows_guest.disk_file_put(fname, data); 8496 8497 // Unmount disk device 8498 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 8499 assert!(cmd_success); 8500 thread::sleep(std::time::Duration::new(5, 0)); 8501 // Verify the device has been removed 8502 let disk_num = 1; 8503 assert_eq!(windows_guest.disk_count(), disk_num); 8504 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8505 8506 // Remount and check the file exists with the expected contents 8507 let (cmd_success, _cmd_output) = remote_command_w_output( 8508 &api_socket, 8509 "add-disk", 8510 Some(format!("path={disk},readonly=off").as_str()), 8511 ); 8512 assert!(cmd_success); 8513 thread::sleep(std::time::Duration::new(5, 0)); 8514 let out = windows_guest.disk_file_read(fname); 8515 assert_eq!(data, out.trim()); 8516 8517 // Intentionally no unmount, it'll happen at shutdown. 8518 8519 windows_guest.shutdown(); 8520 }); 8521 8522 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8523 let _ = child.kill(); 8524 let output = child.wait_with_output().unwrap(); 8525 8526 let _ = child_dnsmasq.kill(); 8527 let _ = child_dnsmasq.wait(); 8528 8529 handle_child_output(r, &output); 8530 } 8531 8532 #[test] 8533 #[ignore = "See #6037"] 8534 #[cfg(not(feature = "mshv"))] 8535 #[cfg(not(target_arch = "aarch64"))] 8536 fn test_windows_guest_disk_hotplug_multi() { 8537 let windows_guest = WindowsGuest::new(); 8538 8539 let mut ovmf_path = dirs::home_dir().unwrap(); 8540 ovmf_path.push("workloads"); 8541 ovmf_path.push(OVMF_NAME); 8542 8543 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8544 let api_socket = temp_api_path(&tmp_dir); 8545 8546 let mut child = GuestCommand::new(windows_guest.guest()) 8547 .args(["--api-socket", &api_socket]) 8548 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8549 .args(["--memory", "size=2G"]) 8550 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8551 .args(["--serial", "tty"]) 8552 .args(["--console", "off"]) 8553 .default_disks() 8554 .default_net() 8555 .capture_output() 8556 .spawn() 8557 .unwrap(); 8558 8559 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8560 8561 // Predefined data to used at various test stages 8562 let disk_test_data: [[String; 4]; 2] = [ 8563 [ 8564 "_disk2".to_string(), 8565 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 8566 "d:\\world".to_string(), 8567 "hello".to_string(), 8568 ], 8569 [ 8570 "_disk3".to_string(), 8571 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 8572 "e:\\hello".to_string(), 8573 "world".to_string(), 8574 ], 8575 ]; 8576 8577 let r = std::panic::catch_unwind(|| { 8578 // Wait to make sure Windows boots up 8579 assert!(windows_guest.wait_for_boot()); 8580 8581 // Initially present disk device 8582 let disk_num = 1; 8583 assert_eq!(windows_guest.disk_count(), disk_num); 8584 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8585 8586 for it in &disk_test_data { 8587 let disk_id = it[0].as_str(); 8588 let disk = it[1].as_str(); 8589 // Hotplug disk device 8590 let (cmd_success, cmd_output) = remote_command_w_output( 8591 &api_socket, 8592 "add-disk", 8593 Some(format!("path={disk},readonly=off").as_str()), 8594 ); 8595 assert!(cmd_success); 8596 assert!(String::from_utf8_lossy(&cmd_output) 8597 .contains(format!("\"id\":\"{disk_id}\"").as_str())); 8598 thread::sleep(std::time::Duration::new(5, 0)); 8599 // Online disk devices 8600 windows_guest.disks_set_rw(); 8601 windows_guest.disks_online(); 8602 } 8603 // Verify the devices are on the system 8604 let disk_num = (disk_test_data.len() + 1) as u8; 8605 assert_eq!(windows_guest.disk_count(), disk_num); 8606 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8607 8608 // Put test data 8609 for it in &disk_test_data { 8610 let fname = it[2].as_str(); 8611 let data = it[3].as_str(); 8612 windows_guest.disk_file_put(fname, data); 8613 } 8614 8615 // Unmount disk devices 8616 for it in &disk_test_data { 8617 let disk_id = it[0].as_str(); 8618 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 8619 assert!(cmd_success); 8620 thread::sleep(std::time::Duration::new(5, 0)); 8621 } 8622 8623 // Verify the devices have been removed 8624 let disk_num = 1; 8625 assert_eq!(windows_guest.disk_count(), disk_num); 8626 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8627 8628 // Remount 8629 for it in &disk_test_data { 8630 let disk = it[1].as_str(); 8631 let (cmd_success, _cmd_output) = remote_command_w_output( 8632 &api_socket, 8633 "add-disk", 8634 Some(format!("path={disk},readonly=off").as_str()), 8635 ); 8636 assert!(cmd_success); 8637 thread::sleep(std::time::Duration::new(5, 0)); 8638 } 8639 8640 // Check the files exists with the expected contents 8641 for it in &disk_test_data { 8642 let fname = it[2].as_str(); 8643 let data = it[3].as_str(); 8644 let out = windows_guest.disk_file_read(fname); 8645 assert_eq!(data, out.trim()); 8646 } 8647 8648 // Intentionally no unmount, it'll happen at shutdown. 8649 8650 windows_guest.shutdown(); 8651 }); 8652 8653 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8654 let _ = child.kill(); 8655 let output = child.wait_with_output().unwrap(); 8656 8657 let _ = child_dnsmasq.kill(); 8658 let _ = child_dnsmasq.wait(); 8659 8660 handle_child_output(r, &output); 8661 } 8662 8663 #[test] 8664 #[cfg(not(feature = "mshv"))] 8665 #[cfg(not(target_arch = "aarch64"))] 8666 fn test_windows_guest_netdev_multi() { 8667 let windows_guest = WindowsGuest::new(); 8668 8669 let mut ovmf_path = dirs::home_dir().unwrap(); 8670 ovmf_path.push("workloads"); 8671 ovmf_path.push(OVMF_NAME); 8672 8673 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8674 let api_socket = temp_api_path(&tmp_dir); 8675 8676 let mut child = GuestCommand::new(windows_guest.guest()) 8677 .args(["--api-socket", &api_socket]) 8678 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8679 .args(["--memory", "size=4G"]) 8680 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8681 .args(["--serial", "tty"]) 8682 .args(["--console", "off"]) 8683 .default_disks() 8684 // The multi net dev config is borrowed from test_multiple_network_interfaces 8685 .args([ 8686 "--net", 8687 windows_guest.guest().default_net_string().as_str(), 8688 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 8689 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 8690 ]) 8691 .capture_output() 8692 .spawn() 8693 .unwrap(); 8694 8695 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8696 8697 let r = std::panic::catch_unwind(|| { 8698 // Wait to make sure Windows boots up 8699 assert!(windows_guest.wait_for_boot()); 8700 8701 let netdev_num = 3; 8702 assert_eq!(windows_guest.netdev_count(), netdev_num); 8703 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8704 8705 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 8706 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 8707 8708 windows_guest.shutdown(); 8709 }); 8710 8711 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8712 let _ = child.kill(); 8713 let output = child.wait_with_output().unwrap(); 8714 8715 let _ = child_dnsmasq.kill(); 8716 let _ = child_dnsmasq.wait(); 8717 8718 handle_child_output(r, &output); 8719 } 8720 } 8721 8722 #[cfg(target_arch = "x86_64")] 8723 mod sgx { 8724 use crate::*; 8725 8726 #[test] 8727 fn test_sgx() { 8728 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 8729 let jammy = UbuntuDiskConfig::new(jammy_image); 8730 let guest = Guest::new(Box::new(jammy)); 8731 8732 let mut child = GuestCommand::new(&guest) 8733 .args(["--cpus", "boot=1"]) 8734 .args(["--memory", "size=512M"]) 8735 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8736 .default_disks() 8737 .default_net() 8738 .args(["--sgx-epc", "id=epc0,size=64M"]) 8739 .capture_output() 8740 .spawn() 8741 .unwrap(); 8742 8743 let r = std::panic::catch_unwind(|| { 8744 guest.wait_vm_boot(None).unwrap(); 8745 8746 // Check if SGX is correctly detected in the guest. 8747 guest.check_sgx_support().unwrap(); 8748 8749 // Validate the SGX EPC section is 64MiB. 8750 assert_eq!( 8751 guest 8752 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 8753 .unwrap() 8754 .trim(), 8755 "0x0000000004000000" 8756 ); 8757 }); 8758 8759 let _ = child.kill(); 8760 let output = child.wait_with_output().unwrap(); 8761 8762 handle_child_output(r, &output); 8763 } 8764 } 8765 8766 #[cfg(target_arch = "x86_64")] 8767 mod vfio { 8768 use crate::*; 8769 8770 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 8771 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8772 let guest = Guest::new(Box::new(jammy)); 8773 let api_socket = temp_api_path(&guest.tmp_dir); 8774 8775 let mut child = GuestCommand::new(&guest) 8776 .args(["--cpus", "boot=4"]) 8777 .args([ 8778 "--memory", 8779 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(), 8780 ]) 8781 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8782 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8783 .args(["--api-socket", &api_socket]) 8784 .default_disks() 8785 .default_net() 8786 .capture_output() 8787 .spawn() 8788 .unwrap(); 8789 8790 let r = std::panic::catch_unwind(|| { 8791 guest.wait_vm_boot(None).unwrap(); 8792 8793 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8794 8795 guest.enable_memory_hotplug(); 8796 8797 // Add RAM to the VM 8798 let desired_ram = 6 << 30; 8799 resize_command(&api_socket, None, Some(desired_ram), None, None); 8800 thread::sleep(std::time::Duration::new(30, 0)); 8801 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8802 8803 // Check the VFIO device works when RAM is increased to 6GiB 8804 guest.check_nvidia_gpu(); 8805 }); 8806 8807 let _ = child.kill(); 8808 let output = child.wait_with_output().unwrap(); 8809 8810 handle_child_output(r, &output); 8811 } 8812 8813 #[test] 8814 fn test_nvidia_card_memory_hotplug_acpi() { 8815 test_nvidia_card_memory_hotplug("acpi") 8816 } 8817 8818 #[test] 8819 fn test_nvidia_card_memory_hotplug_virtio_mem() { 8820 test_nvidia_card_memory_hotplug("virtio-mem") 8821 } 8822 8823 #[test] 8824 fn test_nvidia_card_pci_hotplug() { 8825 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8826 let guest = Guest::new(Box::new(jammy)); 8827 let api_socket = temp_api_path(&guest.tmp_dir); 8828 8829 let mut child = GuestCommand::new(&guest) 8830 .args(["--cpus", "boot=4"]) 8831 .args(["--memory", "size=4G"]) 8832 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8833 .args(["--api-socket", &api_socket]) 8834 .default_disks() 8835 .default_net() 8836 .capture_output() 8837 .spawn() 8838 .unwrap(); 8839 8840 let r = std::panic::catch_unwind(|| { 8841 guest.wait_vm_boot(None).unwrap(); 8842 8843 // Hotplug the card to the VM 8844 let (cmd_success, cmd_output) = remote_command_w_output( 8845 &api_socket, 8846 "add-device", 8847 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 8848 ); 8849 assert!(cmd_success); 8850 assert!(String::from_utf8_lossy(&cmd_output) 8851 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 8852 8853 thread::sleep(std::time::Duration::new(10, 0)); 8854 8855 // Check the VFIO device works after hotplug 8856 guest.check_nvidia_gpu(); 8857 }); 8858 8859 let _ = child.kill(); 8860 let output = child.wait_with_output().unwrap(); 8861 8862 handle_child_output(r, &output); 8863 } 8864 8865 #[test] 8866 fn test_nvidia_card_reboot() { 8867 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8868 let guest = Guest::new(Box::new(jammy)); 8869 let api_socket = temp_api_path(&guest.tmp_dir); 8870 8871 let mut child = GuestCommand::new(&guest) 8872 .args(["--cpus", "boot=4"]) 8873 .args(["--memory", "size=4G"]) 8874 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8875 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8876 .args(["--api-socket", &api_socket]) 8877 .default_disks() 8878 .default_net() 8879 .capture_output() 8880 .spawn() 8881 .unwrap(); 8882 8883 let r = std::panic::catch_unwind(|| { 8884 guest.wait_vm_boot(None).unwrap(); 8885 8886 // Check the VFIO device works after boot 8887 guest.check_nvidia_gpu(); 8888 8889 guest.reboot_linux(0, None); 8890 8891 // Check the VFIO device works after reboot 8892 guest.check_nvidia_gpu(); 8893 }); 8894 8895 let _ = child.kill(); 8896 let output = child.wait_with_output().unwrap(); 8897 8898 handle_child_output(r, &output); 8899 } 8900 } 8901 8902 mod live_migration { 8903 use crate::*; 8904 8905 fn start_live_migration( 8906 migration_socket: &str, 8907 src_api_socket: &str, 8908 dest_api_socket: &str, 8909 local: bool, 8910 ) -> bool { 8911 // Start to receive migration from the destintion VM 8912 let mut receive_migration = Command::new(clh_command("ch-remote")) 8913 .args([ 8914 &format!("--api-socket={dest_api_socket}"), 8915 "receive-migration", 8916 &format! {"unix:{migration_socket}"}, 8917 ]) 8918 .stderr(Stdio::piped()) 8919 .stdout(Stdio::piped()) 8920 .spawn() 8921 .unwrap(); 8922 // Give it '1s' to make sure the 'migration_socket' file is properly created 8923 thread::sleep(std::time::Duration::new(1, 0)); 8924 // Start to send migration from the source VM 8925 8926 let mut args = [ 8927 format!("--api-socket={}", &src_api_socket), 8928 "send-migration".to_string(), 8929 format! {"unix:{migration_socket}"}, 8930 ] 8931 .to_vec(); 8932 8933 if local { 8934 args.insert(2, "--local".to_string()); 8935 } 8936 8937 let mut send_migration = Command::new(clh_command("ch-remote")) 8938 .args(&args) 8939 .stderr(Stdio::piped()) 8940 .stdout(Stdio::piped()) 8941 .spawn() 8942 .unwrap(); 8943 8944 // The 'send-migration' command should be executed successfully within the given timeout 8945 let send_success = if let Some(status) = send_migration 8946 .wait_timeout(std::time::Duration::from_secs(30)) 8947 .unwrap() 8948 { 8949 status.success() 8950 } else { 8951 false 8952 }; 8953 8954 if !send_success { 8955 let _ = send_migration.kill(); 8956 let output = send_migration.wait_with_output().unwrap(); 8957 eprintln!( 8958 "\n\n==== Start 'send_migration' output ==== \ 8959 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8960 \n\n==== End 'send_migration' output ====\n\n", 8961 String::from_utf8_lossy(&output.stdout), 8962 String::from_utf8_lossy(&output.stderr) 8963 ); 8964 } 8965 8966 // The 'receive-migration' command should be executed successfully within the given timeout 8967 let receive_success = if let Some(status) = receive_migration 8968 .wait_timeout(std::time::Duration::from_secs(30)) 8969 .unwrap() 8970 { 8971 status.success() 8972 } else { 8973 false 8974 }; 8975 8976 if !receive_success { 8977 let _ = receive_migration.kill(); 8978 let output = receive_migration.wait_with_output().unwrap(); 8979 eprintln!( 8980 "\n\n==== Start 'receive_migration' output ==== \ 8981 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8982 \n\n==== End 'receive_migration' output ====\n\n", 8983 String::from_utf8_lossy(&output.stdout), 8984 String::from_utf8_lossy(&output.stderr) 8985 ); 8986 } 8987 8988 send_success && receive_success 8989 } 8990 8991 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 8992 let mut src_vm = src_vm; 8993 let mut dest_vm = dest_vm; 8994 8995 let _ = src_vm.kill(); 8996 let src_output = src_vm.wait_with_output().unwrap(); 8997 eprintln!( 8998 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8999 String::from_utf8_lossy(&src_output.stdout) 9000 ); 9001 eprintln!( 9002 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 9003 String::from_utf8_lossy(&src_output.stderr) 9004 ); 9005 let _ = dest_vm.kill(); 9006 let dest_output = dest_vm.wait_with_output().unwrap(); 9007 eprintln!( 9008 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 9009 String::from_utf8_lossy(&dest_output.stdout) 9010 ); 9011 eprintln!( 9012 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 9013 String::from_utf8_lossy(&dest_output.stderr) 9014 ); 9015 9016 if let Some(ovs_vm) = ovs_vm { 9017 let mut ovs_vm = ovs_vm; 9018 let _ = ovs_vm.kill(); 9019 let ovs_output = ovs_vm.wait_with_output().unwrap(); 9020 eprintln!( 9021 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 9022 String::from_utf8_lossy(&ovs_output.stdout) 9023 ); 9024 eprintln!( 9025 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 9026 String::from_utf8_lossy(&ovs_output.stderr) 9027 ); 9028 9029 cleanup_ovs_dpdk(); 9030 } 9031 9032 panic!("Test failed: {message}") 9033 } 9034 9035 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 9036 // same host. It ensures the following behaviors: 9037 // 1. The source VM is up and functional (including various virtio-devices are working properly); 9038 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 9039 // 3. The source VM terminated gracefully after live migration; 9040 // 4. The destination VM is functional (including various virtio-devices are working properly) after 9041 // live migration; 9042 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 9043 fn _test_live_migration(upgrade_test: bool, local: bool) { 9044 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9045 let guest = Guest::new(Box::new(focal)); 9046 let kernel_path = direct_kernel_boot_path(); 9047 let console_text = String::from("On a branch floating down river a cricket, singing."); 9048 let net_id = "net123"; 9049 let net_params = format!( 9050 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9051 net_id, guest.network.guest_mac, guest.network.host_ip 9052 ); 9053 9054 let memory_param: &[&str] = if local { 9055 &["--memory", "size=4G,shared=on"] 9056 } else { 9057 &["--memory", "size=4G"] 9058 }; 9059 9060 let boot_vcpus = 2; 9061 let max_vcpus = 4; 9062 9063 let pmem_temp_file = TempFile::new().unwrap(); 9064 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9065 std::process::Command::new("mkfs.ext4") 9066 .arg(pmem_temp_file.as_path()) 9067 .output() 9068 .expect("Expect creating disk image to succeed"); 9069 let pmem_path = String::from("/dev/pmem0"); 9070 9071 // Start the source VM 9072 let src_vm_path = if !upgrade_test { 9073 clh_command("cloud-hypervisor") 9074 } else { 9075 cloud_hypervisor_release_path() 9076 }; 9077 let src_api_socket = temp_api_path(&guest.tmp_dir); 9078 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9079 src_vm_cmd 9080 .args([ 9081 "--cpus", 9082 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9083 ]) 9084 .args(memory_param) 9085 .args(["--kernel", kernel_path.to_str().unwrap()]) 9086 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9087 .default_disks() 9088 .args(["--net", net_params.as_str()]) 9089 .args(["--api-socket", &src_api_socket]) 9090 .args([ 9091 "--pmem", 9092 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9093 ]); 9094 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9095 9096 // Start the destination VM 9097 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9098 dest_api_socket.push_str(".dest"); 9099 let mut dest_child = GuestCommand::new(&guest) 9100 .args(["--api-socket", &dest_api_socket]) 9101 .capture_output() 9102 .spawn() 9103 .unwrap(); 9104 9105 let r = std::panic::catch_unwind(|| { 9106 guest.wait_vm_boot(None).unwrap(); 9107 9108 // Make sure the source VM is functaionl 9109 // Check the number of vCPUs 9110 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9111 9112 // Check the guest RAM 9113 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9114 9115 // Check the guest virtio-devices, e.g. block, rng, console, and net 9116 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9117 9118 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9119 // to make sure that removing and adding back the virtio-net device does 9120 // not break the live-migration support for virtio-pci. 9121 #[cfg(target_arch = "x86_64")] 9122 { 9123 assert!(remote_command( 9124 &src_api_socket, 9125 "remove-device", 9126 Some(net_id), 9127 )); 9128 thread::sleep(std::time::Duration::new(10, 0)); 9129 9130 // Plug the virtio-net device again 9131 assert!(remote_command( 9132 &src_api_socket, 9133 "add-net", 9134 Some(net_params.as_str()), 9135 )); 9136 thread::sleep(std::time::Duration::new(10, 0)); 9137 } 9138 9139 // Start the live-migration 9140 let migration_socket = String::from( 9141 guest 9142 .tmp_dir 9143 .as_path() 9144 .join("live-migration.sock") 9145 .to_str() 9146 .unwrap(), 9147 ); 9148 9149 assert!( 9150 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9151 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9152 ); 9153 }); 9154 9155 // Check and report any errors occurred during the live-migration 9156 if r.is_err() { 9157 print_and_panic( 9158 src_child, 9159 dest_child, 9160 None, 9161 "Error occurred during live-migration", 9162 ); 9163 } 9164 9165 // Check the source vm has been terminated successful (give it '3s' to settle) 9166 thread::sleep(std::time::Duration::new(3, 0)); 9167 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9168 print_and_panic( 9169 src_child, 9170 dest_child, 9171 None, 9172 "source VM was not terminated successfully.", 9173 ); 9174 }; 9175 9176 // Post live-migration check to make sure the destination VM is funcational 9177 let r = std::panic::catch_unwind(|| { 9178 // Perform same checks to validate VM has been properly migrated 9179 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9180 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9181 9182 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9183 }); 9184 9185 // Clean-up the destination VM and make sure it terminated correctly 9186 let _ = dest_child.kill(); 9187 let dest_output = dest_child.wait_with_output().unwrap(); 9188 handle_child_output(r, &dest_output); 9189 9190 // Check the destination VM has the expected 'concole_text' from its output 9191 let r = std::panic::catch_unwind(|| { 9192 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9193 }); 9194 handle_child_output(r, &dest_output); 9195 } 9196 9197 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 9198 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9199 let guest = Guest::new(Box::new(focal)); 9200 let kernel_path = direct_kernel_boot_path(); 9201 let console_text = String::from("On a branch floating down river a cricket, singing."); 9202 let net_id = "net123"; 9203 let net_params = format!( 9204 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9205 net_id, guest.network.guest_mac, guest.network.host_ip 9206 ); 9207 9208 let memory_param: &[&str] = if local { 9209 &[ 9210 "--memory", 9211 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 9212 "--balloon", 9213 "size=0", 9214 ] 9215 } else { 9216 &[ 9217 "--memory", 9218 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 9219 "--balloon", 9220 "size=0", 9221 ] 9222 }; 9223 9224 let boot_vcpus = 2; 9225 let max_vcpus = 4; 9226 9227 let pmem_temp_file = TempFile::new().unwrap(); 9228 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9229 std::process::Command::new("mkfs.ext4") 9230 .arg(pmem_temp_file.as_path()) 9231 .output() 9232 .expect("Expect creating disk image to succeed"); 9233 let pmem_path = String::from("/dev/pmem0"); 9234 9235 // Start the source VM 9236 let src_vm_path = if !upgrade_test { 9237 clh_command("cloud-hypervisor") 9238 } else { 9239 cloud_hypervisor_release_path() 9240 }; 9241 let src_api_socket = temp_api_path(&guest.tmp_dir); 9242 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9243 src_vm_cmd 9244 .args([ 9245 "--cpus", 9246 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9247 ]) 9248 .args(memory_param) 9249 .args(["--kernel", kernel_path.to_str().unwrap()]) 9250 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9251 .default_disks() 9252 .args(["--net", net_params.as_str()]) 9253 .args(["--api-socket", &src_api_socket]) 9254 .args([ 9255 "--pmem", 9256 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9257 ]); 9258 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9259 9260 // Start the destination VM 9261 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9262 dest_api_socket.push_str(".dest"); 9263 let mut dest_child = GuestCommand::new(&guest) 9264 .args(["--api-socket", &dest_api_socket]) 9265 .capture_output() 9266 .spawn() 9267 .unwrap(); 9268 9269 let r = std::panic::catch_unwind(|| { 9270 guest.wait_vm_boot(None).unwrap(); 9271 9272 // Make sure the source VM is functaionl 9273 // Check the number of vCPUs 9274 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9275 9276 // Check the guest RAM 9277 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9278 // Increase the guest RAM 9279 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 9280 thread::sleep(std::time::Duration::new(5, 0)); 9281 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9282 // Use balloon to remove RAM from the VM 9283 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 9284 thread::sleep(std::time::Duration::new(5, 0)); 9285 let total_memory = guest.get_total_memory().unwrap_or_default(); 9286 assert!(total_memory > 4_800_000); 9287 assert!(total_memory < 5_760_000); 9288 9289 // Check the guest virtio-devices, e.g. block, rng, console, and net 9290 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9291 9292 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9293 // to make sure that removing and adding back the virtio-net device does 9294 // not break the live-migration support for virtio-pci. 9295 #[cfg(target_arch = "x86_64")] 9296 { 9297 assert!(remote_command( 9298 &src_api_socket, 9299 "remove-device", 9300 Some(net_id), 9301 )); 9302 thread::sleep(std::time::Duration::new(10, 0)); 9303 9304 // Plug the virtio-net device again 9305 assert!(remote_command( 9306 &src_api_socket, 9307 "add-net", 9308 Some(net_params.as_str()), 9309 )); 9310 thread::sleep(std::time::Duration::new(10, 0)); 9311 } 9312 9313 // Start the live-migration 9314 let migration_socket = String::from( 9315 guest 9316 .tmp_dir 9317 .as_path() 9318 .join("live-migration.sock") 9319 .to_str() 9320 .unwrap(), 9321 ); 9322 9323 assert!( 9324 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9325 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9326 ); 9327 }); 9328 9329 // Check and report any errors occurred during the live-migration 9330 if r.is_err() { 9331 print_and_panic( 9332 src_child, 9333 dest_child, 9334 None, 9335 "Error occurred during live-migration", 9336 ); 9337 } 9338 9339 // Check the source vm has been terminated successful (give it '3s' to settle) 9340 thread::sleep(std::time::Duration::new(3, 0)); 9341 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9342 print_and_panic( 9343 src_child, 9344 dest_child, 9345 None, 9346 "source VM was not terminated successfully.", 9347 ); 9348 }; 9349 9350 // Post live-migration check to make sure the destination VM is funcational 9351 let r = std::panic::catch_unwind(|| { 9352 // Perform same checks to validate VM has been properly migrated 9353 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9354 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9355 9356 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9357 9358 // Perform checks on guest RAM using balloon 9359 let total_memory = guest.get_total_memory().unwrap_or_default(); 9360 assert!(total_memory > 4_800_000); 9361 assert!(total_memory < 5_760_000); 9362 // Deflate balloon to restore entire RAM to the VM 9363 resize_command(&dest_api_socket, None, None, Some(0), None); 9364 thread::sleep(std::time::Duration::new(5, 0)); 9365 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9366 // Decrease guest RAM with virtio-mem 9367 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 9368 thread::sleep(std::time::Duration::new(5, 0)); 9369 let total_memory = guest.get_total_memory().unwrap_or_default(); 9370 assert!(total_memory > 4_800_000); 9371 assert!(total_memory < 5_760_000); 9372 }); 9373 9374 // Clean-up the destination VM and make sure it terminated correctly 9375 let _ = dest_child.kill(); 9376 let dest_output = dest_child.wait_with_output().unwrap(); 9377 handle_child_output(r, &dest_output); 9378 9379 // Check the destination VM has the expected 'concole_text' from its output 9380 let r = std::panic::catch_unwind(|| { 9381 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9382 }); 9383 handle_child_output(r, &dest_output); 9384 } 9385 9386 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 9387 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9388 let guest = Guest::new(Box::new(focal)); 9389 let kernel_path = direct_kernel_boot_path(); 9390 let console_text = String::from("On a branch floating down river a cricket, singing."); 9391 let net_id = "net123"; 9392 let net_params = format!( 9393 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9394 net_id, guest.network.guest_mac, guest.network.host_ip 9395 ); 9396 9397 let memory_param: &[&str] = if local { 9398 &[ 9399 "--memory", 9400 "size=0,hotplug_method=virtio-mem,shared=on", 9401 "--memory-zone", 9402 "id=mem0,size=1G,hotplug_size=4G,shared=on", 9403 "id=mem1,size=1G,hotplug_size=4G,shared=on", 9404 "id=mem2,size=2G,hotplug_size=4G,shared=on", 9405 "--numa", 9406 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9407 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9408 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9409 ] 9410 } else { 9411 &[ 9412 "--memory", 9413 "size=0,hotplug_method=virtio-mem", 9414 "--memory-zone", 9415 "id=mem0,size=1G,hotplug_size=4G", 9416 "id=mem1,size=1G,hotplug_size=4G", 9417 "id=mem2,size=2G,hotplug_size=4G", 9418 "--numa", 9419 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9420 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9421 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9422 ] 9423 }; 9424 9425 let boot_vcpus = 6; 9426 let max_vcpus = 12; 9427 9428 let pmem_temp_file = TempFile::new().unwrap(); 9429 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9430 std::process::Command::new("mkfs.ext4") 9431 .arg(pmem_temp_file.as_path()) 9432 .output() 9433 .expect("Expect creating disk image to succeed"); 9434 let pmem_path = String::from("/dev/pmem0"); 9435 9436 // Start the source VM 9437 let src_vm_path = if !upgrade_test { 9438 clh_command("cloud-hypervisor") 9439 } else { 9440 cloud_hypervisor_release_path() 9441 }; 9442 let src_api_socket = temp_api_path(&guest.tmp_dir); 9443 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9444 src_vm_cmd 9445 .args([ 9446 "--cpus", 9447 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9448 ]) 9449 .args(memory_param) 9450 .args(["--kernel", kernel_path.to_str().unwrap()]) 9451 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9452 .default_disks() 9453 .args(["--net", net_params.as_str()]) 9454 .args(["--api-socket", &src_api_socket]) 9455 .args([ 9456 "--pmem", 9457 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9458 ]); 9459 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9460 9461 // Start the destination VM 9462 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9463 dest_api_socket.push_str(".dest"); 9464 let mut dest_child = GuestCommand::new(&guest) 9465 .args(["--api-socket", &dest_api_socket]) 9466 .capture_output() 9467 .spawn() 9468 .unwrap(); 9469 9470 let r = std::panic::catch_unwind(|| { 9471 guest.wait_vm_boot(None).unwrap(); 9472 9473 // Make sure the source VM is functaionl 9474 // Check the number of vCPUs 9475 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9476 9477 // Check the guest RAM 9478 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 9479 9480 // Check the guest virtio-devices, e.g. block, rng, console, and net 9481 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9482 9483 // Check the NUMA parameters are applied correctly and resize 9484 // each zone to test the case where we migrate a VM with the 9485 // virtio-mem regions being used. 9486 { 9487 guest.check_numa_common( 9488 Some(&[960_000, 960_000, 1_920_000]), 9489 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9490 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9491 ); 9492 9493 // AArch64 currently does not support hotplug, and therefore we only 9494 // test hotplug-related function on x86_64 here. 9495 #[cfg(target_arch = "x86_64")] 9496 { 9497 guest.enable_memory_hotplug(); 9498 9499 // Resize every memory zone and check each associated NUMA node 9500 // has been assigned the right amount of memory. 9501 resize_zone_command(&src_api_socket, "mem0", "2G"); 9502 resize_zone_command(&src_api_socket, "mem1", "2G"); 9503 resize_zone_command(&src_api_socket, "mem2", "3G"); 9504 thread::sleep(std::time::Duration::new(5, 0)); 9505 9506 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 9507 } 9508 } 9509 9510 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9511 // to make sure that removing and adding back the virtio-net device does 9512 // not break the live-migration support for virtio-pci. 9513 #[cfg(target_arch = "x86_64")] 9514 { 9515 assert!(remote_command( 9516 &src_api_socket, 9517 "remove-device", 9518 Some(net_id), 9519 )); 9520 thread::sleep(std::time::Duration::new(10, 0)); 9521 9522 // Plug the virtio-net device again 9523 assert!(remote_command( 9524 &src_api_socket, 9525 "add-net", 9526 Some(net_params.as_str()), 9527 )); 9528 thread::sleep(std::time::Duration::new(10, 0)); 9529 } 9530 9531 // Start the live-migration 9532 let migration_socket = String::from( 9533 guest 9534 .tmp_dir 9535 .as_path() 9536 .join("live-migration.sock") 9537 .to_str() 9538 .unwrap(), 9539 ); 9540 9541 assert!( 9542 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9543 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9544 ); 9545 }); 9546 9547 // Check and report any errors occurred during the live-migration 9548 if r.is_err() { 9549 print_and_panic( 9550 src_child, 9551 dest_child, 9552 None, 9553 "Error occurred during live-migration", 9554 ); 9555 } 9556 9557 // Check the source vm has been terminated successful (give it '3s' to settle) 9558 thread::sleep(std::time::Duration::new(3, 0)); 9559 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9560 print_and_panic( 9561 src_child, 9562 dest_child, 9563 None, 9564 "source VM was not terminated successfully.", 9565 ); 9566 }; 9567 9568 // Post live-migration check to make sure the destination VM is funcational 9569 let r = std::panic::catch_unwind(|| { 9570 // Perform same checks to validate VM has been properly migrated 9571 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9572 #[cfg(target_arch = "x86_64")] 9573 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 9574 #[cfg(target_arch = "aarch64")] 9575 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9576 9577 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9578 9579 // Perform NUMA related checks 9580 { 9581 #[cfg(target_arch = "aarch64")] 9582 { 9583 guest.check_numa_common( 9584 Some(&[960_000, 960_000, 1_920_000]), 9585 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9586 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9587 ); 9588 } 9589 9590 // AArch64 currently does not support hotplug, and therefore we only 9591 // test hotplug-related function on x86_64 here. 9592 #[cfg(target_arch = "x86_64")] 9593 { 9594 guest.check_numa_common( 9595 Some(&[1_920_000, 1_920_000, 2_880_000]), 9596 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9597 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9598 ); 9599 9600 guest.enable_memory_hotplug(); 9601 9602 // Resize every memory zone and check each associated NUMA node 9603 // has been assigned the right amount of memory. 9604 resize_zone_command(&dest_api_socket, "mem0", "4G"); 9605 resize_zone_command(&dest_api_socket, "mem1", "4G"); 9606 resize_zone_command(&dest_api_socket, "mem2", "4G"); 9607 // Resize to the maximum amount of CPUs and check each NUMA 9608 // node has been assigned the right CPUs set. 9609 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 9610 thread::sleep(std::time::Duration::new(5, 0)); 9611 9612 guest.check_numa_common( 9613 Some(&[3_840_000, 3_840_000, 3_840_000]), 9614 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 9615 None, 9616 ); 9617 } 9618 } 9619 }); 9620 9621 // Clean-up the destination VM and make sure it terminated correctly 9622 let _ = dest_child.kill(); 9623 let dest_output = dest_child.wait_with_output().unwrap(); 9624 handle_child_output(r, &dest_output); 9625 9626 // Check the destination VM has the expected 'concole_text' from its output 9627 let r = std::panic::catch_unwind(|| { 9628 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9629 }); 9630 handle_child_output(r, &dest_output); 9631 } 9632 9633 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 9634 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9635 let guest = Guest::new(Box::new(focal)); 9636 let kernel_path = direct_kernel_boot_path(); 9637 let console_text = String::from("On a branch floating down river a cricket, singing."); 9638 let net_id = "net123"; 9639 let net_params = format!( 9640 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9641 net_id, guest.network.guest_mac, guest.network.host_ip 9642 ); 9643 9644 let memory_param: &[&str] = if local { 9645 &["--memory", "size=4G,shared=on"] 9646 } else { 9647 &["--memory", "size=4G"] 9648 }; 9649 9650 let boot_vcpus = 2; 9651 let max_vcpus = 4; 9652 9653 let pmem_temp_file = TempFile::new().unwrap(); 9654 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9655 std::process::Command::new("mkfs.ext4") 9656 .arg(pmem_temp_file.as_path()) 9657 .output() 9658 .expect("Expect creating disk image to succeed"); 9659 let pmem_path = String::from("/dev/pmem0"); 9660 9661 // Start the source VM 9662 let src_vm_path = if !upgrade_test { 9663 clh_command("cloud-hypervisor") 9664 } else { 9665 cloud_hypervisor_release_path() 9666 }; 9667 let src_api_socket = temp_api_path(&guest.tmp_dir); 9668 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9669 src_vm_cmd 9670 .args([ 9671 "--cpus", 9672 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9673 ]) 9674 .args(memory_param) 9675 .args(["--kernel", kernel_path.to_str().unwrap()]) 9676 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9677 .default_disks() 9678 .args(["--net", net_params.as_str()]) 9679 .args(["--api-socket", &src_api_socket]) 9680 .args([ 9681 "--pmem", 9682 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9683 ]) 9684 .args(["--watchdog"]); 9685 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9686 9687 // Start the destination VM 9688 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9689 dest_api_socket.push_str(".dest"); 9690 let mut dest_child = GuestCommand::new(&guest) 9691 .args(["--api-socket", &dest_api_socket]) 9692 .capture_output() 9693 .spawn() 9694 .unwrap(); 9695 9696 let r = std::panic::catch_unwind(|| { 9697 guest.wait_vm_boot(None).unwrap(); 9698 9699 // Make sure the source VM is functaionl 9700 // Check the number of vCPUs 9701 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9702 // Check the guest RAM 9703 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9704 // Check the guest virtio-devices, e.g. block, rng, console, and net 9705 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9706 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9707 // to make sure that removing and adding back the virtio-net device does 9708 // not break the live-migration support for virtio-pci. 9709 #[cfg(target_arch = "x86_64")] 9710 { 9711 assert!(remote_command( 9712 &src_api_socket, 9713 "remove-device", 9714 Some(net_id), 9715 )); 9716 thread::sleep(std::time::Duration::new(10, 0)); 9717 9718 // Plug the virtio-net device again 9719 assert!(remote_command( 9720 &src_api_socket, 9721 "add-net", 9722 Some(net_params.as_str()), 9723 )); 9724 thread::sleep(std::time::Duration::new(10, 0)); 9725 } 9726 9727 // Enable watchdog and ensure its functional 9728 let expected_reboot_count = 1; 9729 // Enable the watchdog with a 15s timeout 9730 enable_guest_watchdog(&guest, 15); 9731 9732 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9733 assert_eq!( 9734 guest 9735 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 9736 .unwrap() 9737 .trim() 9738 .parse::<u32>() 9739 .unwrap_or_default(), 9740 1 9741 ); 9742 // Allow some normal time to elapse to check we don't get spurious reboots 9743 thread::sleep(std::time::Duration::new(40, 0)); 9744 // Check no reboot 9745 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9746 9747 // Start the live-migration 9748 let migration_socket = String::from( 9749 guest 9750 .tmp_dir 9751 .as_path() 9752 .join("live-migration.sock") 9753 .to_str() 9754 .unwrap(), 9755 ); 9756 9757 assert!( 9758 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9759 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9760 ); 9761 }); 9762 9763 // Check and report any errors occurred during the live-migration 9764 if r.is_err() { 9765 print_and_panic( 9766 src_child, 9767 dest_child, 9768 None, 9769 "Error occurred during live-migration", 9770 ); 9771 } 9772 9773 // Check the source vm has been terminated successful (give it '3s' to settle) 9774 thread::sleep(std::time::Duration::new(3, 0)); 9775 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9776 print_and_panic( 9777 src_child, 9778 dest_child, 9779 None, 9780 "source VM was not terminated successfully.", 9781 ); 9782 }; 9783 9784 // Post live-migration check to make sure the destination VM is funcational 9785 let r = std::panic::catch_unwind(|| { 9786 // Perform same checks to validate VM has been properly migrated 9787 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9788 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9789 9790 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9791 9792 // Perform checks on watchdog 9793 let mut expected_reboot_count = 1; 9794 9795 // Allow some normal time to elapse to check we don't get spurious reboots 9796 thread::sleep(std::time::Duration::new(40, 0)); 9797 // Check no reboot 9798 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9799 9800 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 9801 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 9802 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 9803 guest.wait_vm_boot(Some(50)).unwrap(); 9804 // Check a reboot is triggered by the watchdog 9805 expected_reboot_count += 1; 9806 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9807 9808 #[cfg(target_arch = "x86_64")] 9809 { 9810 // Now pause the VM and remain offline for 30s 9811 assert!(remote_command(&dest_api_socket, "pause", None)); 9812 thread::sleep(std::time::Duration::new(30, 0)); 9813 assert!(remote_command(&dest_api_socket, "resume", None)); 9814 9815 // Check no reboot 9816 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9817 } 9818 }); 9819 9820 // Clean-up the destination VM and make sure it terminated correctly 9821 let _ = dest_child.kill(); 9822 let dest_output = dest_child.wait_with_output().unwrap(); 9823 handle_child_output(r, &dest_output); 9824 9825 // Check the destination VM has the expected 'concole_text' from its output 9826 let r = std::panic::catch_unwind(|| { 9827 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9828 }); 9829 handle_child_output(r, &dest_output); 9830 } 9831 9832 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 9833 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9834 let ovs_guest = Guest::new(Box::new(ovs_focal)); 9835 9836 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9837 let migration_guest = Guest::new(Box::new(migration_focal)); 9838 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 9839 9840 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 9841 let (mut ovs_child, mut src_child) = 9842 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 9843 9844 // Start the destination VM 9845 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 9846 dest_api_socket.push_str(".dest"); 9847 let mut dest_child = GuestCommand::new(&migration_guest) 9848 .args(["--api-socket", &dest_api_socket]) 9849 .capture_output() 9850 .spawn() 9851 .unwrap(); 9852 9853 let r = std::panic::catch_unwind(|| { 9854 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 9855 thread::sleep(std::time::Duration::new(1, 0)); 9856 9857 // Start the live-migration 9858 let migration_socket = String::from( 9859 migration_guest 9860 .tmp_dir 9861 .as_path() 9862 .join("live-migration.sock") 9863 .to_str() 9864 .unwrap(), 9865 ); 9866 9867 assert!( 9868 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9869 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9870 ); 9871 }); 9872 9873 // Check and report any errors occurred during the live-migration 9874 if r.is_err() { 9875 print_and_panic( 9876 src_child, 9877 dest_child, 9878 Some(ovs_child), 9879 "Error occurred during live-migration", 9880 ); 9881 } 9882 9883 // Check the source vm has been terminated successful (give it '3s' to settle) 9884 thread::sleep(std::time::Duration::new(3, 0)); 9885 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9886 print_and_panic( 9887 src_child, 9888 dest_child, 9889 Some(ovs_child), 9890 "source VM was not terminated successfully.", 9891 ); 9892 }; 9893 9894 // Post live-migration check to make sure the destination VM is funcational 9895 let r = std::panic::catch_unwind(|| { 9896 // Perform same checks to validate VM has been properly migrated 9897 // Spawn a new netcat listener in the OVS VM 9898 let guest_ip = ovs_guest.network.guest_ip.clone(); 9899 thread::spawn(move || { 9900 ssh_command_ip( 9901 "nc -l 12345", 9902 &guest_ip, 9903 DEFAULT_SSH_RETRIES, 9904 DEFAULT_SSH_TIMEOUT, 9905 ) 9906 .unwrap(); 9907 }); 9908 9909 // Wait for the server to be listening 9910 thread::sleep(std::time::Duration::new(5, 0)); 9911 9912 // And check the connection is still functional after live-migration 9913 migration_guest 9914 .ssh_command("nc -vz 172.100.0.1 12345") 9915 .unwrap(); 9916 }); 9917 9918 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 9919 let _ = dest_child.kill(); 9920 let _ = ovs_child.kill(); 9921 let dest_output = dest_child.wait_with_output().unwrap(); 9922 let ovs_output = ovs_child.wait_with_output().unwrap(); 9923 9924 cleanup_ovs_dpdk(); 9925 9926 handle_child_output(r, &dest_output); 9927 handle_child_output(Ok(()), &ovs_output); 9928 } 9929 9930 mod live_migration_parallel { 9931 use super::*; 9932 #[test] 9933 fn test_live_migration_basic() { 9934 _test_live_migration(false, false) 9935 } 9936 9937 #[test] 9938 fn test_live_migration_local() { 9939 _test_live_migration(false, true) 9940 } 9941 9942 #[test] 9943 fn test_live_migration_watchdog() { 9944 _test_live_migration_watchdog(false, false) 9945 } 9946 9947 #[test] 9948 fn test_live_migration_watchdog_local() { 9949 _test_live_migration_watchdog(false, true) 9950 } 9951 9952 #[test] 9953 fn test_live_upgrade_basic() { 9954 _test_live_migration(true, false) 9955 } 9956 9957 #[test] 9958 fn test_live_upgrade_local() { 9959 _test_live_migration(true, true) 9960 } 9961 9962 #[test] 9963 fn test_live_upgrade_watchdog() { 9964 _test_live_migration_watchdog(true, false) 9965 } 9966 9967 #[test] 9968 fn test_live_upgrade_watchdog_local() { 9969 _test_live_migration_watchdog(true, true) 9970 } 9971 } 9972 9973 mod live_migration_sequential { 9974 use super::*; 9975 9976 // NUMA & baalloon live migration tests are large so run sequentially 9977 9978 #[test] 9979 fn test_live_migration_balloon() { 9980 _test_live_migration_balloon(false, false) 9981 } 9982 9983 #[test] 9984 fn test_live_migration_balloon_local() { 9985 _test_live_migration_balloon(false, true) 9986 } 9987 9988 #[test] 9989 fn test_live_upgrade_balloon() { 9990 _test_live_migration_balloon(true, false) 9991 } 9992 9993 #[test] 9994 fn test_live_upgrade_balloon_local() { 9995 _test_live_migration_balloon(true, true) 9996 } 9997 9998 #[test] 9999 #[cfg(not(feature = "mshv"))] 10000 fn test_live_migration_numa() { 10001 _test_live_migration_numa(false, false) 10002 } 10003 10004 #[test] 10005 #[cfg(not(feature = "mshv"))] 10006 fn test_live_migration_numa_local() { 10007 _test_live_migration_numa(false, true) 10008 } 10009 10010 #[test] 10011 #[cfg(not(feature = "mshv"))] 10012 fn test_live_upgrade_numa() { 10013 _test_live_migration_numa(true, false) 10014 } 10015 10016 #[test] 10017 #[cfg(not(feature = "mshv"))] 10018 fn test_live_upgrade_numa_local() { 10019 _test_live_migration_numa(true, true) 10020 } 10021 10022 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 10023 #[test] 10024 #[ignore = "See #5532"] 10025 #[cfg(target_arch = "x86_64")] 10026 #[cfg(not(feature = "mshv"))] 10027 fn test_live_migration_ovs_dpdk() { 10028 _test_live_migration_ovs_dpdk(false, false); 10029 } 10030 10031 #[test] 10032 #[cfg(target_arch = "x86_64")] 10033 #[cfg(not(feature = "mshv"))] 10034 fn test_live_migration_ovs_dpdk_local() { 10035 _test_live_migration_ovs_dpdk(false, true); 10036 } 10037 10038 #[test] 10039 #[ignore = "See #5532"] 10040 #[cfg(target_arch = "x86_64")] 10041 #[cfg(not(feature = "mshv"))] 10042 fn test_live_upgrade_ovs_dpdk() { 10043 _test_live_migration_ovs_dpdk(true, false); 10044 } 10045 10046 #[test] 10047 #[ignore = "See #5532"] 10048 #[cfg(target_arch = "x86_64")] 10049 #[cfg(not(feature = "mshv"))] 10050 fn test_live_upgrade_ovs_dpdk_local() { 10051 _test_live_migration_ovs_dpdk(true, true); 10052 } 10053 } 10054 } 10055 10056 #[cfg(target_arch = "aarch64")] 10057 mod aarch64_acpi { 10058 use crate::*; 10059 10060 #[test] 10061 fn test_simple_launch_acpi() { 10062 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10063 10064 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 10065 let guest = Guest::new(disk_config); 10066 10067 let mut child = GuestCommand::new(&guest) 10068 .args(["--cpus", "boot=1"]) 10069 .args(["--memory", "size=512M"]) 10070 .args(["--kernel", edk2_path().to_str().unwrap()]) 10071 .default_disks() 10072 .default_net() 10073 .args(["--serial", "tty", "--console", "off"]) 10074 .capture_output() 10075 .spawn() 10076 .unwrap(); 10077 10078 let r = std::panic::catch_unwind(|| { 10079 guest.wait_vm_boot(Some(120)).unwrap(); 10080 10081 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 10082 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 10083 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 10084 }); 10085 10086 let _ = child.kill(); 10087 let output = child.wait_with_output().unwrap(); 10088 10089 handle_child_output(r, &output); 10090 }); 10091 } 10092 10093 #[test] 10094 fn test_guest_numa_nodes_acpi() { 10095 _test_guest_numa_nodes(true); 10096 } 10097 10098 #[test] 10099 fn test_cpu_topology_421_acpi() { 10100 test_cpu_topology(4, 2, 1, true); 10101 } 10102 10103 #[test] 10104 fn test_cpu_topology_142_acpi() { 10105 test_cpu_topology(1, 4, 2, true); 10106 } 10107 10108 #[test] 10109 fn test_cpu_topology_262_acpi() { 10110 test_cpu_topology(2, 6, 2, true); 10111 } 10112 10113 #[test] 10114 fn test_power_button_acpi() { 10115 _test_power_button(true); 10116 } 10117 10118 #[test] 10119 fn test_virtio_iommu() { 10120 _test_virtio_iommu(true) 10121 } 10122 } 10123 10124 mod rate_limiter { 10125 use super::*; 10126 10127 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 10128 // compared to given 'limit' rate. 10129 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 10130 let upper_limit = limit * (1_f64 + difference); 10131 let lower_limit = limit * (1_f64 - difference); 10132 10133 if measured > lower_limit && measured < upper_limit { 10134 return true; 10135 } 10136 10137 eprintln!( 10138 "\n\n==== Start 'check_rate_limit' failed ==== \ 10139 \n\nmeasured={measured}, , lower_limit={lower_limit}, upper_limit={upper_limit} \ 10140 \n\n==== End 'check_rate_limit' failed ====\n\n" 10141 ); 10142 10143 false 10144 } 10145 10146 fn _test_rate_limiter_net(rx: bool) { 10147 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10148 let guest = Guest::new(Box::new(focal)); 10149 10150 let test_timeout = 10; 10151 let num_queues = 2; 10152 let queue_size = 256; 10153 let bw_size = 10485760_u64; // bytes 10154 let bw_refill_time = 100; // ms 10155 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 10156 10157 let net_params = format!( 10158 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 10159 guest.network.guest_mac, 10160 guest.network.host_ip, 10161 num_queues, 10162 queue_size, 10163 bw_size, 10164 bw_refill_time, 10165 ); 10166 10167 let mut child = GuestCommand::new(&guest) 10168 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 10169 .args(["--memory", "size=4G"]) 10170 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10171 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10172 .default_disks() 10173 .args(["--net", net_params.as_str()]) 10174 .capture_output() 10175 .spawn() 10176 .unwrap(); 10177 10178 let r = std::panic::catch_unwind(|| { 10179 guest.wait_vm_boot(None).unwrap(); 10180 let measured_bps = 10181 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, true) 10182 .unwrap(); 10183 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 10184 }); 10185 10186 let _ = child.kill(); 10187 let output = child.wait_with_output().unwrap(); 10188 handle_child_output(r, &output); 10189 } 10190 10191 #[test] 10192 fn test_rate_limiter_net_rx() { 10193 _test_rate_limiter_net(true); 10194 } 10195 10196 #[test] 10197 fn test_rate_limiter_net_tx() { 10198 _test_rate_limiter_net(false); 10199 } 10200 10201 fn _test_rate_limiter_block(bandwidth: bool, num_queues: u32) { 10202 let test_timeout = 10; 10203 let fio_ops = FioOps::RandRW; 10204 10205 let bw_size = if bandwidth { 10206 10485760_u64 // bytes 10207 } else { 10208 100_u64 // I/O 10209 }; 10210 let bw_refill_time = 100; // ms 10211 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10212 10213 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10214 let guest = Guest::new(Box::new(focal)); 10215 let api_socket = temp_api_path(&guest.tmp_dir); 10216 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10217 let blk_rate_limiter_test_img = 10218 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 10219 10220 // Create the test block image 10221 assert!(exec_host_command_output(&format!( 10222 "dd if=/dev/zero of={blk_rate_limiter_test_img} bs=1M count=1024" 10223 )) 10224 .status 10225 .success()); 10226 10227 let test_blk_params = if bandwidth { 10228 format!( 10229 "path={blk_rate_limiter_test_img},num_queues={num_queues},bw_size={bw_size},bw_refill_time={bw_refill_time}" 10230 ) 10231 } else { 10232 format!( 10233 "path={blk_rate_limiter_test_img},num_queues={num_queues},ops_size={bw_size},ops_refill_time={bw_refill_time}" 10234 ) 10235 }; 10236 10237 let mut child = GuestCommand::new(&guest) 10238 .args(["--cpus", &format!("boot={num_queues}")]) 10239 .args(["--memory", "size=4G"]) 10240 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10241 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10242 .args([ 10243 "--disk", 10244 format!( 10245 "path={}", 10246 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10247 ) 10248 .as_str(), 10249 format!( 10250 "path={}", 10251 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10252 ) 10253 .as_str(), 10254 test_blk_params.as_str(), 10255 ]) 10256 .default_net() 10257 .args(["--api-socket", &api_socket]) 10258 .capture_output() 10259 .spawn() 10260 .unwrap(); 10261 10262 let r = std::panic::catch_unwind(|| { 10263 guest.wait_vm_boot(None).unwrap(); 10264 10265 let fio_command = format!( 10266 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 10267 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10268 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10269 ); 10270 let output = guest.ssh_command(&fio_command).unwrap(); 10271 10272 // Parse fio output 10273 let measured_rate = if bandwidth { 10274 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 10275 } else { 10276 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 10277 }; 10278 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10279 }); 10280 10281 let _ = child.kill(); 10282 let output = child.wait_with_output().unwrap(); 10283 handle_child_output(r, &output); 10284 } 10285 10286 fn _test_rate_limiter_group_block(bandwidth: bool, num_queues: u32, num_disks: u32) { 10287 let test_timeout = 10; 10288 let fio_ops = FioOps::RandRW; 10289 10290 let bw_size = if bandwidth { 10291 10485760_u64 // bytes 10292 } else { 10293 100_u64 // I/O 10294 }; 10295 let bw_refill_time = 100; // ms 10296 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10297 10298 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10299 let guest = Guest::new(Box::new(focal)); 10300 let api_socket = temp_api_path(&guest.tmp_dir); 10301 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10302 10303 let rate_limit_group_arg = if bandwidth { 10304 format!("id=group0,bw_size={bw_size},bw_refill_time={bw_refill_time}") 10305 } else { 10306 format!("id=group0,ops_size={bw_size},ops_refill_time={bw_refill_time}") 10307 }; 10308 10309 let mut disk_args = vec![ 10310 "--disk".to_string(), 10311 format!( 10312 "path={}", 10313 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10314 ), 10315 format!( 10316 "path={}", 10317 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10318 ), 10319 ]; 10320 10321 for i in 0..num_disks { 10322 let test_img_path = String::from( 10323 test_img_dir 10324 .as_path() 10325 .join(format!("blk{}.img", i)) 10326 .to_str() 10327 .unwrap(), 10328 ); 10329 10330 assert!(exec_host_command_output(&format!( 10331 "dd if=/dev/zero of={test_img_path} bs=1M count=1024" 10332 )) 10333 .status 10334 .success()); 10335 10336 disk_args.push(format!( 10337 "path={test_img_path},num_queues={num_queues},rate_limit_group=group0" 10338 )); 10339 } 10340 10341 let mut child = GuestCommand::new(&guest) 10342 .args(["--cpus", &format!("boot={}", num_queues * num_disks)]) 10343 .args(["--memory", "size=4G"]) 10344 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10345 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10346 .args(["--rate-limit-group", &rate_limit_group_arg]) 10347 .args(disk_args) 10348 .default_net() 10349 .args(["--api-socket", &api_socket]) 10350 .capture_output() 10351 .spawn() 10352 .unwrap(); 10353 10354 let r = std::panic::catch_unwind(|| { 10355 guest.wait_vm_boot(None).unwrap(); 10356 10357 let mut fio_command = format!( 10358 "sudo fio --name=global --output-format=json \ 10359 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10360 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10361 ); 10362 10363 // Generate additional argument for each disk: 10364 // --name=job0 --filename=/dev/vdc \ 10365 // --name=job1 --filename=/dev/vdd \ 10366 // --name=job2 --filename=/dev/vde \ 10367 // ... 10368 for i in 0..num_disks { 10369 let c: char = 'c'; 10370 let arg = format!( 10371 " --name=job{i} --filename=/dev/vd{}", 10372 char::from_u32((c as u32) + i).unwrap() 10373 ); 10374 fio_command += &arg; 10375 } 10376 let output = guest.ssh_command(&fio_command).unwrap(); 10377 10378 // Parse fio output 10379 let measured_rate = if bandwidth { 10380 parse_fio_output(&output, &fio_ops, num_queues * num_disks).unwrap() 10381 } else { 10382 parse_fio_output_iops(&output, &fio_ops, num_queues * num_disks).unwrap() 10383 }; 10384 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10385 }); 10386 10387 let _ = child.kill(); 10388 let output = child.wait_with_output().unwrap(); 10389 handle_child_output(r, &output); 10390 } 10391 10392 #[test] 10393 fn test_rate_limiter_block_bandwidth() { 10394 _test_rate_limiter_block(true, 1); 10395 _test_rate_limiter_block(true, 2) 10396 } 10397 10398 #[test] 10399 fn test_rate_limiter_group_block_bandwidth() { 10400 _test_rate_limiter_group_block(true, 1, 1); 10401 _test_rate_limiter_group_block(true, 2, 1); 10402 _test_rate_limiter_group_block(true, 1, 2); 10403 _test_rate_limiter_group_block(true, 2, 2); 10404 } 10405 10406 #[test] 10407 fn test_rate_limiter_block_iops() { 10408 _test_rate_limiter_block(false, 1); 10409 _test_rate_limiter_block(false, 2); 10410 } 10411 10412 #[test] 10413 fn test_rate_limiter_group_block_iops() { 10414 _test_rate_limiter_group_block(false, 1, 1); 10415 _test_rate_limiter_group_block(false, 2, 1); 10416 _test_rate_limiter_group_block(false, 1, 2); 10417 _test_rate_limiter_group_block(false, 2, 2); 10418 } 10419 } 10420