1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 use std::sync::Mutex; 28 use std::thread; 29 use test_infra::*; 30 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 31 use wait_timeout::ChildExt; 32 33 // Constant taken from the VMM crate. 34 const MAX_NUM_PCI_SEGMENTS: u16 = 96; 35 36 #[cfg(target_arch = "x86_64")] 37 mod x86_64 { 38 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 39 pub const JAMMY_NVIDIA_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-nvidia.raw"; 40 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 41 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 42 "focal-server-cloudimg-amd64-custom-20210609-0-backing.qcow2"; 43 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 44 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 45 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20230119-0.raw"; 46 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2022-amd64-2.raw"; 47 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 48 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 49 } 50 51 #[cfg(target_arch = "x86_64")] 52 use x86_64::*; 53 54 #[cfg(target_arch = "aarch64")] 55 mod aarch64 { 56 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 57 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 58 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 59 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 60 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 61 "focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2"; 62 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 63 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 64 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 65 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 66 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 67 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 68 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 69 } 70 71 #[cfg(target_arch = "aarch64")] 72 use aarch64::*; 73 74 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 75 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 76 77 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 78 79 // This enum exists to make it more convenient to 80 // implement test for both D-Bus and REST APIs. 81 enum TargetApi { 82 // API socket 83 HttpApi(String), 84 // well known service name, object path 85 DBusApi(String, String), 86 } 87 88 impl TargetApi { 89 fn new_http_api(tmp_dir: &TempDir) -> Self { 90 Self::HttpApi(temp_api_path(tmp_dir)) 91 } 92 93 fn new_dbus_api(tmp_dir: &TempDir) -> Self { 94 // `tmp_dir` is in the form of "/tmp/chXXXXXX" 95 // and we take the `chXXXXXX` part as a unique identifier for the guest 96 let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); 97 98 Self::DBusApi( 99 format!("org.cloudhypervisor.{id}"), 100 format!("/org/cloudhypervisor/{id}"), 101 ) 102 } 103 104 fn guest_args(&self) -> Vec<String> { 105 match self { 106 TargetApi::HttpApi(api_socket) => { 107 vec![format!("--api-socket={}", api_socket.as_str())] 108 } 109 TargetApi::DBusApi(service_name, object_path) => { 110 vec![ 111 format!("--dbus-service-name={}", service_name.as_str()), 112 format!("--dbus-object-path={}", object_path.as_str()), 113 ] 114 } 115 } 116 } 117 118 fn remote_args(&self) -> Vec<String> { 119 // `guest_args` and `remote_args` are consistent with each other 120 self.guest_args() 121 } 122 123 fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { 124 let mut cmd = Command::new(clh_command("ch-remote")); 125 cmd.args(self.remote_args()); 126 cmd.arg(command); 127 128 if let Some(arg) = arg { 129 cmd.arg(arg); 130 } 131 132 let output = cmd.output().unwrap(); 133 if output.status.success() { 134 true 135 } else { 136 eprintln!("Error running ch-remote command: {:?}", &cmd); 137 let stderr = String::from_utf8_lossy(&output.stderr); 138 eprintln!("stderr: {stderr}"); 139 false 140 } 141 } 142 } 143 144 // Start cloud-hypervisor with no VM parameters, only the API server running. 145 // From the API: Create a VM, boot it and check that it looks as expected. 146 fn _test_api_create_boot(target_api: TargetApi, guest: Guest) { 147 let mut child = GuestCommand::new(&guest) 148 .args(target_api.guest_args()) 149 .capture_output() 150 .spawn() 151 .unwrap(); 152 153 thread::sleep(std::time::Duration::new(1, 0)); 154 155 // Verify API server is running 156 assert!(target_api.remote_command("ping", None)); 157 158 // Create the VM first 159 let cpu_count: u8 = 4; 160 let request_body = guest.api_create_body( 161 cpu_count, 162 direct_kernel_boot_path().to_str().unwrap(), 163 DIRECT_KERNEL_BOOT_CMDLINE, 164 ); 165 166 let temp_config_path = guest.tmp_dir.as_path().join("config"); 167 std::fs::write(&temp_config_path, request_body).unwrap(); 168 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 169 170 assert!(target_api.remote_command("create", Some(create_config),)); 171 172 // Then boot it 173 assert!(target_api.remote_command("boot", None)); 174 thread::sleep(std::time::Duration::new(20, 0)); 175 176 let r = std::panic::catch_unwind(|| { 177 // Check that the VM booted as expected 178 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 179 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 180 }); 181 182 let _ = child.kill(); 183 let output = child.wait_with_output().unwrap(); 184 185 handle_child_output(r, &output); 186 } 187 188 // Start cloud-hypervisor with no VM parameters, only the API server running. 189 // From the API: Create a VM, boot it and check it can be shutdown and then 190 // booted again 191 fn _test_api_shutdown(target_api: TargetApi, guest: Guest) { 192 let mut child = GuestCommand::new(&guest) 193 .args(target_api.guest_args()) 194 .capture_output() 195 .spawn() 196 .unwrap(); 197 198 thread::sleep(std::time::Duration::new(1, 0)); 199 200 // Verify API server is running 201 assert!(target_api.remote_command("ping", None)); 202 203 // Create the VM first 204 let cpu_count: u8 = 4; 205 let request_body = guest.api_create_body( 206 cpu_count, 207 direct_kernel_boot_path().to_str().unwrap(), 208 DIRECT_KERNEL_BOOT_CMDLINE, 209 ); 210 211 let temp_config_path = guest.tmp_dir.as_path().join("config"); 212 std::fs::write(&temp_config_path, request_body).unwrap(); 213 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 214 215 let r = std::panic::catch_unwind(|| { 216 assert!(target_api.remote_command("create", Some(create_config))); 217 218 // Then boot it 219 assert!(target_api.remote_command("boot", None)); 220 221 guest.wait_vm_boot(None).unwrap(); 222 223 // Check that the VM booted as expected 224 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 225 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 226 227 // Sync and shutdown without powering off to prevent filesystem 228 // corruption. 229 guest.ssh_command("sync").unwrap(); 230 guest.ssh_command("sudo shutdown -H now").unwrap(); 231 232 // Wait for the guest to be fully shutdown 233 thread::sleep(std::time::Duration::new(20, 0)); 234 235 // Then shut it down 236 assert!(target_api.remote_command("shutdown", None)); 237 238 // Then boot it again 239 assert!(target_api.remote_command("boot", None)); 240 241 guest.wait_vm_boot(None).unwrap(); 242 243 // Check that the VM booted as expected 244 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 245 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 246 }); 247 248 let _ = child.kill(); 249 let output = child.wait_with_output().unwrap(); 250 251 handle_child_output(r, &output); 252 } 253 254 // Start cloud-hypervisor with no VM parameters, only the API server running. 255 // From the API: Create a VM, boot it and check it can be deleted and then recreated 256 // booted again. 257 fn _test_api_delete(target_api: TargetApi, guest: Guest) { 258 let mut child = GuestCommand::new(&guest) 259 .args(target_api.guest_args()) 260 .capture_output() 261 .spawn() 262 .unwrap(); 263 264 thread::sleep(std::time::Duration::new(1, 0)); 265 266 // Verify API server is running 267 assert!(target_api.remote_command("ping", None)); 268 269 // Create the VM first 270 let cpu_count: u8 = 4; 271 let request_body = guest.api_create_body( 272 cpu_count, 273 direct_kernel_boot_path().to_str().unwrap(), 274 DIRECT_KERNEL_BOOT_CMDLINE, 275 ); 276 let temp_config_path = guest.tmp_dir.as_path().join("config"); 277 std::fs::write(&temp_config_path, request_body).unwrap(); 278 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 279 280 let r = std::panic::catch_unwind(|| { 281 assert!(target_api.remote_command("create", Some(create_config))); 282 283 // Then boot it 284 assert!(target_api.remote_command("boot", None)); 285 286 guest.wait_vm_boot(None).unwrap(); 287 288 // Check that the VM booted as expected 289 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 290 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 291 292 // Sync and shutdown without powering off to prevent filesystem 293 // corruption. 294 guest.ssh_command("sync").unwrap(); 295 guest.ssh_command("sudo shutdown -H now").unwrap(); 296 297 // Wait for the guest to be fully shutdown 298 thread::sleep(std::time::Duration::new(20, 0)); 299 300 // Then delete it 301 assert!(target_api.remote_command("delete", None)); 302 303 assert!(target_api.remote_command("create", Some(create_config))); 304 305 // Then boot it again 306 assert!(target_api.remote_command("boot", None)); 307 308 guest.wait_vm_boot(None).unwrap(); 309 310 // Check that the VM booted as expected 311 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 312 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 313 }); 314 315 let _ = child.kill(); 316 let output = child.wait_with_output().unwrap(); 317 318 handle_child_output(r, &output); 319 } 320 321 // Start cloud-hypervisor with no VM parameters, only the API server running. 322 // From the API: Create a VM, boot it and check that it looks as expected. 323 // Then we pause the VM, check that it's no longer available. 324 // Finally we resume the VM and check that it's available. 325 fn _test_api_pause_resume(target_api: TargetApi, guest: Guest) { 326 let mut child = GuestCommand::new(&guest) 327 .args(target_api.guest_args()) 328 .capture_output() 329 .spawn() 330 .unwrap(); 331 332 thread::sleep(std::time::Duration::new(1, 0)); 333 334 // Verify API server is running 335 assert!(target_api.remote_command("ping", None)); 336 337 // Create the VM first 338 let cpu_count: u8 = 4; 339 let request_body = guest.api_create_body( 340 cpu_count, 341 direct_kernel_boot_path().to_str().unwrap(), 342 DIRECT_KERNEL_BOOT_CMDLINE, 343 ); 344 345 let temp_config_path = guest.tmp_dir.as_path().join("config"); 346 std::fs::write(&temp_config_path, request_body).unwrap(); 347 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 348 349 assert!(target_api.remote_command("create", Some(create_config))); 350 351 // Then boot it 352 assert!(target_api.remote_command("boot", None)); 353 thread::sleep(std::time::Duration::new(20, 0)); 354 355 let r = std::panic::catch_unwind(|| { 356 // Check that the VM booted as expected 357 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 358 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 359 360 // We now pause the VM 361 assert!(target_api.remote_command("pause", None)); 362 363 // Check pausing again fails 364 assert!(!target_api.remote_command("pause", None)); 365 366 thread::sleep(std::time::Duration::new(2, 0)); 367 368 // SSH into the VM should fail 369 assert!(ssh_command_ip( 370 "grep -c processor /proc/cpuinfo", 371 &guest.network.guest_ip, 372 2, 373 5 374 ) 375 .is_err()); 376 377 // Resume the VM 378 assert!(target_api.remote_command("resume", None)); 379 380 // Check resuming again fails 381 assert!(!target_api.remote_command("resume", None)); 382 383 thread::sleep(std::time::Duration::new(2, 0)); 384 385 // Now we should be able to SSH back in and get the right number of CPUs 386 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 387 }); 388 389 let _ = child.kill(); 390 let output = child.wait_with_output().unwrap(); 391 392 handle_child_output(r, &output); 393 } 394 395 fn _test_pty_interaction(pty_path: PathBuf) { 396 let mut cf = std::fs::OpenOptions::new() 397 .write(true) 398 .read(true) 399 .open(pty_path) 400 .unwrap(); 401 402 // Some dumb sleeps but we don't want to write 403 // before the console is up and we don't want 404 // to try and write the next line before the 405 // login process is ready. 406 thread::sleep(std::time::Duration::new(5, 0)); 407 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 408 thread::sleep(std::time::Duration::new(2, 0)); 409 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 410 thread::sleep(std::time::Duration::new(2, 0)); 411 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 412 thread::sleep(std::time::Duration::new(2, 0)); 413 414 // read pty and ensure they have a login shell 415 // some fairly hacky workarounds to avoid looping 416 // forever in case the channel is blocked getting output 417 let ptyc = pty_read(cf); 418 let mut empty = 0; 419 let mut prev = String::new(); 420 loop { 421 thread::sleep(std::time::Duration::new(2, 0)); 422 match ptyc.try_recv() { 423 Ok(line) => { 424 empty = 0; 425 prev = prev + &line; 426 if prev.contains("test_pty_console") { 427 break; 428 } 429 } 430 Err(mpsc::TryRecvError::Empty) => { 431 empty += 1; 432 assert!(empty <= 5, "No login on pty"); 433 } 434 _ => { 435 panic!("No login on pty") 436 } 437 } 438 } 439 } 440 441 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 442 let mut workload_path = dirs::home_dir().unwrap(); 443 workload_path.push("workloads"); 444 445 let mut virtiofsd_path = workload_path; 446 virtiofsd_path.push("virtiofsd"); 447 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 448 449 let virtiofsd_socket_path = 450 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 451 452 // Start the daemon 453 let child = Command::new(virtiofsd_path.as_str()) 454 .args(["--shared-dir", shared_dir]) 455 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 456 .args(["--cache", "never"]) 457 .spawn() 458 .unwrap(); 459 460 thread::sleep(std::time::Duration::new(10, 0)); 461 462 (child, virtiofsd_socket_path) 463 } 464 465 fn prepare_vubd( 466 tmp_dir: &TempDir, 467 blk_img: &str, 468 num_queues: usize, 469 rdonly: bool, 470 direct: bool, 471 ) -> (std::process::Child, String) { 472 let mut workload_path = dirs::home_dir().unwrap(); 473 workload_path.push("workloads"); 474 475 let mut blk_file_path = workload_path; 476 blk_file_path.push(blk_img); 477 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 478 479 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 480 481 // Start the daemon 482 let child = Command::new(clh_command("vhost_user_block")) 483 .args([ 484 "--block-backend", 485 format!( 486 "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" 487 ) 488 .as_str(), 489 ]) 490 .spawn() 491 .unwrap(); 492 493 thread::sleep(std::time::Duration::new(10, 0)); 494 495 (child, vubd_socket_path) 496 } 497 498 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 499 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 500 } 501 502 fn temp_api_path(tmp_dir: &TempDir) -> String { 503 String::from( 504 tmp_dir 505 .as_path() 506 .join("cloud-hypervisor.sock") 507 .to_str() 508 .unwrap(), 509 ) 510 } 511 512 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 513 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 514 } 515 516 // Creates the directory and returns the path. 517 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 518 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 519 std::fs::create_dir(&snapshot_dir).unwrap(); 520 snapshot_dir 521 } 522 523 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 524 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 525 vmcore_file 526 } 527 528 // Creates the path for direct kernel boot and return the path. 529 // For x86_64, this function returns the vmlinux kernel path. 530 // For AArch64, this function returns the PE kernel path. 531 fn direct_kernel_boot_path() -> PathBuf { 532 let mut workload_path = dirs::home_dir().unwrap(); 533 workload_path.push("workloads"); 534 535 let mut kernel_path = workload_path; 536 #[cfg(target_arch = "x86_64")] 537 kernel_path.push("vmlinux"); 538 #[cfg(target_arch = "aarch64")] 539 kernel_path.push("Image"); 540 541 kernel_path 542 } 543 544 fn edk2_path() -> PathBuf { 545 let mut workload_path = dirs::home_dir().unwrap(); 546 workload_path.push("workloads"); 547 let mut edk2_path = workload_path; 548 edk2_path.push(OVMF_NAME); 549 550 edk2_path 551 } 552 553 fn cloud_hypervisor_release_path() -> String { 554 let mut workload_path = dirs::home_dir().unwrap(); 555 workload_path.push("workloads"); 556 557 let mut ch_release_path = workload_path; 558 #[cfg(target_arch = "x86_64")] 559 ch_release_path.push("cloud-hypervisor-static"); 560 #[cfg(target_arch = "aarch64")] 561 ch_release_path.push("cloud-hypervisor-static-aarch64"); 562 563 ch_release_path.into_os_string().into_string().unwrap() 564 } 565 566 fn prepare_vhost_user_net_daemon( 567 tmp_dir: &TempDir, 568 ip: &str, 569 tap: Option<&str>, 570 mtu: Option<u16>, 571 num_queues: usize, 572 client_mode: bool, 573 ) -> (std::process::Command, String) { 574 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 575 576 // Start the daemon 577 let mut net_params = format!( 578 "ip={ip},mask=255.255.255.0,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" 579 ); 580 581 if let Some(tap) = tap { 582 net_params.push_str(format!(",tap={tap}").as_str()); 583 } 584 585 if let Some(mtu) = mtu { 586 net_params.push_str(format!(",mtu={mtu}").as_str()); 587 } 588 589 let mut command = Command::new(clh_command("vhost_user_net")); 590 command.args(["--net-backend", net_params.as_str()]); 591 592 (command, vunet_socket_path) 593 } 594 595 fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { 596 let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); 597 let swtpm_socket_path = String::from( 598 tmp_dir 599 .as_path() 600 .join("swtpm") 601 .join("swtpm.sock") 602 .to_str() 603 .unwrap(), 604 ); 605 std::fs::create_dir(&swtpm_tpm_dir).unwrap(); 606 607 let mut swtpm_command = Command::new("swtpm"); 608 let swtpm_args = [ 609 "socket", 610 "--tpmstate", 611 &format!("dir={swtpm_tpm_dir}"), 612 "--ctrl", 613 &format!("type=unixio,path={swtpm_socket_path}"), 614 "--flags", 615 "startup-clear", 616 "--tpm2", 617 ]; 618 swtpm_command.args(swtpm_args); 619 620 (swtpm_command, swtpm_socket_path) 621 } 622 623 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 624 let mut cmd = Command::new(clh_command("ch-remote")); 625 cmd.args([&format!("--api-socket={api_socket}"), command]); 626 627 if let Some(arg) = arg { 628 cmd.arg(arg); 629 } 630 let output = cmd.output().unwrap(); 631 if output.status.success() { 632 true 633 } else { 634 eprintln!("Error running ch-remote command: {:?}", &cmd); 635 let stderr = String::from_utf8_lossy(&output.stderr); 636 eprintln!("stderr: {stderr}"); 637 false 638 } 639 } 640 641 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 642 let mut cmd = Command::new(clh_command("ch-remote")); 643 cmd.args([&format!("--api-socket={api_socket}"), command]); 644 645 if let Some(arg) = arg { 646 cmd.arg(arg); 647 } 648 649 let output = cmd.output().expect("Failed to launch ch-remote"); 650 651 (output.status.success(), output.stdout) 652 } 653 654 fn resize_command( 655 api_socket: &str, 656 desired_vcpus: Option<u8>, 657 desired_ram: Option<usize>, 658 desired_balloon: Option<usize>, 659 event_file: Option<&str>, 660 ) -> bool { 661 let mut cmd = Command::new(clh_command("ch-remote")); 662 cmd.args([&format!("--api-socket={api_socket}"), "resize"]); 663 664 if let Some(desired_vcpus) = desired_vcpus { 665 cmd.arg(format!("--cpus={desired_vcpus}")); 666 } 667 668 if let Some(desired_ram) = desired_ram { 669 cmd.arg(format!("--memory={desired_ram}")); 670 } 671 672 if let Some(desired_balloon) = desired_balloon { 673 cmd.arg(format!("--balloon={desired_balloon}")); 674 } 675 676 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 677 678 if let Some(event_path) = event_file { 679 let latest_events = [ 680 &MetaEvent { 681 event: "resizing".to_string(), 682 device_id: None, 683 }, 684 &MetaEvent { 685 event: "resized".to_string(), 686 device_id: None, 687 }, 688 ]; 689 // See: #5938 690 thread::sleep(std::time::Duration::new(1, 0)); 691 assert!(check_latest_events_exact(&latest_events, event_path)); 692 } 693 694 ret 695 } 696 697 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 698 let mut cmd = Command::new(clh_command("ch-remote")); 699 cmd.args([ 700 &format!("--api-socket={api_socket}"), 701 "resize-zone", 702 &format!("--id={id}"), 703 &format!("--size={desired_size}"), 704 ]); 705 706 cmd.status().expect("Failed to launch ch-remote").success() 707 } 708 709 // setup OVS-DPDK bridge and ports 710 fn setup_ovs_dpdk() { 711 // setup OVS-DPDK 712 assert!(exec_host_command_status("service openvswitch-switch start").success()); 713 assert!(exec_host_command_status("ovs-vsctl init").success()); 714 assert!( 715 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 716 .success() 717 ); 718 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 719 720 // Create OVS-DPDK bridge and ports 721 assert!(exec_host_command_status( 722 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 723 ) 724 .success()); 725 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 726 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 727 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 728 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 729 } 730 fn cleanup_ovs_dpdk() { 731 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 732 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 733 } 734 // Setup two guests and ensure they are connected through ovs-dpdk 735 fn setup_ovs_dpdk_guests( 736 guest1: &Guest, 737 guest2: &Guest, 738 api_socket: &str, 739 release_binary: bool, 740 ) -> (Child, Child) { 741 setup_ovs_dpdk(); 742 743 let clh_path = if !release_binary { 744 clh_command("cloud-hypervisor") 745 } else { 746 cloud_hypervisor_release_path() 747 }; 748 749 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 750 .args(["--cpus", "boot=2"]) 751 .args(["--memory", "size=0,shared=on"]) 752 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 753 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 754 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 755 .default_disks() 756 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 757 .capture_output() 758 .spawn() 759 .unwrap(); 760 761 #[cfg(target_arch = "x86_64")] 762 let guest_net_iface = "ens5"; 763 #[cfg(target_arch = "aarch64")] 764 let guest_net_iface = "enp0s5"; 765 766 let r = std::panic::catch_unwind(|| { 767 guest1.wait_vm_boot(None).unwrap(); 768 769 guest1 770 .ssh_command(&format!( 771 "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" 772 )) 773 .unwrap(); 774 guest1 775 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 776 .unwrap(); 777 778 let guest_ip = guest1.network.guest_ip.clone(); 779 thread::spawn(move || { 780 ssh_command_ip( 781 "nc -l 12345", 782 &guest_ip, 783 DEFAULT_SSH_RETRIES, 784 DEFAULT_SSH_TIMEOUT, 785 ) 786 .unwrap(); 787 }); 788 }); 789 if r.is_err() { 790 cleanup_ovs_dpdk(); 791 792 let _ = child1.kill(); 793 let output = child1.wait_with_output().unwrap(); 794 handle_child_output(r, &output); 795 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 796 } 797 798 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 799 .args(["--api-socket", api_socket]) 800 .args(["--cpus", "boot=2"]) 801 .args(["--memory", "size=0,shared=on"]) 802 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 803 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 804 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 805 .default_disks() 806 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 807 .capture_output() 808 .spawn() 809 .unwrap(); 810 811 let r = std::panic::catch_unwind(|| { 812 guest2.wait_vm_boot(None).unwrap(); 813 814 guest2 815 .ssh_command(&format!( 816 "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" 817 )) 818 .unwrap(); 819 guest2 820 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 821 .unwrap(); 822 823 // Check the connection works properly between the two VMs 824 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 825 }); 826 if r.is_err() { 827 cleanup_ovs_dpdk(); 828 829 let _ = child1.kill(); 830 let _ = child2.kill(); 831 let output = child2.wait_with_output().unwrap(); 832 handle_child_output(r, &output); 833 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 834 } 835 836 (child1, child2) 837 } 838 839 enum FwType { 840 Ovmf, 841 RustHypervisorFirmware, 842 } 843 844 fn fw_path(_fw_type: FwType) -> String { 845 let mut workload_path = dirs::home_dir().unwrap(); 846 workload_path.push("workloads"); 847 848 let mut fw_path = workload_path; 849 #[cfg(target_arch = "aarch64")] 850 fw_path.push("CLOUDHV_EFI.fd"); 851 #[cfg(target_arch = "x86_64")] 852 { 853 match _fw_type { 854 FwType::Ovmf => fw_path.push(OVMF_NAME), 855 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 856 } 857 } 858 859 fw_path.to_str().unwrap().to_string() 860 } 861 862 #[derive(Debug)] 863 struct MetaEvent { 864 event: String, 865 device_id: Option<String>, 866 } 867 868 impl MetaEvent { 869 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 870 let mut matched = false; 871 if v["event"].as_str().unwrap() == self.event { 872 if let Some(device_id) = &self.device_id { 873 if v["properties"]["id"].as_str().unwrap() == device_id { 874 matched = true 875 } 876 } else { 877 matched = true; 878 } 879 } 880 matched 881 } 882 } 883 884 // Parse the event_monitor file based on the format that each event 885 // is followed by a double newline 886 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 887 let content = fs::read(event_file).unwrap(); 888 let mut ret = Vec::new(); 889 for entry in String::from_utf8_lossy(&content) 890 .trim() 891 .split("\n\n") 892 .collect::<Vec<&str>>() 893 { 894 ret.push(serde_json::from_str(entry).unwrap()); 895 } 896 897 ret 898 } 899 900 // Return true if all events from the input 'expected_events' are matched sequentially 901 // with events from the 'event_file' 902 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 903 let json_events = parse_event_file(event_file); 904 let len = expected_events.len(); 905 let mut idx = 0; 906 for e in &json_events { 907 if idx == len { 908 break; 909 } 910 if expected_events[idx].match_with_json_event(e) { 911 idx += 1; 912 } 913 } 914 915 let ret = idx == len; 916 917 if !ret { 918 eprintln!( 919 "\n\n==== Start 'check_sequential_events' failed ==== \ 920 \n\nexpected_events={:?}\nactual_events={:?} \ 921 \n\n==== End 'check_sequential_events' failed ====", 922 expected_events, json_events, 923 ); 924 } 925 926 ret 927 } 928 929 // Return true if all events from the input 'expected_events' are matched exactly 930 // with events from the 'event_file' 931 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 932 let json_events = parse_event_file(event_file); 933 assert!(expected_events.len() <= json_events.len()); 934 let json_events = &json_events[..expected_events.len()]; 935 936 for (idx, e) in json_events.iter().enumerate() { 937 if !expected_events[idx].match_with_json_event(e) { 938 eprintln!( 939 "\n\n==== Start 'check_sequential_events_exact' failed ==== \ 940 \n\nexpected_events={:?}\nactual_events={:?} \ 941 \n\n==== End 'check_sequential_events_exact' failed ====", 942 expected_events, json_events, 943 ); 944 945 return false; 946 } 947 } 948 949 true 950 } 951 952 // Return true if events from the input 'latest_events' are matched exactly 953 // with the most recent events from the 'event_file' 954 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 955 let json_events = parse_event_file(event_file); 956 assert!(latest_events.len() <= json_events.len()); 957 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 958 959 for (idx, e) in json_events.iter().enumerate() { 960 if !latest_events[idx].match_with_json_event(e) { 961 eprintln!( 962 "\n\n==== Start 'check_latest_events_exact' failed ==== \ 963 \n\nexpected_events={:?}\nactual_events={:?} \ 964 \n\n==== End 'check_latest_events_exact' failed ====", 965 latest_events, json_events, 966 ); 967 968 return false; 969 } 970 } 971 972 true 973 } 974 975 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 976 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 977 let guest = Guest::new(Box::new(focal)); 978 let total_vcpus = threads_per_core * cores_per_package * packages; 979 let direct_kernel_boot_path = direct_kernel_boot_path(); 980 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 981 let fw_path = fw_path(FwType::RustHypervisorFirmware); 982 if use_fw { 983 kernel_path = fw_path.as_str(); 984 } 985 986 let mut child = GuestCommand::new(&guest) 987 .args([ 988 "--cpus", 989 &format!( 990 "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" 991 ), 992 ]) 993 .args(["--memory", "size=512M"]) 994 .args(["--kernel", kernel_path]) 995 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 996 .default_disks() 997 .default_net() 998 .capture_output() 999 .spawn() 1000 .unwrap(); 1001 1002 let r = std::panic::catch_unwind(|| { 1003 guest.wait_vm_boot(None).unwrap(); 1004 assert_eq!( 1005 guest.get_cpu_count().unwrap_or_default(), 1006 u32::from(total_vcpus) 1007 ); 1008 assert_eq!( 1009 guest 1010 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1011 .unwrap() 1012 .trim() 1013 .parse::<u8>() 1014 .unwrap_or(0), 1015 threads_per_core 1016 ); 1017 1018 assert_eq!( 1019 guest 1020 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1021 .unwrap() 1022 .trim() 1023 .parse::<u8>() 1024 .unwrap_or(0), 1025 cores_per_package 1026 ); 1027 1028 assert_eq!( 1029 guest 1030 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1031 .unwrap() 1032 .trim() 1033 .parse::<u8>() 1034 .unwrap_or(0), 1035 packages 1036 ); 1037 1038 #[cfg(target_arch = "x86_64")] 1039 { 1040 let mut cpu_id = 0; 1041 for package_id in 0..packages { 1042 for core_id in 0..cores_per_package { 1043 for _ in 0..threads_per_core { 1044 assert_eq!( 1045 guest 1046 .ssh_command(&format!("cat /sys/devices/system/cpu/cpu{cpu_id}/topology/physical_package_id")) 1047 .unwrap() 1048 .trim() 1049 .parse::<u8>() 1050 .unwrap_or(0), 1051 package_id 1052 ); 1053 1054 assert_eq!( 1055 guest 1056 .ssh_command(&format!( 1057 "cat /sys/devices/system/cpu/cpu{cpu_id}/topology/core_id" 1058 )) 1059 .unwrap() 1060 .trim() 1061 .parse::<u8>() 1062 .unwrap_or(0), 1063 core_id 1064 ); 1065 1066 cpu_id += 1; 1067 } 1068 } 1069 } 1070 } 1071 }); 1072 1073 let _ = child.kill(); 1074 let output = child.wait_with_output().unwrap(); 1075 1076 handle_child_output(r, &output); 1077 } 1078 1079 #[allow(unused_variables)] 1080 fn _test_guest_numa_nodes(acpi: bool) { 1081 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1082 let guest = Guest::new(Box::new(focal)); 1083 let api_socket = temp_api_path(&guest.tmp_dir); 1084 #[cfg(target_arch = "x86_64")] 1085 let kernel_path = direct_kernel_boot_path(); 1086 #[cfg(target_arch = "aarch64")] 1087 let kernel_path = if acpi { 1088 edk2_path() 1089 } else { 1090 direct_kernel_boot_path() 1091 }; 1092 1093 let mut child = GuestCommand::new(&guest) 1094 .args(["--cpus", "boot=6,max=12"]) 1095 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 1096 .args([ 1097 "--memory-zone", 1098 "id=mem0,size=1G,hotplug_size=3G", 1099 "id=mem1,size=2G,hotplug_size=3G", 1100 "id=mem2,size=3G,hotplug_size=3G", 1101 ]) 1102 .args([ 1103 "--numa", 1104 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 1105 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 1106 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 1107 ]) 1108 .args(["--kernel", kernel_path.to_str().unwrap()]) 1109 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1110 .args(["--api-socket", &api_socket]) 1111 .capture_output() 1112 .default_disks() 1113 .default_net() 1114 .spawn() 1115 .unwrap(); 1116 1117 let r = std::panic::catch_unwind(|| { 1118 guest.wait_vm_boot(None).unwrap(); 1119 1120 guest.check_numa_common( 1121 Some(&[960_000, 1_920_000, 2_880_000]), 1122 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 1123 Some(&["10 15 20", "20 10 25", "25 30 10"]), 1124 ); 1125 1126 // AArch64 currently does not support hotplug, and therefore we only 1127 // test hotplug-related function on x86_64 here. 1128 #[cfg(target_arch = "x86_64")] 1129 { 1130 guest.enable_memory_hotplug(); 1131 1132 // Resize every memory zone and check each associated NUMA node 1133 // has been assigned the right amount of memory. 1134 resize_zone_command(&api_socket, "mem0", "4G"); 1135 resize_zone_command(&api_socket, "mem1", "4G"); 1136 resize_zone_command(&api_socket, "mem2", "4G"); 1137 // Resize to the maximum amount of CPUs and check each NUMA 1138 // node has been assigned the right CPUs set. 1139 resize_command(&api_socket, Some(12), None, None, None); 1140 thread::sleep(std::time::Duration::new(5, 0)); 1141 1142 guest.check_numa_common( 1143 Some(&[3_840_000, 3_840_000, 3_840_000]), 1144 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 1145 None, 1146 ); 1147 } 1148 }); 1149 1150 let _ = child.kill(); 1151 let output = child.wait_with_output().unwrap(); 1152 1153 handle_child_output(r, &output); 1154 } 1155 1156 #[allow(unused_variables)] 1157 fn _test_power_button(acpi: bool) { 1158 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1159 let guest = Guest::new(Box::new(focal)); 1160 let mut cmd = GuestCommand::new(&guest); 1161 let api_socket = temp_api_path(&guest.tmp_dir); 1162 1163 #[cfg(target_arch = "x86_64")] 1164 let kernel_path = direct_kernel_boot_path(); 1165 #[cfg(target_arch = "aarch64")] 1166 let kernel_path = if acpi { 1167 edk2_path() 1168 } else { 1169 direct_kernel_boot_path() 1170 }; 1171 1172 cmd.args(["--cpus", "boot=1"]) 1173 .args(["--memory", "size=512M"]) 1174 .args(["--kernel", kernel_path.to_str().unwrap()]) 1175 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1176 .capture_output() 1177 .default_disks() 1178 .default_net() 1179 .args(["--api-socket", &api_socket]); 1180 1181 let child = cmd.spawn().unwrap(); 1182 1183 let r = std::panic::catch_unwind(|| { 1184 guest.wait_vm_boot(None).unwrap(); 1185 assert!(remote_command(&api_socket, "power-button", None)); 1186 }); 1187 1188 let output = child.wait_with_output().unwrap(); 1189 assert!(output.status.success()); 1190 handle_child_output(r, &output); 1191 } 1192 1193 type PrepareNetDaemon = dyn Fn( 1194 &TempDir, 1195 &str, 1196 Option<&str>, 1197 Option<u16>, 1198 usize, 1199 bool, 1200 ) -> (std::process::Command, String); 1201 1202 fn test_vhost_user_net( 1203 tap: Option<&str>, 1204 num_queues: usize, 1205 prepare_daemon: &PrepareNetDaemon, 1206 generate_host_mac: bool, 1207 client_mode_daemon: bool, 1208 ) { 1209 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1210 let guest = Guest::new(Box::new(focal)); 1211 let api_socket = temp_api_path(&guest.tmp_dir); 1212 1213 let kernel_path = direct_kernel_boot_path(); 1214 1215 let host_mac = if generate_host_mac { 1216 Some(MacAddr::local_random()) 1217 } else { 1218 None 1219 }; 1220 1221 let mtu = Some(3000); 1222 1223 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 1224 &guest.tmp_dir, 1225 &guest.network.host_ip, 1226 tap, 1227 mtu, 1228 num_queues, 1229 client_mode_daemon, 1230 ); 1231 1232 let net_params = format!( 1233 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 1234 guest.network.guest_mac, 1235 vunet_socket_path, 1236 num_queues, 1237 if let Some(host_mac) = host_mac { 1238 format!(",host_mac={host_mac}") 1239 } else { 1240 "".to_owned() 1241 }, 1242 if client_mode_daemon { 1243 "server" 1244 } else { 1245 "client" 1246 }, 1247 ); 1248 1249 let mut ch_command = GuestCommand::new(&guest); 1250 ch_command 1251 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 1252 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1253 .args(["--kernel", kernel_path.to_str().unwrap()]) 1254 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1255 .default_disks() 1256 .args(["--net", net_params.as_str()]) 1257 .args(["--api-socket", &api_socket]) 1258 .capture_output(); 1259 1260 let mut daemon_child: std::process::Child; 1261 let mut child: std::process::Child; 1262 1263 if client_mode_daemon { 1264 child = ch_command.spawn().unwrap(); 1265 // Make sure the VMM is waiting for the backend to connect 1266 thread::sleep(std::time::Duration::new(10, 0)); 1267 daemon_child = daemon_command.spawn().unwrap(); 1268 } else { 1269 daemon_child = daemon_command.spawn().unwrap(); 1270 // Make sure the backend is waiting for the VMM to connect 1271 thread::sleep(std::time::Duration::new(10, 0)); 1272 child = ch_command.spawn().unwrap(); 1273 } 1274 1275 let r = std::panic::catch_unwind(|| { 1276 guest.wait_vm_boot(None).unwrap(); 1277 1278 if let Some(tap_name) = tap { 1279 let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); 1280 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1281 } 1282 1283 if let Some(host_mac) = tap { 1284 let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); 1285 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1286 } 1287 1288 #[cfg(target_arch = "aarch64")] 1289 let iface = "enp0s4"; 1290 #[cfg(target_arch = "x86_64")] 1291 let iface = "ens4"; 1292 1293 assert_eq!( 1294 guest 1295 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 1296 .unwrap() 1297 .trim(), 1298 "3000" 1299 ); 1300 1301 // 1 network interface + default localhost ==> 2 interfaces 1302 // It's important to note that this test is fully exercising the 1303 // vhost-user-net implementation and the associated backend since 1304 // it does not define any --net network interface. That means all 1305 // the ssh communication in that test happens through the network 1306 // interface backed by vhost-user-net. 1307 assert_eq!( 1308 guest 1309 .ssh_command("ip -o link | wc -l") 1310 .unwrap() 1311 .trim() 1312 .parse::<u32>() 1313 .unwrap_or_default(), 1314 2 1315 ); 1316 1317 // The following pci devices will appear on guest with PCI-MSI 1318 // interrupt vectors assigned. 1319 // 1 virtio-console with 3 vectors: config, Rx, Tx 1320 // 1 virtio-blk with 2 vectors: config, Request 1321 // 1 virtio-blk with 2 vectors: config, Request 1322 // 1 virtio-rng with 2 vectors: config, Request 1323 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1324 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1325 // Based on the above, the total vectors should 14. 1326 #[cfg(target_arch = "x86_64")] 1327 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1328 #[cfg(target_arch = "aarch64")] 1329 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1330 assert_eq!( 1331 guest 1332 .ssh_command(grep_cmd) 1333 .unwrap() 1334 .trim() 1335 .parse::<u32>() 1336 .unwrap_or_default(), 1337 10 + (num_queues as u32) 1338 ); 1339 1340 // ACPI feature is needed. 1341 #[cfg(target_arch = "x86_64")] 1342 { 1343 guest.enable_memory_hotplug(); 1344 1345 // Add RAM to the VM 1346 let desired_ram = 1024 << 20; 1347 resize_command(&api_socket, None, Some(desired_ram), None, None); 1348 1349 thread::sleep(std::time::Duration::new(10, 0)); 1350 1351 // Here by simply checking the size (through ssh), we validate 1352 // the connection is still working, which means vhost-user-net 1353 // keeps working after the resize. 1354 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1355 } 1356 }); 1357 1358 let _ = child.kill(); 1359 let output = child.wait_with_output().unwrap(); 1360 1361 thread::sleep(std::time::Duration::new(5, 0)); 1362 let _ = daemon_child.kill(); 1363 let _ = daemon_child.wait(); 1364 1365 handle_child_output(r, &output); 1366 } 1367 1368 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1369 1370 fn test_vhost_user_blk( 1371 num_queues: usize, 1372 readonly: bool, 1373 direct: bool, 1374 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1375 ) { 1376 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1377 let guest = Guest::new(Box::new(focal)); 1378 let api_socket = temp_api_path(&guest.tmp_dir); 1379 1380 let kernel_path = direct_kernel_boot_path(); 1381 1382 let (blk_params, daemon_child) = { 1383 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1384 // Start the daemon 1385 let (daemon_child, vubd_socket_path) = 1386 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1387 1388 ( 1389 format!( 1390 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1391 ), 1392 Some(daemon_child), 1393 ) 1394 }; 1395 1396 let mut child = GuestCommand::new(&guest) 1397 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1398 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1399 .args(["--kernel", kernel_path.to_str().unwrap()]) 1400 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1401 .args([ 1402 "--disk", 1403 format!( 1404 "path={}", 1405 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1406 ) 1407 .as_str(), 1408 format!( 1409 "path={}", 1410 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1411 ) 1412 .as_str(), 1413 blk_params.as_str(), 1414 ]) 1415 .default_net() 1416 .args(["--api-socket", &api_socket]) 1417 .capture_output() 1418 .spawn() 1419 .unwrap(); 1420 1421 let r = std::panic::catch_unwind(|| { 1422 guest.wait_vm_boot(None).unwrap(); 1423 1424 // Check both if /dev/vdc exists and if the block size is 16M. 1425 assert_eq!( 1426 guest 1427 .ssh_command("lsblk | grep vdc | grep -c 16M") 1428 .unwrap() 1429 .trim() 1430 .parse::<u32>() 1431 .unwrap_or_default(), 1432 1 1433 ); 1434 1435 // Check if this block is RO or RW. 1436 assert_eq!( 1437 guest 1438 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1439 .unwrap() 1440 .trim() 1441 .parse::<u32>() 1442 .unwrap_or_default(), 1443 readonly as u32 1444 ); 1445 1446 // Check if the number of queues in /sys/block/vdc/mq matches the 1447 // expected num_queues. 1448 assert_eq!( 1449 guest 1450 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1451 .unwrap() 1452 .trim() 1453 .parse::<u32>() 1454 .unwrap_or_default(), 1455 num_queues as u32 1456 ); 1457 1458 // Mount the device 1459 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1460 guest.ssh_command("mkdir mount_image").unwrap(); 1461 guest 1462 .ssh_command( 1463 format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), 1464 ) 1465 .unwrap(); 1466 1467 // Check the content of the block device. The file "foo" should 1468 // contain "bar". 1469 assert_eq!( 1470 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1471 "bar" 1472 ); 1473 1474 // ACPI feature is needed. 1475 #[cfg(target_arch = "x86_64")] 1476 { 1477 guest.enable_memory_hotplug(); 1478 1479 // Add RAM to the VM 1480 let desired_ram = 1024 << 20; 1481 resize_command(&api_socket, None, Some(desired_ram), None, None); 1482 1483 thread::sleep(std::time::Duration::new(10, 0)); 1484 1485 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1486 1487 // Check again the content of the block device after the resize 1488 // has been performed. 1489 assert_eq!( 1490 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1491 "bar" 1492 ); 1493 } 1494 1495 // Unmount the device 1496 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1497 guest.ssh_command("rm -r mount_image").unwrap(); 1498 }); 1499 1500 let _ = child.kill(); 1501 let output = child.wait_with_output().unwrap(); 1502 1503 if let Some(mut daemon_child) = daemon_child { 1504 thread::sleep(std::time::Duration::new(5, 0)); 1505 let _ = daemon_child.kill(); 1506 let _ = daemon_child.wait(); 1507 } 1508 1509 handle_child_output(r, &output); 1510 } 1511 1512 fn test_boot_from_vhost_user_blk( 1513 num_queues: usize, 1514 readonly: bool, 1515 direct: bool, 1516 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1517 ) { 1518 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1519 let guest = Guest::new(Box::new(focal)); 1520 1521 let kernel_path = direct_kernel_boot_path(); 1522 1523 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1524 1525 let (blk_boot_params, daemon_child) = { 1526 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1527 // Start the daemon 1528 let (daemon_child, vubd_socket_path) = prepare_daemon( 1529 &guest.tmp_dir, 1530 disk_path.as_str(), 1531 num_queues, 1532 readonly, 1533 direct, 1534 ); 1535 1536 ( 1537 format!( 1538 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1539 ), 1540 Some(daemon_child), 1541 ) 1542 }; 1543 1544 let mut child = GuestCommand::new(&guest) 1545 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1546 .args(["--memory", "size=512M,shared=on"]) 1547 .args(["--kernel", kernel_path.to_str().unwrap()]) 1548 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1549 .args([ 1550 "--disk", 1551 blk_boot_params.as_str(), 1552 format!( 1553 "path={}", 1554 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1555 ) 1556 .as_str(), 1557 ]) 1558 .default_net() 1559 .capture_output() 1560 .spawn() 1561 .unwrap(); 1562 1563 let r = std::panic::catch_unwind(|| { 1564 guest.wait_vm_boot(None).unwrap(); 1565 1566 // Just check the VM booted correctly. 1567 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1568 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1569 }); 1570 let _ = child.kill(); 1571 let output = child.wait_with_output().unwrap(); 1572 1573 if let Some(mut daemon_child) = daemon_child { 1574 thread::sleep(std::time::Duration::new(5, 0)); 1575 let _ = daemon_child.kill(); 1576 let _ = daemon_child.wait(); 1577 } 1578 1579 handle_child_output(r, &output); 1580 } 1581 1582 fn _test_virtio_fs( 1583 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1584 hotplug: bool, 1585 pci_segment: Option<u16>, 1586 ) { 1587 #[cfg(target_arch = "aarch64")] 1588 let focal_image = if hotplug { 1589 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1590 } else { 1591 FOCAL_IMAGE_NAME.to_string() 1592 }; 1593 #[cfg(target_arch = "x86_64")] 1594 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1595 let focal = UbuntuDiskConfig::new(focal_image); 1596 let guest = Guest::new(Box::new(focal)); 1597 let api_socket = temp_api_path(&guest.tmp_dir); 1598 1599 let mut workload_path = dirs::home_dir().unwrap(); 1600 workload_path.push("workloads"); 1601 1602 let mut shared_dir = workload_path; 1603 shared_dir.push("shared_dir"); 1604 1605 #[cfg(target_arch = "x86_64")] 1606 let kernel_path = direct_kernel_boot_path(); 1607 #[cfg(target_arch = "aarch64")] 1608 let kernel_path = if hotplug { 1609 edk2_path() 1610 } else { 1611 direct_kernel_boot_path() 1612 }; 1613 1614 let (mut daemon_child, virtiofsd_socket_path) = 1615 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1616 1617 let mut guest_command = GuestCommand::new(&guest); 1618 guest_command 1619 .args(["--cpus", "boot=1"]) 1620 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1621 .args(["--kernel", kernel_path.to_str().unwrap()]) 1622 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1623 .default_disks() 1624 .default_net() 1625 .args(["--api-socket", &api_socket]); 1626 if pci_segment.is_some() { 1627 guest_command.args([ 1628 "--platform", 1629 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 1630 ]); 1631 } 1632 1633 let fs_params = format!( 1634 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1635 virtiofsd_socket_path, 1636 if let Some(pci_segment) = pci_segment { 1637 format!(",pci_segment={pci_segment}") 1638 } else { 1639 "".to_owned() 1640 } 1641 ); 1642 1643 if !hotplug { 1644 guest_command.args(["--fs", fs_params.as_str()]); 1645 } 1646 1647 let mut child = guest_command.capture_output().spawn().unwrap(); 1648 1649 let r = std::panic::catch_unwind(|| { 1650 guest.wait_vm_boot(None).unwrap(); 1651 1652 if hotplug { 1653 // Add fs to the VM 1654 let (cmd_success, cmd_output) = 1655 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1656 assert!(cmd_success); 1657 1658 if let Some(pci_segment) = pci_segment { 1659 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1660 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1661 ))); 1662 } else { 1663 assert!(String::from_utf8_lossy(&cmd_output) 1664 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1665 } 1666 1667 thread::sleep(std::time::Duration::new(10, 0)); 1668 } 1669 1670 // Mount shared directory through virtio_fs filesystem 1671 guest 1672 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1673 .unwrap(); 1674 1675 // Check file1 exists and its content is "foo" 1676 assert_eq!( 1677 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1678 "foo" 1679 ); 1680 // Check file2 does not exist 1681 guest 1682 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1683 .unwrap(); 1684 1685 // Check file3 exists and its content is "bar" 1686 assert_eq!( 1687 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1688 "bar" 1689 ); 1690 1691 // ACPI feature is needed. 1692 #[cfg(target_arch = "x86_64")] 1693 { 1694 guest.enable_memory_hotplug(); 1695 1696 // Add RAM to the VM 1697 let desired_ram = 1024 << 20; 1698 resize_command(&api_socket, None, Some(desired_ram), None, None); 1699 1700 thread::sleep(std::time::Duration::new(30, 0)); 1701 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1702 1703 // After the resize, check again that file1 exists and its 1704 // content is "foo". 1705 assert_eq!( 1706 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1707 "foo" 1708 ); 1709 } 1710 1711 if hotplug { 1712 // Remove from VM 1713 guest.ssh_command("sudo umount mount_dir").unwrap(); 1714 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1715 } 1716 }); 1717 1718 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1719 thread::sleep(std::time::Duration::new(10, 0)); 1720 let (daemon_child, virtiofsd_socket_path) = 1721 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1722 1723 let r = std::panic::catch_unwind(|| { 1724 thread::sleep(std::time::Duration::new(10, 0)); 1725 let fs_params = format!( 1726 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1727 virtiofsd_socket_path, 1728 if let Some(pci_segment) = pci_segment { 1729 format!(",pci_segment={pci_segment}") 1730 } else { 1731 "".to_owned() 1732 } 1733 ); 1734 1735 // Add back and check it works 1736 let (cmd_success, cmd_output) = 1737 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1738 assert!(cmd_success); 1739 if let Some(pci_segment) = pci_segment { 1740 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1741 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1742 ))); 1743 } else { 1744 assert!(String::from_utf8_lossy(&cmd_output) 1745 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1746 } 1747 1748 thread::sleep(std::time::Duration::new(10, 0)); 1749 // Mount shared directory through virtio_fs filesystem 1750 guest 1751 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1752 .unwrap(); 1753 1754 // Check file1 exists and its content is "foo" 1755 assert_eq!( 1756 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1757 "foo" 1758 ); 1759 }); 1760 1761 (r, Some(daemon_child)) 1762 } else { 1763 (r, None) 1764 }; 1765 1766 let _ = child.kill(); 1767 let output = child.wait_with_output().unwrap(); 1768 1769 let _ = daemon_child.kill(); 1770 let _ = daemon_child.wait(); 1771 1772 if let Some(mut daemon_child) = hotplug_daemon_child { 1773 let _ = daemon_child.kill(); 1774 let _ = daemon_child.wait(); 1775 } 1776 1777 handle_child_output(r, &output); 1778 } 1779 1780 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1781 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1782 let guest = Guest::new(Box::new(focal)); 1783 1784 let kernel_path = direct_kernel_boot_path(); 1785 1786 let pmem_temp_file = TempFile::new().unwrap(); 1787 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1788 1789 std::process::Command::new("mkfs.ext4") 1790 .arg(pmem_temp_file.as_path()) 1791 .output() 1792 .expect("Expect creating disk image to succeed"); 1793 1794 let mut child = GuestCommand::new(&guest) 1795 .args(["--cpus", "boot=1"]) 1796 .args(["--memory", "size=512M"]) 1797 .args(["--kernel", kernel_path.to_str().unwrap()]) 1798 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1799 .default_disks() 1800 .default_net() 1801 .args([ 1802 "--pmem", 1803 format!( 1804 "file={}{}{}", 1805 pmem_temp_file.as_path().to_str().unwrap(), 1806 if specify_size { ",size=128M" } else { "" }, 1807 if discard_writes { 1808 ",discard_writes=on" 1809 } else { 1810 "" 1811 } 1812 ) 1813 .as_str(), 1814 ]) 1815 .capture_output() 1816 .spawn() 1817 .unwrap(); 1818 1819 let r = std::panic::catch_unwind(|| { 1820 guest.wait_vm_boot(None).unwrap(); 1821 1822 // Check for the presence of /dev/pmem0 1823 assert_eq!( 1824 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1825 "/dev/pmem0" 1826 ); 1827 1828 // Check changes persist after reboot 1829 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1830 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1831 guest 1832 .ssh_command("echo test123 | sudo tee /mnt/test") 1833 .unwrap(); 1834 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1835 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1836 1837 guest.reboot_linux(0, None); 1838 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1839 assert_eq!( 1840 guest 1841 .ssh_command("sudo cat /mnt/test || true") 1842 .unwrap() 1843 .trim(), 1844 if discard_writes { "" } else { "test123" } 1845 ); 1846 }); 1847 1848 let _ = child.kill(); 1849 let output = child.wait_with_output().unwrap(); 1850 1851 handle_child_output(r, &output); 1852 } 1853 1854 fn get_fd_count(pid: u32) -> usize { 1855 fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() 1856 } 1857 1858 fn _test_virtio_vsock(hotplug: bool) { 1859 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1860 let guest = Guest::new(Box::new(focal)); 1861 1862 #[cfg(target_arch = "x86_64")] 1863 let kernel_path = direct_kernel_boot_path(); 1864 #[cfg(target_arch = "aarch64")] 1865 let kernel_path = if hotplug { 1866 edk2_path() 1867 } else { 1868 direct_kernel_boot_path() 1869 }; 1870 1871 let socket = temp_vsock_path(&guest.tmp_dir); 1872 let api_socket = temp_api_path(&guest.tmp_dir); 1873 1874 let mut cmd = GuestCommand::new(&guest); 1875 cmd.args(["--api-socket", &api_socket]); 1876 cmd.args(["--cpus", "boot=1"]); 1877 cmd.args(["--memory", "size=512M"]); 1878 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1879 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1880 cmd.default_disks(); 1881 cmd.default_net(); 1882 1883 if !hotplug { 1884 cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); 1885 } 1886 1887 let mut child = cmd.capture_output().spawn().unwrap(); 1888 1889 let r = std::panic::catch_unwind(|| { 1890 guest.wait_vm_boot(None).unwrap(); 1891 1892 if hotplug { 1893 let (cmd_success, cmd_output) = remote_command_w_output( 1894 &api_socket, 1895 "add-vsock", 1896 Some(format!("cid=3,socket={socket},id=test0").as_str()), 1897 ); 1898 assert!(cmd_success); 1899 assert!(String::from_utf8_lossy(&cmd_output) 1900 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1901 thread::sleep(std::time::Duration::new(10, 0)); 1902 // Check adding a second one fails 1903 assert!(!remote_command( 1904 &api_socket, 1905 "add-vsock", 1906 Some("cid=1234,socket=/tmp/fail") 1907 )); 1908 } 1909 1910 // Validate vsock works as expected. 1911 guest.check_vsock(socket.as_str()); 1912 guest.reboot_linux(0, None); 1913 // Validate vsock still works after a reboot. 1914 guest.check_vsock(socket.as_str()); 1915 1916 if hotplug { 1917 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1918 } 1919 }); 1920 1921 let _ = child.kill(); 1922 let output = child.wait_with_output().unwrap(); 1923 1924 handle_child_output(r, &output); 1925 } 1926 1927 fn get_ksm_pages_shared() -> u32 { 1928 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1929 .unwrap() 1930 .trim() 1931 .parse::<u32>() 1932 .unwrap() 1933 } 1934 1935 fn test_memory_mergeable(mergeable: bool) { 1936 let memory_param = if mergeable { 1937 "mergeable=on" 1938 } else { 1939 "mergeable=off" 1940 }; 1941 1942 // We are assuming the rest of the system in our CI is not using mergeable memory 1943 let ksm_ps_init = get_ksm_pages_shared(); 1944 assert!(ksm_ps_init == 0); 1945 1946 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1947 let guest1 = Guest::new(Box::new(focal1)); 1948 let mut child1 = GuestCommand::new(&guest1) 1949 .args(["--cpus", "boot=1"]) 1950 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1951 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1952 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1953 .default_disks() 1954 .args(["--net", guest1.default_net_string().as_str()]) 1955 .args(["--serial", "tty", "--console", "off"]) 1956 .capture_output() 1957 .spawn() 1958 .unwrap(); 1959 1960 let r = std::panic::catch_unwind(|| { 1961 guest1.wait_vm_boot(None).unwrap(); 1962 }); 1963 if r.is_err() { 1964 let _ = child1.kill(); 1965 let output = child1.wait_with_output().unwrap(); 1966 handle_child_output(r, &output); 1967 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1968 } 1969 1970 let ksm_ps_guest1 = get_ksm_pages_shared(); 1971 1972 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1973 let guest2 = Guest::new(Box::new(focal2)); 1974 let mut child2 = GuestCommand::new(&guest2) 1975 .args(["--cpus", "boot=1"]) 1976 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1977 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1978 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1979 .default_disks() 1980 .args(["--net", guest2.default_net_string().as_str()]) 1981 .args(["--serial", "tty", "--console", "off"]) 1982 .capture_output() 1983 .spawn() 1984 .unwrap(); 1985 1986 let r = std::panic::catch_unwind(|| { 1987 guest2.wait_vm_boot(None).unwrap(); 1988 let ksm_ps_guest2 = get_ksm_pages_shared(); 1989 1990 if mergeable { 1991 println!( 1992 "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" 1993 ); 1994 // We are expecting the number of shared pages to increase as the number of VM increases 1995 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1996 } else { 1997 assert!(ksm_ps_guest1 == 0); 1998 assert!(ksm_ps_guest2 == 0); 1999 } 2000 }); 2001 2002 let _ = child1.kill(); 2003 let _ = child2.kill(); 2004 2005 let output = child1.wait_with_output().unwrap(); 2006 child2.wait().unwrap(); 2007 2008 handle_child_output(r, &output); 2009 } 2010 2011 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 2012 let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); 2013 let reader = io::BufReader::new(smaps); 2014 2015 let mut skip_map: bool = false; 2016 let mut region_name: String = "".to_string(); 2017 let mut region_maps = HashMap::new(); 2018 for line in reader.lines() { 2019 let l = line.unwrap(); 2020 2021 if l.contains('-') { 2022 let values: Vec<&str> = l.split_whitespace().collect(); 2023 region_name = values.last().unwrap().trim().to_string(); 2024 if region_name == "0" { 2025 region_name = "anonymous".to_string() 2026 } 2027 } 2028 2029 // Each section begins with something that looks like: 2030 // Size: 2184 kB 2031 if l.starts_with("Size:") { 2032 let values: Vec<&str> = l.split_whitespace().collect(); 2033 let map_size = values[1].parse::<u32>().unwrap(); 2034 // We skip the assigned guest RAM map, its RSS is only 2035 // dependent on the guest actual memory usage. 2036 // Everything else can be added to the VMM overhead. 2037 skip_map = map_size >= guest_memory_size; 2038 continue; 2039 } 2040 2041 // If this is a map we're taking into account, then we only 2042 // count the RSS. The sum of all counted RSS is the VMM overhead. 2043 if !skip_map && l.starts_with("Rss:") { 2044 let values: Vec<&str> = l.split_whitespace().collect(); 2045 let value = values[1].trim().parse::<u32>().unwrap(); 2046 *region_maps.entry(region_name.clone()).or_insert(0) += value; 2047 } 2048 } 2049 2050 region_maps 2051 } 2052 2053 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 2054 let mut total = 0; 2055 2056 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 2057 eprintln!("{region_name}: {value}"); 2058 total += value; 2059 } 2060 2061 total 2062 } 2063 2064 fn process_rss_kib(pid: u32) -> usize { 2065 let command = format!("ps -q {pid} -o rss="); 2066 let rss = exec_host_command_output(&command); 2067 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 2068 } 2069 2070 // 10MB is our maximum accepted overhead. 2071 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 2072 2073 #[derive(PartialEq, Eq, PartialOrd)] 2074 struct Counters { 2075 rx_bytes: u64, 2076 rx_frames: u64, 2077 tx_bytes: u64, 2078 tx_frames: u64, 2079 read_bytes: u64, 2080 write_bytes: u64, 2081 read_ops: u64, 2082 write_ops: u64, 2083 } 2084 2085 fn get_counters(api_socket: &str) -> Counters { 2086 // Get counters 2087 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 2088 assert!(cmd_success); 2089 2090 let counters: HashMap<&str, HashMap<&str, u64>> = 2091 serde_json::from_slice(&cmd_output).unwrap_or_default(); 2092 2093 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 2094 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 2095 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 2096 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 2097 2098 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 2099 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 2100 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 2101 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 2102 2103 Counters { 2104 rx_bytes, 2105 rx_frames, 2106 tx_bytes, 2107 tx_frames, 2108 read_bytes, 2109 write_bytes, 2110 read_ops, 2111 write_ops, 2112 } 2113 } 2114 2115 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 2116 let (tx, rx) = mpsc::channel::<String>(); 2117 thread::spawn(move || loop { 2118 thread::sleep(std::time::Duration::new(1, 0)); 2119 let mut buf = [0; 512]; 2120 match pty.read(&mut buf) { 2121 Ok(_bytes) => { 2122 let output = std::str::from_utf8(&buf).unwrap().to_string(); 2123 match tx.send(output) { 2124 Ok(_) => (), 2125 Err(_) => break, 2126 } 2127 } 2128 Err(_) => break, 2129 } 2130 }); 2131 rx 2132 } 2133 2134 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 2135 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2136 assert!(cmd_success); 2137 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2138 assert_eq!("Pty", info["config"][pty_type]["mode"]); 2139 PathBuf::from( 2140 info["config"][pty_type]["file"] 2141 .as_str() 2142 .expect("Missing pty path"), 2143 ) 2144 } 2145 2146 // VFIO test network setup. 2147 // We reserve a different IP class for it: 172.18.0.0/24. 2148 #[cfg(target_arch = "x86_64")] 2149 fn setup_vfio_network_interfaces() { 2150 // 'vfio-br0' 2151 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 2152 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 2153 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 2154 // 'vfio-tap0' 2155 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 2156 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 2157 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 2158 // 'vfio-tap1' 2159 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 2160 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 2161 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 2162 // 'vfio-tap2' 2163 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 2164 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 2165 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 2166 // 'vfio-tap3' 2167 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 2168 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 2169 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 2170 } 2171 2172 // Tear VFIO test network down 2173 #[cfg(target_arch = "x86_64")] 2174 fn cleanup_vfio_network_interfaces() { 2175 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 2176 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 2177 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 2178 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 2179 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 2180 } 2181 2182 fn balloon_size(api_socket: &str) -> u64 { 2183 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2184 assert!(cmd_success); 2185 2186 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2187 let total_mem = &info["config"]["memory"]["size"] 2188 .to_string() 2189 .parse::<u64>() 2190 .unwrap(); 2191 let actual_mem = &info["memory_actual_size"] 2192 .to_string() 2193 .parse::<u64>() 2194 .unwrap(); 2195 total_mem - actual_mem 2196 } 2197 2198 fn vm_state(api_socket: &str) -> String { 2199 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2200 assert!(cmd_success); 2201 2202 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2203 let state = &info["state"].as_str().unwrap(); 2204 2205 state.to_string() 2206 } 2207 2208 // This test validates that it can find the virtio-iommu device at first. 2209 // It also verifies that both disks and the network card are attached to 2210 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 2211 // The last interesting part of this test is that it exercises the network 2212 // interface attached to the virtual IOMMU since this is the one used to 2213 // send all commands through SSH. 2214 fn _test_virtio_iommu(acpi: bool) { 2215 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 2216 // Focal image is still old. 2217 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 2218 // the kernel binary has been updated. 2219 #[cfg(target_arch = "aarch64")] 2220 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 2221 #[cfg(target_arch = "x86_64")] 2222 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2223 let focal = UbuntuDiskConfig::new(focal_image); 2224 let guest = Guest::new(Box::new(focal)); 2225 2226 #[cfg(target_arch = "x86_64")] 2227 let kernel_path = direct_kernel_boot_path(); 2228 #[cfg(target_arch = "aarch64")] 2229 let kernel_path = if acpi { 2230 edk2_path() 2231 } else { 2232 direct_kernel_boot_path() 2233 }; 2234 2235 let mut child = GuestCommand::new(&guest) 2236 .args(["--cpus", "boot=1"]) 2237 .args(["--memory", "size=512M"]) 2238 .args(["--kernel", kernel_path.to_str().unwrap()]) 2239 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2240 .args([ 2241 "--disk", 2242 format!( 2243 "path={},iommu=on", 2244 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2245 ) 2246 .as_str(), 2247 format!( 2248 "path={},iommu=on", 2249 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2250 ) 2251 .as_str(), 2252 ]) 2253 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 2254 .capture_output() 2255 .spawn() 2256 .unwrap(); 2257 2258 let r = std::panic::catch_unwind(|| { 2259 guest.wait_vm_boot(None).unwrap(); 2260 2261 // Verify the virtio-iommu device is present. 2262 assert!(guest 2263 .does_device_vendor_pair_match("0x1057", "0x1af4") 2264 .unwrap_or_default()); 2265 2266 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 2267 // different with ACPI. 2268 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 2269 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 2270 // and appear under folder '/sys/kernel/iommu_groups/'. 2271 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 2272 // which is the console. The first disk "0000:00:02.0" is in group '1'. 2273 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 2274 // contains "0000:00:02.0" which is the first disk. 2275 // 2276 // Verify the iommu group of the first disk. 2277 let iommu_group = !acpi as i32; 2278 assert_eq!( 2279 guest 2280 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2281 .unwrap() 2282 .trim(), 2283 "0000:00:02.0" 2284 ); 2285 2286 // Verify the iommu group of the second disk. 2287 let iommu_group = if acpi { 1 } else { 2 }; 2288 assert_eq!( 2289 guest 2290 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2291 .unwrap() 2292 .trim(), 2293 "0000:00:03.0" 2294 ); 2295 2296 // Verify the iommu group of the network card. 2297 let iommu_group = if acpi { 2 } else { 3 }; 2298 assert_eq!( 2299 guest 2300 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2301 .unwrap() 2302 .trim(), 2303 "0000:00:04.0" 2304 ); 2305 }); 2306 2307 let _ = child.kill(); 2308 let output = child.wait_with_output().unwrap(); 2309 2310 handle_child_output(r, &output); 2311 } 2312 2313 fn get_reboot_count(guest: &Guest) -> u32 { 2314 guest 2315 .ssh_command("sudo last | grep -c reboot") 2316 .unwrap() 2317 .trim() 2318 .parse::<u32>() 2319 .unwrap_or_default() 2320 } 2321 2322 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 2323 // Check for PCI device 2324 assert!(guest 2325 .does_device_vendor_pair_match("0x1063", "0x1af4") 2326 .unwrap_or_default()); 2327 2328 // Enable systemd watchdog 2329 guest 2330 .ssh_command(&format!( 2331 "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" 2332 )) 2333 .unwrap(); 2334 } 2335 2336 fn make_guest_panic(guest: &Guest) { 2337 // Check for pvpanic device 2338 assert!(guest 2339 .does_device_vendor_pair_match("0x0011", "0x1b36") 2340 .unwrap_or_default()); 2341 2342 // Trigger guest a panic 2343 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 2344 } 2345 2346 mod common_parallel { 2347 use std::{fs::OpenOptions, io::SeekFrom}; 2348 2349 use crate::*; 2350 2351 #[test] 2352 #[cfg(target_arch = "x86_64")] 2353 fn test_focal_hypervisor_fw() { 2354 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 2355 } 2356 2357 #[test] 2358 #[cfg(target_arch = "x86_64")] 2359 fn test_focal_ovmf() { 2360 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 2361 } 2362 2363 #[cfg(target_arch = "x86_64")] 2364 fn test_simple_launch(fw_path: String, disk_path: &str) { 2365 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 2366 let guest = Guest::new(disk_config); 2367 let event_path = temp_event_monitor_path(&guest.tmp_dir); 2368 2369 let mut child = GuestCommand::new(&guest) 2370 .args(["--cpus", "boot=1"]) 2371 .args(["--memory", "size=512M"]) 2372 .args(["--kernel", fw_path.as_str()]) 2373 .default_disks() 2374 .default_net() 2375 .args(["--serial", "tty", "--console", "off"]) 2376 .args(["--event-monitor", format!("path={event_path}").as_str()]) 2377 .capture_output() 2378 .spawn() 2379 .unwrap(); 2380 2381 let r = std::panic::catch_unwind(|| { 2382 guest.wait_vm_boot(Some(120)).unwrap(); 2383 2384 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2385 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2386 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2387 2388 let expected_sequential_events = [ 2389 &MetaEvent { 2390 event: "starting".to_string(), 2391 device_id: None, 2392 }, 2393 &MetaEvent { 2394 event: "booting".to_string(), 2395 device_id: None, 2396 }, 2397 &MetaEvent { 2398 event: "booted".to_string(), 2399 device_id: None, 2400 }, 2401 &MetaEvent { 2402 event: "activated".to_string(), 2403 device_id: Some("_disk0".to_string()), 2404 }, 2405 &MetaEvent { 2406 event: "reset".to_string(), 2407 device_id: Some("_disk0".to_string()), 2408 }, 2409 ]; 2410 assert!(check_sequential_events( 2411 &expected_sequential_events, 2412 &event_path 2413 )); 2414 2415 // It's been observed on the Bionic image that udev and snapd 2416 // services can cause some delay in the VM's shutdown. Disabling 2417 // them improves the reliability of this test. 2418 let _ = guest.ssh_command("sudo systemctl disable udev"); 2419 let _ = guest.ssh_command("sudo systemctl stop udev"); 2420 let _ = guest.ssh_command("sudo systemctl disable snapd"); 2421 let _ = guest.ssh_command("sudo systemctl stop snapd"); 2422 2423 guest.ssh_command("sudo poweroff").unwrap(); 2424 thread::sleep(std::time::Duration::new(20, 0)); 2425 let latest_events = [ 2426 &MetaEvent { 2427 event: "shutdown".to_string(), 2428 device_id: None, 2429 }, 2430 &MetaEvent { 2431 event: "deleted".to_string(), 2432 device_id: None, 2433 }, 2434 &MetaEvent { 2435 event: "shutdown".to_string(), 2436 device_id: None, 2437 }, 2438 ]; 2439 assert!(check_latest_events_exact(&latest_events, &event_path)); 2440 }); 2441 2442 let _ = child.kill(); 2443 let output = child.wait_with_output().unwrap(); 2444 2445 handle_child_output(r, &output); 2446 } 2447 2448 #[test] 2449 fn test_multi_cpu() { 2450 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2451 let jammy = UbuntuDiskConfig::new(jammy_image); 2452 let guest = Guest::new(Box::new(jammy)); 2453 2454 let mut cmd = GuestCommand::new(&guest); 2455 cmd.args(["--cpus", "boot=2,max=4"]) 2456 .args(["--memory", "size=512M"]) 2457 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2458 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2459 .capture_output() 2460 .default_disks() 2461 .default_net(); 2462 2463 let mut child = cmd.spawn().unwrap(); 2464 2465 let r = std::panic::catch_unwind(|| { 2466 guest.wait_vm_boot(Some(120)).unwrap(); 2467 2468 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2469 2470 assert_eq!( 2471 guest 2472 .ssh_command( 2473 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2474 ) 2475 .unwrap() 2476 .trim(), 2477 "smp: Brought up 1 node, 2 CPUs" 2478 ); 2479 }); 2480 2481 let _ = child.kill(); 2482 let output = child.wait_with_output().unwrap(); 2483 2484 handle_child_output(r, &output); 2485 } 2486 2487 #[test] 2488 fn test_cpu_topology_421() { 2489 test_cpu_topology(4, 2, 1, false); 2490 } 2491 2492 #[test] 2493 fn test_cpu_topology_142() { 2494 test_cpu_topology(1, 4, 2, false); 2495 } 2496 2497 #[test] 2498 fn test_cpu_topology_262() { 2499 test_cpu_topology(2, 6, 2, false); 2500 } 2501 2502 #[test] 2503 #[cfg(target_arch = "x86_64")] 2504 #[cfg(not(feature = "mshv"))] 2505 fn test_cpu_physical_bits() { 2506 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2507 let guest = Guest::new(Box::new(focal)); 2508 let max_phys_bits: u8 = 36; 2509 let mut child = GuestCommand::new(&guest) 2510 .args(["--cpus", &format!("max_phys_bits={max_phys_bits}")]) 2511 .args(["--memory", "size=512M"]) 2512 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2513 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2514 .default_disks() 2515 .default_net() 2516 .capture_output() 2517 .spawn() 2518 .unwrap(); 2519 2520 let r = std::panic::catch_unwind(|| { 2521 guest.wait_vm_boot(None).unwrap(); 2522 2523 assert!( 2524 guest 2525 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2526 .unwrap() 2527 .trim() 2528 .parse::<u8>() 2529 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2530 ); 2531 }); 2532 2533 let _ = child.kill(); 2534 let output = child.wait_with_output().unwrap(); 2535 2536 handle_child_output(r, &output); 2537 } 2538 2539 #[test] 2540 fn test_cpu_affinity() { 2541 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2542 let guest = Guest::new(Box::new(focal)); 2543 2544 // We need the host to have at least 4 CPUs if we want to be able 2545 // to run this test. 2546 let host_cpus_count = exec_host_command_output("nproc"); 2547 assert!( 2548 String::from_utf8_lossy(&host_cpus_count.stdout) 2549 .trim() 2550 .parse::<u16>() 2551 .unwrap_or(0) 2552 >= 4 2553 ); 2554 2555 let mut child = GuestCommand::new(&guest) 2556 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2557 .args(["--memory", "size=512M"]) 2558 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2559 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2560 .default_disks() 2561 .default_net() 2562 .capture_output() 2563 .spawn() 2564 .unwrap(); 2565 2566 let r = std::panic::catch_unwind(|| { 2567 guest.wait_vm_boot(None).unwrap(); 2568 let pid = child.id(); 2569 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2570 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2571 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2572 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2573 }); 2574 2575 let _ = child.kill(); 2576 let output = child.wait_with_output().unwrap(); 2577 handle_child_output(r, &output); 2578 } 2579 2580 #[test] 2581 fn test_virtio_queue_affinity() { 2582 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2583 let guest = Guest::new(Box::new(focal)); 2584 2585 // We need the host to have at least 4 CPUs if we want to be able 2586 // to run this test. 2587 let host_cpus_count = exec_host_command_output("nproc"); 2588 assert!( 2589 String::from_utf8_lossy(&host_cpus_count.stdout) 2590 .trim() 2591 .parse::<u16>() 2592 .unwrap_or(0) 2593 >= 4 2594 ); 2595 2596 let mut child = GuestCommand::new(&guest) 2597 .args(["--cpus", "boot=4"]) 2598 .args(["--memory", "size=512M"]) 2599 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2600 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2601 .args([ 2602 "--disk", 2603 format!( 2604 "path={}", 2605 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2606 ) 2607 .as_str(), 2608 format!( 2609 "path={},num_queues=4,queue_affinity=[0@[0,2],1@[1,3],2@[1],3@[3]]", 2610 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2611 ) 2612 .as_str(), 2613 ]) 2614 .default_net() 2615 .capture_output() 2616 .spawn() 2617 .unwrap(); 2618 2619 let r = std::panic::catch_unwind(|| { 2620 guest.wait_vm_boot(None).unwrap(); 2621 let pid = child.id(); 2622 let taskset_q0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2623 assert_eq!(String::from_utf8_lossy(&taskset_q0.stdout).trim(), "0,2"); 2624 let taskset_q1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2625 assert_eq!(String::from_utf8_lossy(&taskset_q1.stdout).trim(), "1,3"); 2626 let taskset_q2 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q2 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2627 assert_eq!(String::from_utf8_lossy(&taskset_q2.stdout).trim(), "1"); 2628 let taskset_q3 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q3 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2629 assert_eq!(String::from_utf8_lossy(&taskset_q3.stdout).trim(), "3"); 2630 }); 2631 2632 let _ = child.kill(); 2633 let output = child.wait_with_output().unwrap(); 2634 handle_child_output(r, &output); 2635 } 2636 2637 #[test] 2638 #[cfg(not(feature = "mshv"))] 2639 fn test_large_vm() { 2640 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2641 let guest = Guest::new(Box::new(focal)); 2642 let mut cmd = GuestCommand::new(&guest); 2643 cmd.args(["--cpus", "boot=48"]) 2644 .args(["--memory", "size=5120M"]) 2645 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2646 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2647 .args(["--serial", "tty"]) 2648 .args(["--console", "off"]) 2649 .capture_output() 2650 .default_disks() 2651 .default_net(); 2652 2653 let mut child = cmd.spawn().unwrap(); 2654 2655 guest.wait_vm_boot(None).unwrap(); 2656 2657 let r = std::panic::catch_unwind(|| { 2658 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2659 assert_eq!( 2660 guest 2661 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2662 .unwrap() 2663 .trim(), 2664 "0-47" 2665 ); 2666 2667 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2668 }); 2669 2670 let _ = child.kill(); 2671 let output = child.wait_with_output().unwrap(); 2672 2673 handle_child_output(r, &output); 2674 } 2675 2676 #[test] 2677 #[cfg(not(feature = "mshv"))] 2678 fn test_huge_memory() { 2679 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2680 let guest = Guest::new(Box::new(focal)); 2681 let mut cmd = GuestCommand::new(&guest); 2682 cmd.args(["--cpus", "boot=1"]) 2683 .args(["--memory", "size=128G"]) 2684 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2685 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2686 .capture_output() 2687 .default_disks() 2688 .default_net(); 2689 2690 let mut child = cmd.spawn().unwrap(); 2691 2692 guest.wait_vm_boot(Some(120)).unwrap(); 2693 2694 let r = std::panic::catch_unwind(|| { 2695 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2696 }); 2697 2698 let _ = child.kill(); 2699 let output = child.wait_with_output().unwrap(); 2700 2701 handle_child_output(r, &output); 2702 } 2703 2704 #[test] 2705 fn test_power_button() { 2706 _test_power_button(false); 2707 } 2708 2709 #[test] 2710 #[cfg(not(feature = "mshv"))] 2711 fn test_user_defined_memory_regions() { 2712 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2713 let guest = Guest::new(Box::new(focal)); 2714 let api_socket = temp_api_path(&guest.tmp_dir); 2715 2716 let kernel_path = direct_kernel_boot_path(); 2717 2718 let mut child = GuestCommand::new(&guest) 2719 .args(["--cpus", "boot=1"]) 2720 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2721 .args([ 2722 "--memory-zone", 2723 "id=mem0,size=1G,hotplug_size=2G", 2724 "id=mem1,size=1G,shared=on", 2725 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2726 ]) 2727 .args(["--kernel", kernel_path.to_str().unwrap()]) 2728 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2729 .args(["--api-socket", &api_socket]) 2730 .capture_output() 2731 .default_disks() 2732 .default_net() 2733 .spawn() 2734 .unwrap(); 2735 2736 let r = std::panic::catch_unwind(|| { 2737 guest.wait_vm_boot(None).unwrap(); 2738 2739 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2740 2741 guest.enable_memory_hotplug(); 2742 2743 resize_zone_command(&api_socket, "mem0", "3G"); 2744 thread::sleep(std::time::Duration::new(5, 0)); 2745 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2746 resize_zone_command(&api_socket, "mem2", "3G"); 2747 thread::sleep(std::time::Duration::new(5, 0)); 2748 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2749 resize_zone_command(&api_socket, "mem0", "2G"); 2750 thread::sleep(std::time::Duration::new(5, 0)); 2751 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2752 resize_zone_command(&api_socket, "mem2", "2G"); 2753 thread::sleep(std::time::Duration::new(5, 0)); 2754 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2755 2756 guest.reboot_linux(0, None); 2757 2758 // Check the amount of RAM after reboot 2759 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2760 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2761 2762 // Check if we can still resize down to the initial 'boot'size 2763 resize_zone_command(&api_socket, "mem0", "1G"); 2764 thread::sleep(std::time::Duration::new(5, 0)); 2765 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2766 resize_zone_command(&api_socket, "mem2", "1G"); 2767 thread::sleep(std::time::Duration::new(5, 0)); 2768 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2769 }); 2770 2771 let _ = child.kill(); 2772 let output = child.wait_with_output().unwrap(); 2773 2774 handle_child_output(r, &output); 2775 } 2776 2777 #[test] 2778 #[cfg(not(feature = "mshv"))] 2779 fn test_guest_numa_nodes() { 2780 _test_guest_numa_nodes(false); 2781 } 2782 2783 #[test] 2784 #[cfg(target_arch = "x86_64")] 2785 fn test_iommu_segments() { 2786 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2787 let focal = UbuntuDiskConfig::new(focal_image); 2788 let guest = Guest::new(Box::new(focal)); 2789 2790 // Prepare another disk file for the virtio-disk device 2791 let test_disk_path = String::from( 2792 guest 2793 .tmp_dir 2794 .as_path() 2795 .join("test-disk.raw") 2796 .to_str() 2797 .unwrap(), 2798 ); 2799 assert!( 2800 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2801 ); 2802 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2803 2804 let api_socket = temp_api_path(&guest.tmp_dir); 2805 let mut cmd = GuestCommand::new(&guest); 2806 2807 cmd.args(["--cpus", "boot=1"]) 2808 .args(["--api-socket", &api_socket]) 2809 .args(["--memory", "size=512M"]) 2810 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2811 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2812 .args([ 2813 "--platform", 2814 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS},iommu_segments=[1]"), 2815 ]) 2816 .default_disks() 2817 .capture_output() 2818 .default_net(); 2819 2820 let mut child = cmd.spawn().unwrap(); 2821 2822 guest.wait_vm_boot(None).unwrap(); 2823 2824 let r = std::panic::catch_unwind(|| { 2825 let (cmd_success, cmd_output) = remote_command_w_output( 2826 &api_socket, 2827 "add-disk", 2828 Some( 2829 format!( 2830 "path={},id=test0,pci_segment=1,iommu=on", 2831 test_disk_path.as_str() 2832 ) 2833 .as_str(), 2834 ), 2835 ); 2836 assert!(cmd_success); 2837 assert!(String::from_utf8_lossy(&cmd_output) 2838 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2839 2840 // Check IOMMU setup 2841 assert!(guest 2842 .does_device_vendor_pair_match("0x1057", "0x1af4") 2843 .unwrap_or_default()); 2844 assert_eq!( 2845 guest 2846 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2847 .unwrap() 2848 .trim(), 2849 "0001:00:01.0" 2850 ); 2851 }); 2852 2853 let _ = child.kill(); 2854 let output = child.wait_with_output().unwrap(); 2855 2856 handle_child_output(r, &output); 2857 } 2858 2859 #[test] 2860 fn test_pci_msi() { 2861 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2862 let guest = Guest::new(Box::new(focal)); 2863 let mut cmd = GuestCommand::new(&guest); 2864 cmd.args(["--cpus", "boot=1"]) 2865 .args(["--memory", "size=512M"]) 2866 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2867 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2868 .capture_output() 2869 .default_disks() 2870 .default_net(); 2871 2872 let mut child = cmd.spawn().unwrap(); 2873 2874 guest.wait_vm_boot(None).unwrap(); 2875 2876 #[cfg(target_arch = "x86_64")] 2877 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2878 #[cfg(target_arch = "aarch64")] 2879 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2880 2881 let r = std::panic::catch_unwind(|| { 2882 assert_eq!( 2883 guest 2884 .ssh_command(grep_cmd) 2885 .unwrap() 2886 .trim() 2887 .parse::<u32>() 2888 .unwrap_or_default(), 2889 12 2890 ); 2891 }); 2892 2893 let _ = child.kill(); 2894 let output = child.wait_with_output().unwrap(); 2895 2896 handle_child_output(r, &output); 2897 } 2898 2899 #[test] 2900 fn test_virtio_net_ctrl_queue() { 2901 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2902 let guest = Guest::new(Box::new(focal)); 2903 let mut cmd = GuestCommand::new(&guest); 2904 cmd.args(["--cpus", "boot=1"]) 2905 .args(["--memory", "size=512M"]) 2906 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2907 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2908 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2909 .capture_output() 2910 .default_disks(); 2911 2912 let mut child = cmd.spawn().unwrap(); 2913 2914 guest.wait_vm_boot(None).unwrap(); 2915 2916 #[cfg(target_arch = "aarch64")] 2917 let iface = "enp0s4"; 2918 #[cfg(target_arch = "x86_64")] 2919 let iface = "ens4"; 2920 2921 let r = std::panic::catch_unwind(|| { 2922 assert_eq!( 2923 guest 2924 .ssh_command( 2925 format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() 2926 ) 2927 .unwrap() 2928 .trim(), 2929 "success" 2930 ); 2931 assert_eq!( 2932 guest 2933 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 2934 .unwrap() 2935 .trim(), 2936 "3000" 2937 ); 2938 }); 2939 2940 let _ = child.kill(); 2941 let output = child.wait_with_output().unwrap(); 2942 2943 handle_child_output(r, &output); 2944 } 2945 2946 #[test] 2947 fn test_pci_multiple_segments() { 2948 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2949 let guest = Guest::new(Box::new(focal)); 2950 2951 // Prepare another disk file for the virtio-disk device 2952 let test_disk_path = String::from( 2953 guest 2954 .tmp_dir 2955 .as_path() 2956 .join("test-disk.raw") 2957 .to_str() 2958 .unwrap(), 2959 ); 2960 assert!( 2961 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2962 ); 2963 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2964 2965 let mut cmd = GuestCommand::new(&guest); 2966 cmd.args(["--cpus", "boot=1"]) 2967 .args(["--memory", "size=512M"]) 2968 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2969 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2970 .args([ 2971 "--platform", 2972 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 2973 ]) 2974 .args([ 2975 "--disk", 2976 format!( 2977 "path={}", 2978 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2979 ) 2980 .as_str(), 2981 format!( 2982 "path={}", 2983 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2984 ) 2985 .as_str(), 2986 format!("path={test_disk_path},pci_segment=15").as_str(), 2987 ]) 2988 .capture_output() 2989 .default_net(); 2990 2991 let mut child = cmd.spawn().unwrap(); 2992 2993 guest.wait_vm_boot(None).unwrap(); 2994 2995 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2996 2997 let r = std::panic::catch_unwind(|| { 2998 // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. 2999 assert_eq!( 3000 guest 3001 .ssh_command(grep_cmd) 3002 .unwrap() 3003 .trim() 3004 .parse::<u16>() 3005 .unwrap_or_default(), 3006 MAX_NUM_PCI_SEGMENTS 3007 ); 3008 3009 // Check both if /dev/vdc exists and if the block size is 4M. 3010 assert_eq!( 3011 guest 3012 .ssh_command("lsblk | grep vdc | grep -c 4M") 3013 .unwrap() 3014 .trim() 3015 .parse::<u32>() 3016 .unwrap_or_default(), 3017 1 3018 ); 3019 3020 // Mount the device. 3021 guest.ssh_command("mkdir mount_image").unwrap(); 3022 guest 3023 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 3024 .unwrap(); 3025 // Grant all users with write permission. 3026 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 3027 3028 // Write something to the device. 3029 guest 3030 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 3031 .unwrap(); 3032 3033 // Check the content of the block device. The file "foo" should 3034 // contain "bar". 3035 assert_eq!( 3036 guest 3037 .ssh_command("sudo cat mount_image/foo") 3038 .unwrap() 3039 .trim(), 3040 "bar" 3041 ); 3042 }); 3043 3044 let _ = child.kill(); 3045 let output = child.wait_with_output().unwrap(); 3046 3047 handle_child_output(r, &output); 3048 } 3049 3050 #[test] 3051 fn test_pci_multiple_segments_numa_node() { 3052 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3053 let guest = Guest::new(Box::new(focal)); 3054 let api_socket = temp_api_path(&guest.tmp_dir); 3055 #[cfg(target_arch = "x86_64")] 3056 let kernel_path = direct_kernel_boot_path(); 3057 #[cfg(target_arch = "aarch64")] 3058 let kernel_path = edk2_path(); 3059 3060 // Prepare another disk file for the virtio-disk device 3061 let test_disk_path = String::from( 3062 guest 3063 .tmp_dir 3064 .as_path() 3065 .join("test-disk.raw") 3066 .to_str() 3067 .unwrap(), 3068 ); 3069 assert!( 3070 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 3071 ); 3072 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 3073 const TEST_DISK_NODE: u16 = 1; 3074 3075 let mut child = GuestCommand::new(&guest) 3076 .args(["--platform", "num_pci_segments=2"]) 3077 .args(["--cpus", "boot=2"]) 3078 .args(["--memory", "size=0"]) 3079 .args(["--memory-zone", "id=mem0,size=256M", "id=mem1,size=256M"]) 3080 .args([ 3081 "--numa", 3082 "guest_numa_id=0,cpus=[0],distances=[1@20],memory_zones=mem0,pci_segments=[0]", 3083 "guest_numa_id=1,cpus=[1],distances=[0@20],memory_zones=mem1,pci_segments=[1]", 3084 ]) 3085 .args(["--kernel", kernel_path.to_str().unwrap()]) 3086 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3087 .args(["--api-socket", &api_socket]) 3088 .capture_output() 3089 .args([ 3090 "--disk", 3091 format!( 3092 "path={}", 3093 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3094 ) 3095 .as_str(), 3096 format!( 3097 "path={}", 3098 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3099 ) 3100 .as_str(), 3101 format!("path={test_disk_path},pci_segment={TEST_DISK_NODE}").as_str(), 3102 ]) 3103 .default_net() 3104 .spawn() 3105 .unwrap(); 3106 3107 let cmd = "cat /sys/block/vdc/device/../numa_node"; 3108 3109 let r = std::panic::catch_unwind(|| { 3110 guest.wait_vm_boot(None).unwrap(); 3111 3112 assert_eq!( 3113 guest 3114 .ssh_command(cmd) 3115 .unwrap() 3116 .trim() 3117 .parse::<u16>() 3118 .unwrap_or_default(), 3119 TEST_DISK_NODE 3120 ); 3121 }); 3122 3123 let _ = child.kill(); 3124 let output = child.wait_with_output().unwrap(); 3125 3126 handle_child_output(r, &output); 3127 } 3128 3129 #[test] 3130 fn test_direct_kernel_boot() { 3131 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3132 let guest = Guest::new(Box::new(focal)); 3133 3134 let kernel_path = direct_kernel_boot_path(); 3135 3136 let mut child = GuestCommand::new(&guest) 3137 .args(["--cpus", "boot=1"]) 3138 .args(["--memory", "size=512M"]) 3139 .args(["--kernel", kernel_path.to_str().unwrap()]) 3140 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3141 .default_disks() 3142 .default_net() 3143 .capture_output() 3144 .spawn() 3145 .unwrap(); 3146 3147 let r = std::panic::catch_unwind(|| { 3148 guest.wait_vm_boot(None).unwrap(); 3149 3150 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3151 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3152 3153 let grep_cmd = if cfg!(target_arch = "x86_64") { 3154 "grep -c PCI-MSI /proc/interrupts" 3155 } else { 3156 "grep -c ITS-MSI /proc/interrupts" 3157 }; 3158 assert_eq!( 3159 guest 3160 .ssh_command(grep_cmd) 3161 .unwrap() 3162 .trim() 3163 .parse::<u32>() 3164 .unwrap_or_default(), 3165 12 3166 ); 3167 }); 3168 3169 let _ = child.kill(); 3170 let output = child.wait_with_output().unwrap(); 3171 3172 handle_child_output(r, &output); 3173 } 3174 3175 #[test] 3176 #[cfg(target_arch = "x86_64")] 3177 fn test_direct_kernel_boot_bzimage() { 3178 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3179 let guest = Guest::new(Box::new(focal)); 3180 3181 let mut kernel_path = direct_kernel_boot_path(); 3182 // Replace the default kernel with the bzImage. 3183 kernel_path.pop(); 3184 kernel_path.push("bzImage"); 3185 3186 let mut child = GuestCommand::new(&guest) 3187 .args(["--cpus", "boot=1"]) 3188 .args(["--memory", "size=512M"]) 3189 .args(["--kernel", kernel_path.to_str().unwrap()]) 3190 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3191 .default_disks() 3192 .default_net() 3193 .capture_output() 3194 .spawn() 3195 .unwrap(); 3196 3197 let r = std::panic::catch_unwind(|| { 3198 guest.wait_vm_boot(None).unwrap(); 3199 3200 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3201 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3202 3203 let grep_cmd = if cfg!(target_arch = "x86_64") { 3204 "grep -c PCI-MSI /proc/interrupts" 3205 } else { 3206 "grep -c ITS-MSI /proc/interrupts" 3207 }; 3208 assert_eq!( 3209 guest 3210 .ssh_command(grep_cmd) 3211 .unwrap() 3212 .trim() 3213 .parse::<u32>() 3214 .unwrap_or_default(), 3215 12 3216 ); 3217 }); 3218 3219 let _ = child.kill(); 3220 let output = child.wait_with_output().unwrap(); 3221 3222 handle_child_output(r, &output); 3223 } 3224 3225 fn _test_virtio_block(image_name: &str, disable_io_uring: bool, disable_aio: bool) { 3226 let focal = UbuntuDiskConfig::new(image_name.to_string()); 3227 let guest = Guest::new(Box::new(focal)); 3228 3229 let mut workload_path = dirs::home_dir().unwrap(); 3230 workload_path.push("workloads"); 3231 3232 let mut blk_file_path = workload_path; 3233 blk_file_path.push("blk.img"); 3234 3235 let kernel_path = direct_kernel_boot_path(); 3236 3237 let mut cloud_child = GuestCommand::new(&guest) 3238 .args(["--cpus", "boot=4"]) 3239 .args(["--memory", "size=512M,shared=on"]) 3240 .args(["--kernel", kernel_path.to_str().unwrap()]) 3241 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3242 .args([ 3243 "--disk", 3244 format!( 3245 "path={}", 3246 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3247 ) 3248 .as_str(), 3249 format!( 3250 "path={}", 3251 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3252 ) 3253 .as_str(), 3254 format!( 3255 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", 3256 blk_file_path.to_str().unwrap(), 3257 disable_io_uring, 3258 disable_aio, 3259 ) 3260 .as_str(), 3261 ]) 3262 .default_net() 3263 .capture_output() 3264 .spawn() 3265 .unwrap(); 3266 3267 let r = std::panic::catch_unwind(|| { 3268 guest.wait_vm_boot(None).unwrap(); 3269 3270 // Check both if /dev/vdc exists and if the block size is 16M. 3271 assert_eq!( 3272 guest 3273 .ssh_command("lsblk | grep vdc | grep -c 16M") 3274 .unwrap() 3275 .trim() 3276 .parse::<u32>() 3277 .unwrap_or_default(), 3278 1 3279 ); 3280 3281 // Check both if /dev/vdc exists and if this block is RO. 3282 assert_eq!( 3283 guest 3284 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 3285 .unwrap() 3286 .trim() 3287 .parse::<u32>() 3288 .unwrap_or_default(), 3289 1 3290 ); 3291 3292 // Check if the number of queues is 4. 3293 assert_eq!( 3294 guest 3295 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 3296 .unwrap() 3297 .trim() 3298 .parse::<u32>() 3299 .unwrap_or_default(), 3300 4 3301 ); 3302 }); 3303 3304 let _ = cloud_child.kill(); 3305 let output = cloud_child.wait_with_output().unwrap(); 3306 3307 handle_child_output(r, &output); 3308 } 3309 3310 #[test] 3311 fn test_virtio_block_io_uring() { 3312 _test_virtio_block(FOCAL_IMAGE_NAME, false, true) 3313 } 3314 3315 #[test] 3316 fn test_virtio_block_aio() { 3317 _test_virtio_block(FOCAL_IMAGE_NAME, true, false) 3318 } 3319 3320 #[test] 3321 fn test_virtio_block_sync() { 3322 _test_virtio_block(FOCAL_IMAGE_NAME, true, true) 3323 } 3324 3325 #[test] 3326 fn test_virtio_block_qcow2() { 3327 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false, false) 3328 } 3329 3330 #[test] 3331 fn test_virtio_block_qcow2_backing_file() { 3332 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE, false, false) 3333 } 3334 3335 #[test] 3336 fn test_virtio_block_vhd() { 3337 let mut workload_path = dirs::home_dir().unwrap(); 3338 workload_path.push("workloads"); 3339 3340 let mut raw_file_path = workload_path.clone(); 3341 let mut vhd_file_path = workload_path; 3342 raw_file_path.push(FOCAL_IMAGE_NAME); 3343 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 3344 3345 // Generate VHD file from RAW file 3346 std::process::Command::new("qemu-img") 3347 .arg("convert") 3348 .arg("-p") 3349 .args(["-f", "raw"]) 3350 .args(["-O", "vpc"]) 3351 .args(["-o", "subformat=fixed"]) 3352 .arg(raw_file_path.to_str().unwrap()) 3353 .arg(vhd_file_path.to_str().unwrap()) 3354 .output() 3355 .expect("Expect generating VHD image from RAW image"); 3356 3357 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false, false) 3358 } 3359 3360 #[test] 3361 fn test_virtio_block_vhdx() { 3362 let mut workload_path = dirs::home_dir().unwrap(); 3363 workload_path.push("workloads"); 3364 3365 let mut raw_file_path = workload_path.clone(); 3366 let mut vhdx_file_path = workload_path; 3367 raw_file_path.push(FOCAL_IMAGE_NAME); 3368 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 3369 3370 // Generate dynamic VHDX file from RAW file 3371 std::process::Command::new("qemu-img") 3372 .arg("convert") 3373 .arg("-p") 3374 .args(["-f", "raw"]) 3375 .args(["-O", "vhdx"]) 3376 .arg(raw_file_path.to_str().unwrap()) 3377 .arg(vhdx_file_path.to_str().unwrap()) 3378 .output() 3379 .expect("Expect generating dynamic VHDx image from RAW image"); 3380 3381 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false, false) 3382 } 3383 3384 #[test] 3385 fn test_virtio_block_dynamic_vhdx_expand() { 3386 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 3387 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 3388 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 3389 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 3390 3391 let mut workload_path = dirs::home_dir().unwrap(); 3392 workload_path.push("workloads"); 3393 3394 let mut vhdx_file_path = workload_path; 3395 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 3396 let vhdx_path = vhdx_file_path.to_str().unwrap(); 3397 3398 // Generate a 100 MiB dynamic VHDX file 3399 std::process::Command::new("qemu-img") 3400 .arg("create") 3401 .args(["-f", "vhdx"]) 3402 .arg(vhdx_path) 3403 .arg(VIRTUAL_DISK_SIZE.to_string()) 3404 .output() 3405 .expect("Expect generating dynamic VHDx image from RAW image"); 3406 3407 // Check if the size matches with empty VHDx file size 3408 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 3409 3410 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3411 let guest = Guest::new(Box::new(focal)); 3412 let kernel_path = direct_kernel_boot_path(); 3413 3414 let mut cloud_child = GuestCommand::new(&guest) 3415 .args(["--cpus", "boot=1"]) 3416 .args(["--memory", "size=512M"]) 3417 .args(["--kernel", kernel_path.to_str().unwrap()]) 3418 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3419 .args([ 3420 "--disk", 3421 format!( 3422 "path={}", 3423 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3424 ) 3425 .as_str(), 3426 format!( 3427 "path={}", 3428 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3429 ) 3430 .as_str(), 3431 format!("path={vhdx_path}").as_str(), 3432 ]) 3433 .default_net() 3434 .capture_output() 3435 .spawn() 3436 .unwrap(); 3437 3438 let r = std::panic::catch_unwind(|| { 3439 guest.wait_vm_boot(None).unwrap(); 3440 3441 // Check both if /dev/vdc exists and if the block size is 100 MiB. 3442 assert_eq!( 3443 guest 3444 .ssh_command("lsblk | grep vdc | grep -c 100M") 3445 .unwrap() 3446 .trim() 3447 .parse::<u32>() 3448 .unwrap_or_default(), 3449 1 3450 ); 3451 3452 // Write 100 MB of data to the VHDx disk 3453 guest 3454 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 3455 .unwrap(); 3456 }); 3457 3458 // Check if the size matches with expected expanded VHDx file size 3459 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 3460 3461 let _ = cloud_child.kill(); 3462 let output = cloud_child.wait_with_output().unwrap(); 3463 3464 handle_child_output(r, &output); 3465 } 3466 3467 fn vhdx_image_size(disk_name: &str) -> u64 { 3468 std::fs::File::open(disk_name) 3469 .unwrap() 3470 .seek(SeekFrom::End(0)) 3471 .unwrap() 3472 } 3473 3474 #[test] 3475 fn test_virtio_block_direct_and_firmware() { 3476 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3477 let guest = Guest::new(Box::new(focal)); 3478 3479 // The OS disk must be copied to a location that is not backed by 3480 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 3481 // with EINVAL because tmpfs doesn't support this flag. 3482 let mut workloads_path = dirs::home_dir().unwrap(); 3483 workloads_path.push("workloads"); 3484 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 3485 let mut os_path = os_dir.as_path().to_path_buf(); 3486 os_path.push("osdisk.img"); 3487 rate_limited_copy( 3488 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3489 os_path.as_path(), 3490 ) 3491 .expect("copying of OS disk failed"); 3492 3493 let mut child = GuestCommand::new(&guest) 3494 .args(["--cpus", "boot=1"]) 3495 .args(["--memory", "size=512M"]) 3496 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 3497 .args([ 3498 "--disk", 3499 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 3500 format!( 3501 "path={}", 3502 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3503 ) 3504 .as_str(), 3505 ]) 3506 .default_net() 3507 .capture_output() 3508 .spawn() 3509 .unwrap(); 3510 3511 let r = std::panic::catch_unwind(|| { 3512 guest.wait_vm_boot(Some(120)).unwrap(); 3513 }); 3514 3515 let _ = child.kill(); 3516 let output = child.wait_with_output().unwrap(); 3517 3518 handle_child_output(r, &output); 3519 } 3520 3521 #[test] 3522 fn test_vhost_user_net_default() { 3523 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 3524 } 3525 3526 #[test] 3527 fn test_vhost_user_net_named_tap() { 3528 test_vhost_user_net( 3529 Some("mytap0"), 3530 2, 3531 &prepare_vhost_user_net_daemon, 3532 false, 3533 false, 3534 ) 3535 } 3536 3537 #[test] 3538 fn test_vhost_user_net_existing_tap() { 3539 test_vhost_user_net( 3540 Some("vunet-tap0"), 3541 2, 3542 &prepare_vhost_user_net_daemon, 3543 false, 3544 false, 3545 ) 3546 } 3547 3548 #[test] 3549 fn test_vhost_user_net_multiple_queues() { 3550 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 3551 } 3552 3553 #[test] 3554 fn test_vhost_user_net_tap_multiple_queues() { 3555 test_vhost_user_net( 3556 Some("vunet-tap1"), 3557 4, 3558 &prepare_vhost_user_net_daemon, 3559 false, 3560 false, 3561 ) 3562 } 3563 3564 #[test] 3565 fn test_vhost_user_net_host_mac() { 3566 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 3567 } 3568 3569 #[test] 3570 fn test_vhost_user_net_client_mode() { 3571 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 3572 } 3573 3574 #[test] 3575 #[cfg(not(target_arch = "aarch64"))] 3576 fn test_vhost_user_blk_default() { 3577 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 3578 } 3579 3580 #[test] 3581 #[cfg(not(target_arch = "aarch64"))] 3582 fn test_vhost_user_blk_readonly() { 3583 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3584 } 3585 3586 #[test] 3587 #[cfg(not(target_arch = "aarch64"))] 3588 fn test_vhost_user_blk_direct() { 3589 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3590 } 3591 3592 #[test] 3593 fn test_boot_from_vhost_user_blk_default() { 3594 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3595 } 3596 3597 #[test] 3598 #[cfg(target_arch = "x86_64")] 3599 fn test_split_irqchip() { 3600 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3601 let guest = Guest::new(Box::new(focal)); 3602 3603 let mut child = GuestCommand::new(&guest) 3604 .args(["--cpus", "boot=1"]) 3605 .args(["--memory", "size=512M"]) 3606 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3607 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3608 .default_disks() 3609 .default_net() 3610 .capture_output() 3611 .spawn() 3612 .unwrap(); 3613 3614 let r = std::panic::catch_unwind(|| { 3615 guest.wait_vm_boot(None).unwrap(); 3616 3617 assert_eq!( 3618 guest 3619 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3620 .unwrap() 3621 .trim() 3622 .parse::<u32>() 3623 .unwrap_or(1), 3624 0 3625 ); 3626 assert_eq!( 3627 guest 3628 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3629 .unwrap() 3630 .trim() 3631 .parse::<u32>() 3632 .unwrap_or(1), 3633 0 3634 ); 3635 }); 3636 3637 let _ = child.kill(); 3638 let output = child.wait_with_output().unwrap(); 3639 3640 handle_child_output(r, &output); 3641 } 3642 3643 #[test] 3644 #[cfg(target_arch = "x86_64")] 3645 fn test_dmi_serial_number() { 3646 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3647 let guest = Guest::new(Box::new(focal)); 3648 3649 let mut child = GuestCommand::new(&guest) 3650 .args(["--cpus", "boot=1"]) 3651 .args(["--memory", "size=512M"]) 3652 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3653 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3654 .args(["--platform", "serial_number=a=b;c=d"]) 3655 .default_disks() 3656 .default_net() 3657 .capture_output() 3658 .spawn() 3659 .unwrap(); 3660 3661 let r = std::panic::catch_unwind(|| { 3662 guest.wait_vm_boot(None).unwrap(); 3663 3664 assert_eq!( 3665 guest 3666 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3667 .unwrap() 3668 .trim(), 3669 "a=b;c=d" 3670 ); 3671 }); 3672 3673 let _ = child.kill(); 3674 let output = child.wait_with_output().unwrap(); 3675 3676 handle_child_output(r, &output); 3677 } 3678 3679 #[test] 3680 #[cfg(target_arch = "x86_64")] 3681 fn test_dmi_uuid() { 3682 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3683 let guest = Guest::new(Box::new(focal)); 3684 3685 let mut child = GuestCommand::new(&guest) 3686 .args(["--cpus", "boot=1"]) 3687 .args(["--memory", "size=512M"]) 3688 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3689 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3690 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3691 .default_disks() 3692 .default_net() 3693 .capture_output() 3694 .spawn() 3695 .unwrap(); 3696 3697 let r = std::panic::catch_unwind(|| { 3698 guest.wait_vm_boot(None).unwrap(); 3699 3700 assert_eq!( 3701 guest 3702 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3703 .unwrap() 3704 .trim(), 3705 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3706 ); 3707 }); 3708 3709 let _ = child.kill(); 3710 let output = child.wait_with_output().unwrap(); 3711 3712 handle_child_output(r, &output); 3713 } 3714 3715 #[test] 3716 #[cfg(target_arch = "x86_64")] 3717 fn test_dmi_oem_strings() { 3718 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3719 let guest = Guest::new(Box::new(focal)); 3720 3721 let s1 = "io.systemd.credential:xx=yy"; 3722 let s2 = "This is a test string"; 3723 3724 let oem_strings = format!("oem_strings=[{s1},{s2}]"); 3725 3726 let mut child = GuestCommand::new(&guest) 3727 .args(["--cpus", "boot=1"]) 3728 .args(["--memory", "size=512M"]) 3729 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3730 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3731 .args(["--platform", &oem_strings]) 3732 .default_disks() 3733 .default_net() 3734 .capture_output() 3735 .spawn() 3736 .unwrap(); 3737 3738 let r = std::panic::catch_unwind(|| { 3739 guest.wait_vm_boot(None).unwrap(); 3740 3741 assert_eq!( 3742 guest 3743 .ssh_command("sudo dmidecode --oem-string count") 3744 .unwrap() 3745 .trim(), 3746 "2" 3747 ); 3748 3749 assert_eq!( 3750 guest 3751 .ssh_command("sudo dmidecode --oem-string 1") 3752 .unwrap() 3753 .trim(), 3754 s1 3755 ); 3756 3757 assert_eq!( 3758 guest 3759 .ssh_command("sudo dmidecode --oem-string 2") 3760 .unwrap() 3761 .trim(), 3762 s2 3763 ); 3764 }); 3765 3766 let _ = child.kill(); 3767 let output = child.wait_with_output().unwrap(); 3768 3769 handle_child_output(r, &output); 3770 } 3771 3772 #[test] 3773 fn test_virtio_fs() { 3774 _test_virtio_fs(&prepare_virtiofsd, false, None) 3775 } 3776 3777 #[test] 3778 fn test_virtio_fs_hotplug() { 3779 _test_virtio_fs(&prepare_virtiofsd, true, None) 3780 } 3781 3782 #[test] 3783 #[cfg(not(feature = "mshv"))] 3784 fn test_virtio_fs_multi_segment_hotplug() { 3785 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3786 } 3787 3788 #[test] 3789 #[cfg(not(feature = "mshv"))] 3790 fn test_virtio_fs_multi_segment() { 3791 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3792 } 3793 3794 #[test] 3795 fn test_virtio_pmem_persist_writes() { 3796 test_virtio_pmem(false, false) 3797 } 3798 3799 #[test] 3800 fn test_virtio_pmem_discard_writes() { 3801 test_virtio_pmem(true, false) 3802 } 3803 3804 #[test] 3805 fn test_virtio_pmem_with_size() { 3806 test_virtio_pmem(true, true) 3807 } 3808 3809 #[test] 3810 fn test_boot_from_virtio_pmem() { 3811 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3812 let guest = Guest::new(Box::new(focal)); 3813 3814 let kernel_path = direct_kernel_boot_path(); 3815 3816 let mut child = GuestCommand::new(&guest) 3817 .args(["--cpus", "boot=1"]) 3818 .args(["--memory", "size=512M"]) 3819 .args(["--kernel", kernel_path.to_str().unwrap()]) 3820 .args([ 3821 "--disk", 3822 format!( 3823 "path={}", 3824 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3825 ) 3826 .as_str(), 3827 ]) 3828 .default_net() 3829 .args([ 3830 "--pmem", 3831 format!( 3832 "file={},size={}", 3833 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3834 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3835 .unwrap() 3836 .len() 3837 ) 3838 .as_str(), 3839 ]) 3840 .args([ 3841 "--cmdline", 3842 DIRECT_KERNEL_BOOT_CMDLINE 3843 .replace("vda1", "pmem0p1") 3844 .as_str(), 3845 ]) 3846 .capture_output() 3847 .spawn() 3848 .unwrap(); 3849 3850 let r = std::panic::catch_unwind(|| { 3851 guest.wait_vm_boot(None).unwrap(); 3852 3853 // Simple checks to validate the VM booted properly 3854 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3855 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3856 }); 3857 3858 let _ = child.kill(); 3859 let output = child.wait_with_output().unwrap(); 3860 3861 handle_child_output(r, &output); 3862 } 3863 3864 #[test] 3865 fn test_multiple_network_interfaces() { 3866 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3867 let guest = Guest::new(Box::new(focal)); 3868 3869 let kernel_path = direct_kernel_boot_path(); 3870 3871 let mut child = GuestCommand::new(&guest) 3872 .args(["--cpus", "boot=1"]) 3873 .args(["--memory", "size=512M"]) 3874 .args(["--kernel", kernel_path.to_str().unwrap()]) 3875 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3876 .default_disks() 3877 .args([ 3878 "--net", 3879 guest.default_net_string().as_str(), 3880 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3881 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3882 ]) 3883 .capture_output() 3884 .spawn() 3885 .unwrap(); 3886 3887 let r = std::panic::catch_unwind(|| { 3888 guest.wait_vm_boot(None).unwrap(); 3889 3890 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3891 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3892 3893 // 3 network interfaces + default localhost ==> 4 interfaces 3894 assert_eq!( 3895 guest 3896 .ssh_command("ip -o link | wc -l") 3897 .unwrap() 3898 .trim() 3899 .parse::<u32>() 3900 .unwrap_or_default(), 3901 4 3902 ); 3903 }); 3904 3905 let _ = child.kill(); 3906 let output = child.wait_with_output().unwrap(); 3907 3908 handle_child_output(r, &output); 3909 } 3910 3911 #[test] 3912 #[cfg(target_arch = "aarch64")] 3913 fn test_pmu_on() { 3914 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3915 let guest = Guest::new(Box::new(focal)); 3916 let mut child = GuestCommand::new(&guest) 3917 .args(["--cpus", "boot=1"]) 3918 .args(["--memory", "size=512M"]) 3919 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3920 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3921 .default_disks() 3922 .default_net() 3923 .capture_output() 3924 .spawn() 3925 .unwrap(); 3926 3927 let r = std::panic::catch_unwind(|| { 3928 guest.wait_vm_boot(None).unwrap(); 3929 3930 // Test that PMU exists. 3931 assert_eq!( 3932 guest 3933 .ssh_command(GREP_PMU_IRQ_CMD) 3934 .unwrap() 3935 .trim() 3936 .parse::<u32>() 3937 .unwrap_or_default(), 3938 1 3939 ); 3940 }); 3941 3942 let _ = child.kill(); 3943 let output = child.wait_with_output().unwrap(); 3944 3945 handle_child_output(r, &output); 3946 } 3947 3948 #[test] 3949 fn test_serial_off() { 3950 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3951 let guest = Guest::new(Box::new(focal)); 3952 let mut child = GuestCommand::new(&guest) 3953 .args(["--cpus", "boot=1"]) 3954 .args(["--memory", "size=512M"]) 3955 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3956 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3957 .default_disks() 3958 .default_net() 3959 .args(["--serial", "off"]) 3960 .capture_output() 3961 .spawn() 3962 .unwrap(); 3963 3964 let r = std::panic::catch_unwind(|| { 3965 guest.wait_vm_boot(None).unwrap(); 3966 3967 // Test that there is no ttyS0 3968 assert_eq!( 3969 guest 3970 .ssh_command(GREP_SERIAL_IRQ_CMD) 3971 .unwrap() 3972 .trim() 3973 .parse::<u32>() 3974 .unwrap_or(1), 3975 0 3976 ); 3977 }); 3978 3979 let _ = child.kill(); 3980 let output = child.wait_with_output().unwrap(); 3981 3982 handle_child_output(r, &output); 3983 } 3984 3985 #[test] 3986 fn test_serial_null() { 3987 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3988 let guest = Guest::new(Box::new(focal)); 3989 let mut cmd = GuestCommand::new(&guest); 3990 #[cfg(target_arch = "x86_64")] 3991 let console_str: &str = "console=ttyS0"; 3992 #[cfg(target_arch = "aarch64")] 3993 let console_str: &str = "console=ttyAMA0"; 3994 3995 cmd.args(["--cpus", "boot=1"]) 3996 .args(["--memory", "size=512M"]) 3997 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3998 .args([ 3999 "--cmdline", 4000 DIRECT_KERNEL_BOOT_CMDLINE 4001 .replace("console=hvc0 ", console_str) 4002 .as_str(), 4003 ]) 4004 .default_disks() 4005 .default_net() 4006 .args(["--serial", "null"]) 4007 .args(["--console", "off"]) 4008 .capture_output(); 4009 4010 let mut child = cmd.spawn().unwrap(); 4011 4012 let r = std::panic::catch_unwind(|| { 4013 guest.wait_vm_boot(None).unwrap(); 4014 4015 // Test that there is a ttyS0 4016 assert_eq!( 4017 guest 4018 .ssh_command(GREP_SERIAL_IRQ_CMD) 4019 .unwrap() 4020 .trim() 4021 .parse::<u32>() 4022 .unwrap_or_default(), 4023 1 4024 ); 4025 }); 4026 4027 let _ = child.kill(); 4028 let output = child.wait_with_output().unwrap(); 4029 handle_child_output(r, &output); 4030 4031 let r = std::panic::catch_unwind(|| { 4032 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4033 }); 4034 4035 handle_child_output(r, &output); 4036 } 4037 4038 #[test] 4039 fn test_serial_tty() { 4040 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4041 let guest = Guest::new(Box::new(focal)); 4042 4043 let kernel_path = direct_kernel_boot_path(); 4044 4045 #[cfg(target_arch = "x86_64")] 4046 let console_str: &str = "console=ttyS0"; 4047 #[cfg(target_arch = "aarch64")] 4048 let console_str: &str = "console=ttyAMA0"; 4049 4050 let mut child = GuestCommand::new(&guest) 4051 .args(["--cpus", "boot=1"]) 4052 .args(["--memory", "size=512M"]) 4053 .args(["--kernel", kernel_path.to_str().unwrap()]) 4054 .args([ 4055 "--cmdline", 4056 DIRECT_KERNEL_BOOT_CMDLINE 4057 .replace("console=hvc0 ", console_str) 4058 .as_str(), 4059 ]) 4060 .default_disks() 4061 .default_net() 4062 .args(["--serial", "tty"]) 4063 .args(["--console", "off"]) 4064 .capture_output() 4065 .spawn() 4066 .unwrap(); 4067 4068 let r = std::panic::catch_unwind(|| { 4069 guest.wait_vm_boot(None).unwrap(); 4070 4071 // Test that there is a ttyS0 4072 assert_eq!( 4073 guest 4074 .ssh_command(GREP_SERIAL_IRQ_CMD) 4075 .unwrap() 4076 .trim() 4077 .parse::<u32>() 4078 .unwrap_or_default(), 4079 1 4080 ); 4081 }); 4082 4083 // This sleep is needed to wait for the login prompt 4084 thread::sleep(std::time::Duration::new(2, 0)); 4085 4086 let _ = child.kill(); 4087 let output = child.wait_with_output().unwrap(); 4088 handle_child_output(r, &output); 4089 4090 let r = std::panic::catch_unwind(|| { 4091 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4092 }); 4093 4094 handle_child_output(r, &output); 4095 } 4096 4097 #[test] 4098 fn test_serial_file() { 4099 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4100 let guest = Guest::new(Box::new(focal)); 4101 4102 let serial_path = guest.tmp_dir.as_path().join("serial-output"); 4103 #[cfg(target_arch = "x86_64")] 4104 let console_str: &str = "console=ttyS0"; 4105 #[cfg(target_arch = "aarch64")] 4106 let console_str: &str = "console=ttyAMA0"; 4107 4108 let mut child = GuestCommand::new(&guest) 4109 .args(["--cpus", "boot=1"]) 4110 .args(["--memory", "size=512M"]) 4111 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4112 .args([ 4113 "--cmdline", 4114 DIRECT_KERNEL_BOOT_CMDLINE 4115 .replace("console=hvc0 ", console_str) 4116 .as_str(), 4117 ]) 4118 .default_disks() 4119 .default_net() 4120 .args([ 4121 "--serial", 4122 format!("file={}", serial_path.to_str().unwrap()).as_str(), 4123 ]) 4124 .capture_output() 4125 .spawn() 4126 .unwrap(); 4127 4128 let r = std::panic::catch_unwind(|| { 4129 guest.wait_vm_boot(None).unwrap(); 4130 4131 // Test that there is a ttyS0 4132 assert_eq!( 4133 guest 4134 .ssh_command(GREP_SERIAL_IRQ_CMD) 4135 .unwrap() 4136 .trim() 4137 .parse::<u32>() 4138 .unwrap_or_default(), 4139 1 4140 ); 4141 4142 guest.ssh_command("sudo shutdown -h now").unwrap(); 4143 }); 4144 4145 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4146 let _ = child.kill(); 4147 let output = child.wait_with_output().unwrap(); 4148 handle_child_output(r, &output); 4149 4150 let r = std::panic::catch_unwind(|| { 4151 // Check that the cloud-hypervisor binary actually terminated 4152 assert!(output.status.success()); 4153 4154 // Do this check after shutdown of the VM as an easy way to ensure 4155 // all writes are flushed to disk 4156 let mut f = std::fs::File::open(serial_path).unwrap(); 4157 let mut buf = String::new(); 4158 f.read_to_string(&mut buf).unwrap(); 4159 assert!(buf.contains(CONSOLE_TEST_STRING)); 4160 }); 4161 4162 handle_child_output(r, &output); 4163 } 4164 4165 #[test] 4166 fn test_pty_interaction() { 4167 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4168 let guest = Guest::new(Box::new(focal)); 4169 let api_socket = temp_api_path(&guest.tmp_dir); 4170 let serial_option = if cfg!(target_arch = "x86_64") { 4171 " console=ttyS0" 4172 } else { 4173 " console=ttyAMA0" 4174 }; 4175 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4176 4177 let mut child = GuestCommand::new(&guest) 4178 .args(["--cpus", "boot=1"]) 4179 .args(["--memory", "size=512M"]) 4180 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4181 .args(["--cmdline", &cmdline]) 4182 .default_disks() 4183 .default_net() 4184 .args(["--serial", "null"]) 4185 .args(["--console", "pty"]) 4186 .args(["--api-socket", &api_socket]) 4187 .spawn() 4188 .unwrap(); 4189 4190 let r = std::panic::catch_unwind(|| { 4191 guest.wait_vm_boot(None).unwrap(); 4192 // Get pty fd for console 4193 let console_path = get_pty_path(&api_socket, "console"); 4194 _test_pty_interaction(console_path); 4195 4196 guest.ssh_command("sudo shutdown -h now").unwrap(); 4197 }); 4198 4199 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4200 let _ = child.kill(); 4201 let output = child.wait_with_output().unwrap(); 4202 handle_child_output(r, &output); 4203 4204 let r = std::panic::catch_unwind(|| { 4205 // Check that the cloud-hypervisor binary actually terminated 4206 assert!(output.status.success()) 4207 }); 4208 handle_child_output(r, &output); 4209 } 4210 4211 #[test] 4212 fn test_serial_socket_interaction() { 4213 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4214 let guest = Guest::new(Box::new(focal)); 4215 let serial_socket = guest.tmp_dir.as_path().join("serial.socket"); 4216 let serial_socket_pty = guest.tmp_dir.as_path().join("serial.pty"); 4217 let serial_option = if cfg!(target_arch = "x86_64") { 4218 " console=ttyS0" 4219 } else { 4220 " console=ttyAMA0" 4221 }; 4222 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4223 4224 let mut child = GuestCommand::new(&guest) 4225 .args(["--cpus", "boot=1"]) 4226 .args(["--memory", "size=512M"]) 4227 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4228 .args(["--cmdline", &cmdline]) 4229 .default_disks() 4230 .default_net() 4231 .args(["--console", "null"]) 4232 .args([ 4233 "--serial", 4234 format!("socket={}", serial_socket.to_str().unwrap()).as_str(), 4235 ]) 4236 .spawn() 4237 .unwrap(); 4238 4239 let _ = std::panic::catch_unwind(|| { 4240 guest.wait_vm_boot(None).unwrap(); 4241 }); 4242 4243 let mut socat_command = Command::new("socat"); 4244 let socat_args = [ 4245 &format!("pty,link={},raw", serial_socket_pty.display()), 4246 &format!("UNIX-CONNECT:{}", serial_socket.display()), 4247 ]; 4248 socat_command.args(socat_args); 4249 4250 let mut socat_child = socat_command.spawn().unwrap(); 4251 thread::sleep(std::time::Duration::new(1, 0)); 4252 4253 let _ = std::panic::catch_unwind(|| { 4254 _test_pty_interaction(serial_socket_pty); 4255 }); 4256 4257 let _ = socat_child.kill(); 4258 4259 let r = std::panic::catch_unwind(|| { 4260 guest.ssh_command("sudo shutdown -h now").unwrap(); 4261 }); 4262 4263 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4264 let _ = child.kill(); 4265 let output = child.wait_with_output().unwrap(); 4266 handle_child_output(r, &output); 4267 4268 let r = std::panic::catch_unwind(|| { 4269 // Check that the cloud-hypervisor binary actually terminated 4270 if !output.status.success() { 4271 panic!( 4272 "Cloud Hypervisor process failed to terminate gracefully: {:?}", 4273 output.status 4274 ); 4275 } 4276 }); 4277 handle_child_output(r, &output); 4278 } 4279 4280 #[test] 4281 fn test_virtio_console() { 4282 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4283 let guest = Guest::new(Box::new(focal)); 4284 4285 let kernel_path = direct_kernel_boot_path(); 4286 4287 let mut child = GuestCommand::new(&guest) 4288 .args(["--cpus", "boot=1"]) 4289 .args(["--memory", "size=512M"]) 4290 .args(["--kernel", kernel_path.to_str().unwrap()]) 4291 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4292 .default_disks() 4293 .default_net() 4294 .args(["--console", "tty"]) 4295 .args(["--serial", "null"]) 4296 .capture_output() 4297 .spawn() 4298 .unwrap(); 4299 4300 let text = String::from("On a branch floating down river a cricket, singing."); 4301 let cmd = format!("echo {text} | sudo tee /dev/hvc0"); 4302 4303 let r = std::panic::catch_unwind(|| { 4304 guest.wait_vm_boot(None).unwrap(); 4305 4306 assert!(guest 4307 .does_device_vendor_pair_match("0x1043", "0x1af4") 4308 .unwrap_or_default()); 4309 4310 guest.ssh_command(&cmd).unwrap(); 4311 }); 4312 4313 let _ = child.kill(); 4314 let output = child.wait_with_output().unwrap(); 4315 handle_child_output(r, &output); 4316 4317 let r = std::panic::catch_unwind(|| { 4318 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 4319 }); 4320 4321 handle_child_output(r, &output); 4322 } 4323 4324 #[test] 4325 fn test_console_file() { 4326 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4327 let guest = Guest::new(Box::new(focal)); 4328 4329 let console_path = guest.tmp_dir.as_path().join("console-output"); 4330 let mut child = GuestCommand::new(&guest) 4331 .args(["--cpus", "boot=1"]) 4332 .args(["--memory", "size=512M"]) 4333 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4334 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4335 .default_disks() 4336 .default_net() 4337 .args([ 4338 "--console", 4339 format!("file={}", console_path.to_str().unwrap()).as_str(), 4340 ]) 4341 .capture_output() 4342 .spawn() 4343 .unwrap(); 4344 4345 guest.wait_vm_boot(None).unwrap(); 4346 4347 guest.ssh_command("sudo shutdown -h now").unwrap(); 4348 4349 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4350 let _ = child.kill(); 4351 let output = child.wait_with_output().unwrap(); 4352 4353 let r = std::panic::catch_unwind(|| { 4354 // Check that the cloud-hypervisor binary actually terminated 4355 assert!(output.status.success()); 4356 4357 // Do this check after shutdown of the VM as an easy way to ensure 4358 // all writes are flushed to disk 4359 let mut f = std::fs::File::open(console_path).unwrap(); 4360 let mut buf = String::new(); 4361 f.read_to_string(&mut buf).unwrap(); 4362 4363 if !buf.contains(CONSOLE_TEST_STRING) { 4364 eprintln!( 4365 "\n\n==== Console file output ====\n\n{buf}\n\n==== End console file output ====" 4366 ); 4367 } 4368 assert!(buf.contains(CONSOLE_TEST_STRING)); 4369 }); 4370 4371 handle_child_output(r, &output); 4372 } 4373 4374 #[test] 4375 #[cfg(target_arch = "x86_64")] 4376 #[cfg(not(feature = "mshv"))] 4377 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 4378 // backed networking interfaces, bound through a simple bridge on the host. 4379 // So if the nested cloud-hypervisor succeeds in getting a directly 4380 // assigned interface from its cloud-hypervisor host, we should be able to 4381 // ssh into it, and verify that it's running with the right kernel command 4382 // line (We tag the command line from cloud-hypervisor for that purpose). 4383 // The third device is added to validate that hotplug works correctly since 4384 // it is being added to the L2 VM through hotplugging mechanism. 4385 // Also, we pass-through a virtio-blk device to the L2 VM to test the 32-bit 4386 // vfio device support 4387 fn test_vfio() { 4388 setup_vfio_network_interfaces(); 4389 4390 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 4391 let guest = Guest::new_from_ip_range(Box::new(jammy), "172.18", 0); 4392 4393 let mut workload_path = dirs::home_dir().unwrap(); 4394 workload_path.push("workloads"); 4395 4396 let kernel_path = direct_kernel_boot_path(); 4397 4398 let mut vfio_path = workload_path.clone(); 4399 vfio_path.push("vfio"); 4400 4401 let mut cloud_init_vfio_base_path = vfio_path.clone(); 4402 cloud_init_vfio_base_path.push("cloudinit.img"); 4403 4404 // We copy our cloudinit into the vfio mount point, for the nested 4405 // cloud-hypervisor guest to use. 4406 rate_limited_copy( 4407 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 4408 &cloud_init_vfio_base_path, 4409 ) 4410 .expect("copying of cloud-init disk failed"); 4411 4412 let mut vfio_disk_path = workload_path.clone(); 4413 vfio_disk_path.push("vfio.img"); 4414 4415 // Create the vfio disk image 4416 let output = Command::new("mkfs.ext4") 4417 .arg("-d") 4418 .arg(vfio_path.to_str().unwrap()) 4419 .arg(vfio_disk_path.to_str().unwrap()) 4420 .arg("2g") 4421 .output() 4422 .unwrap(); 4423 if !output.status.success() { 4424 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 4425 panic!("mkfs.ext4 command generated an error"); 4426 } 4427 4428 let mut blk_file_path = workload_path; 4429 blk_file_path.push("blk.img"); 4430 4431 let vfio_tap0 = "vfio-tap0"; 4432 let vfio_tap1 = "vfio-tap1"; 4433 let vfio_tap2 = "vfio-tap2"; 4434 let vfio_tap3 = "vfio-tap3"; 4435 4436 let mut child = GuestCommand::new(&guest) 4437 .args(["--cpus", "boot=4"]) 4438 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 4439 .args(["--kernel", kernel_path.to_str().unwrap()]) 4440 .args([ 4441 "--disk", 4442 format!( 4443 "path={}", 4444 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4445 ) 4446 .as_str(), 4447 format!( 4448 "path={}", 4449 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4450 ) 4451 .as_str(), 4452 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 4453 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 4454 ]) 4455 .args([ 4456 "--cmdline", 4457 format!( 4458 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 4459 ) 4460 .as_str(), 4461 ]) 4462 .args([ 4463 "--net", 4464 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 4465 format!( 4466 "tap={},mac={},iommu=on", 4467 vfio_tap1, guest.network.l2_guest_mac1 4468 ) 4469 .as_str(), 4470 format!( 4471 "tap={},mac={},iommu=on", 4472 vfio_tap2, guest.network.l2_guest_mac2 4473 ) 4474 .as_str(), 4475 format!( 4476 "tap={},mac={},iommu=on", 4477 vfio_tap3, guest.network.l2_guest_mac3 4478 ) 4479 .as_str(), 4480 ]) 4481 .capture_output() 4482 .spawn() 4483 .unwrap(); 4484 4485 thread::sleep(std::time::Duration::new(30, 0)); 4486 4487 let r = std::panic::catch_unwind(|| { 4488 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 4489 thread::sleep(std::time::Duration::new(120, 0)); 4490 4491 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 4492 // added to its kernel command line. 4493 // Let's ssh into it and verify that it's there. If it is it means 4494 // we're in the right guest (The L2 one) because the QEMU L1 guest 4495 // does not have this command line tag. 4496 assert!(check_matched_lines_count( 4497 guest.ssh_command_l2_1("cat /proc/cmdline").unwrap().trim(), 4498 vec!["VFIOTAG"], 4499 1 4500 )); 4501 4502 // Let's also verify from the second virtio-net device passed to 4503 // the L2 VM. 4504 assert!(check_matched_lines_count( 4505 guest.ssh_command_l2_2("cat /proc/cmdline").unwrap().trim(), 4506 vec!["VFIOTAG"], 4507 1 4508 )); 4509 4510 // Check the amount of PCI devices appearing in L2 VM. 4511 assert!(check_lines_count( 4512 guest 4513 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4514 .unwrap() 4515 .trim(), 4516 8 4517 )); 4518 4519 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 4520 assert!(check_matched_lines_count( 4521 guest.ssh_command_l2_1("lsblk").unwrap().trim(), 4522 vec!["vdc", "16M"], 4523 1 4524 )); 4525 4526 // Hotplug an extra virtio-net device through L2 VM. 4527 guest 4528 .ssh_command_l1( 4529 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 4530 ) 4531 .unwrap(); 4532 guest 4533 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 4534 .unwrap(); 4535 let vfio_hotplug_output = guest 4536 .ssh_command_l1( 4537 "sudo /mnt/ch-remote \ 4538 --api-socket=/tmp/ch_api.sock \ 4539 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 4540 ) 4541 .unwrap(); 4542 assert!(check_matched_lines_count( 4543 vfio_hotplug_output.trim(), 4544 vec!["{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}"], 4545 1 4546 )); 4547 4548 thread::sleep(std::time::Duration::new(10, 0)); 4549 4550 // Let's also verify from the third virtio-net device passed to 4551 // the L2 VM. This third device has been hotplugged through the L2 4552 // VM, so this is our way to validate hotplug works for VFIO PCI. 4553 assert!(check_matched_lines_count( 4554 guest.ssh_command_l2_3("cat /proc/cmdline").unwrap().trim(), 4555 vec!["VFIOTAG"], 4556 1 4557 )); 4558 4559 // Check the amount of PCI devices appearing in L2 VM. 4560 // There should be one more device than before, raising the count 4561 // up to 9 PCI devices. 4562 assert!(check_lines_count( 4563 guest 4564 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4565 .unwrap() 4566 .trim(), 4567 9 4568 )); 4569 4570 // Let's now verify that we can correctly remove the virtio-net 4571 // device through the "remove-device" command responsible for 4572 // unplugging VFIO devices. 4573 guest 4574 .ssh_command_l1( 4575 "sudo /mnt/ch-remote \ 4576 --api-socket=/tmp/ch_api.sock \ 4577 remove-device vfio123", 4578 ) 4579 .unwrap(); 4580 thread::sleep(std::time::Duration::new(10, 0)); 4581 4582 // Check the amount of PCI devices appearing in L2 VM is back down 4583 // to 8 devices. 4584 assert!(check_lines_count( 4585 guest 4586 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4587 .unwrap() 4588 .trim(), 4589 8 4590 )); 4591 4592 // Perform memory hotplug in L2 and validate the memory is showing 4593 // up as expected. In order to check, we will use the virtio-net 4594 // device already passed through L2 as a VFIO device, this will 4595 // verify that VFIO devices are functional with memory hotplug. 4596 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 4597 guest 4598 .ssh_command_l2_1( 4599 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 4600 ) 4601 .unwrap(); 4602 guest 4603 .ssh_command_l1( 4604 "sudo /mnt/ch-remote \ 4605 --api-socket=/tmp/ch_api.sock \ 4606 resize --memory=1073741824", 4607 ) 4608 .unwrap(); 4609 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 4610 }); 4611 4612 let _ = child.kill(); 4613 let output = child.wait_with_output().unwrap(); 4614 4615 cleanup_vfio_network_interfaces(); 4616 4617 handle_child_output(r, &output); 4618 } 4619 4620 #[test] 4621 fn test_direct_kernel_boot_noacpi() { 4622 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4623 let guest = Guest::new(Box::new(focal)); 4624 4625 let kernel_path = direct_kernel_boot_path(); 4626 4627 let mut child = GuestCommand::new(&guest) 4628 .args(["--cpus", "boot=1"]) 4629 .args(["--memory", "size=512M"]) 4630 .args(["--kernel", kernel_path.to_str().unwrap()]) 4631 .args([ 4632 "--cmdline", 4633 format!("{DIRECT_KERNEL_BOOT_CMDLINE} acpi=off").as_str(), 4634 ]) 4635 .default_disks() 4636 .default_net() 4637 .capture_output() 4638 .spawn() 4639 .unwrap(); 4640 4641 let r = std::panic::catch_unwind(|| { 4642 guest.wait_vm_boot(None).unwrap(); 4643 4644 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4645 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4646 }); 4647 4648 let _ = child.kill(); 4649 let output = child.wait_with_output().unwrap(); 4650 4651 handle_child_output(r, &output); 4652 } 4653 4654 #[test] 4655 fn test_virtio_vsock() { 4656 _test_virtio_vsock(false) 4657 } 4658 4659 #[test] 4660 fn test_virtio_vsock_hotplug() { 4661 _test_virtio_vsock(true); 4662 } 4663 4664 #[test] 4665 fn test_api_http_shutdown() { 4666 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4667 let guest = Guest::new(Box::new(focal)); 4668 4669 _test_api_shutdown(TargetApi::new_http_api(&guest.tmp_dir), guest) 4670 } 4671 4672 #[test] 4673 fn test_api_http_delete() { 4674 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4675 let guest = Guest::new(Box::new(focal)); 4676 4677 _test_api_delete(TargetApi::new_http_api(&guest.tmp_dir), guest); 4678 } 4679 4680 #[test] 4681 fn test_api_http_pause_resume() { 4682 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4683 let guest = Guest::new(Box::new(focal)); 4684 4685 _test_api_pause_resume(TargetApi::new_http_api(&guest.tmp_dir), guest) 4686 } 4687 4688 #[test] 4689 fn test_api_http_create_boot() { 4690 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4691 let guest = Guest::new(Box::new(focal)); 4692 4693 _test_api_create_boot(TargetApi::new_http_api(&guest.tmp_dir), guest) 4694 } 4695 4696 #[test] 4697 fn test_virtio_iommu() { 4698 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4699 } 4700 4701 #[test] 4702 // We cannot force the software running in the guest to reprogram the BAR 4703 // with some different addresses, but we have a reliable way of testing it 4704 // with a standard Linux kernel. 4705 // By removing a device from the PCI tree, and then rescanning the tree, 4706 // Linux consistently chooses to reorganize the PCI device BARs to other 4707 // locations in the guest address space. 4708 // This test creates a dedicated PCI network device to be checked as being 4709 // properly probed first, then removing it, and adding it again by doing a 4710 // rescan. 4711 fn test_pci_bar_reprogramming() { 4712 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4713 let guest = Guest::new(Box::new(focal)); 4714 4715 #[cfg(target_arch = "x86_64")] 4716 let kernel_path = direct_kernel_boot_path(); 4717 #[cfg(target_arch = "aarch64")] 4718 let kernel_path = edk2_path(); 4719 4720 let mut child = GuestCommand::new(&guest) 4721 .args(["--cpus", "boot=1"]) 4722 .args(["--memory", "size=512M"]) 4723 .args(["--kernel", kernel_path.to_str().unwrap()]) 4724 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4725 .default_disks() 4726 .args([ 4727 "--net", 4728 guest.default_net_string().as_str(), 4729 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4730 ]) 4731 .capture_output() 4732 .spawn() 4733 .unwrap(); 4734 4735 let r = std::panic::catch_unwind(|| { 4736 guest.wait_vm_boot(None).unwrap(); 4737 4738 // 2 network interfaces + default localhost ==> 3 interfaces 4739 assert_eq!( 4740 guest 4741 .ssh_command("ip -o link | wc -l") 4742 .unwrap() 4743 .trim() 4744 .parse::<u32>() 4745 .unwrap_or_default(), 4746 3 4747 ); 4748 4749 let init_bar_addr = guest 4750 .ssh_command( 4751 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4752 ) 4753 .unwrap(); 4754 4755 // Remove the PCI device 4756 guest 4757 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4758 .unwrap(); 4759 4760 // Only 1 network interface left + default localhost ==> 2 interfaces 4761 assert_eq!( 4762 guest 4763 .ssh_command("ip -o link | wc -l") 4764 .unwrap() 4765 .trim() 4766 .parse::<u32>() 4767 .unwrap_or_default(), 4768 2 4769 ); 4770 4771 // Remove the PCI device 4772 guest 4773 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4774 .unwrap(); 4775 4776 // Back to 2 network interface + default localhost ==> 3 interfaces 4777 assert_eq!( 4778 guest 4779 .ssh_command("ip -o link | wc -l") 4780 .unwrap() 4781 .trim() 4782 .parse::<u32>() 4783 .unwrap_or_default(), 4784 3 4785 ); 4786 4787 let new_bar_addr = guest 4788 .ssh_command( 4789 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4790 ) 4791 .unwrap(); 4792 4793 // Let's compare the BAR addresses for our virtio-net device. 4794 // They should be different as we expect the BAR reprogramming 4795 // to have happened. 4796 assert_ne!(init_bar_addr, new_bar_addr); 4797 }); 4798 4799 let _ = child.kill(); 4800 let output = child.wait_with_output().unwrap(); 4801 4802 handle_child_output(r, &output); 4803 } 4804 4805 #[test] 4806 fn test_memory_mergeable_off() { 4807 test_memory_mergeable(false) 4808 } 4809 4810 #[test] 4811 #[cfg(target_arch = "x86_64")] 4812 fn test_cpu_hotplug() { 4813 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4814 let guest = Guest::new(Box::new(focal)); 4815 let api_socket = temp_api_path(&guest.tmp_dir); 4816 4817 let kernel_path = direct_kernel_boot_path(); 4818 4819 let mut child = GuestCommand::new(&guest) 4820 .args(["--cpus", "boot=2,max=4"]) 4821 .args(["--memory", "size=512M"]) 4822 .args(["--kernel", kernel_path.to_str().unwrap()]) 4823 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4824 .default_disks() 4825 .default_net() 4826 .args(["--api-socket", &api_socket]) 4827 .capture_output() 4828 .spawn() 4829 .unwrap(); 4830 4831 let r = std::panic::catch_unwind(|| { 4832 guest.wait_vm_boot(None).unwrap(); 4833 4834 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4835 4836 // Resize the VM 4837 let desired_vcpus = 4; 4838 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4839 4840 guest 4841 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4842 .unwrap(); 4843 guest 4844 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4845 .unwrap(); 4846 thread::sleep(std::time::Duration::new(10, 0)); 4847 assert_eq!( 4848 guest.get_cpu_count().unwrap_or_default(), 4849 u32::from(desired_vcpus) 4850 ); 4851 4852 guest.reboot_linux(0, None); 4853 4854 assert_eq!( 4855 guest.get_cpu_count().unwrap_or_default(), 4856 u32::from(desired_vcpus) 4857 ); 4858 4859 // Resize the VM 4860 let desired_vcpus = 2; 4861 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4862 4863 thread::sleep(std::time::Duration::new(10, 0)); 4864 assert_eq!( 4865 guest.get_cpu_count().unwrap_or_default(), 4866 u32::from(desired_vcpus) 4867 ); 4868 4869 // Resize the VM back up to 4 4870 let desired_vcpus = 4; 4871 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4872 4873 guest 4874 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4875 .unwrap(); 4876 guest 4877 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4878 .unwrap(); 4879 thread::sleep(std::time::Duration::new(10, 0)); 4880 assert_eq!( 4881 guest.get_cpu_count().unwrap_or_default(), 4882 u32::from(desired_vcpus) 4883 ); 4884 }); 4885 4886 let _ = child.kill(); 4887 let output = child.wait_with_output().unwrap(); 4888 4889 handle_child_output(r, &output); 4890 } 4891 4892 #[test] 4893 fn test_memory_hotplug() { 4894 #[cfg(target_arch = "aarch64")] 4895 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4896 #[cfg(target_arch = "x86_64")] 4897 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4898 let focal = UbuntuDiskConfig::new(focal_image); 4899 let guest = Guest::new(Box::new(focal)); 4900 let api_socket = temp_api_path(&guest.tmp_dir); 4901 4902 #[cfg(target_arch = "aarch64")] 4903 let kernel_path = edk2_path(); 4904 #[cfg(target_arch = "x86_64")] 4905 let kernel_path = direct_kernel_boot_path(); 4906 4907 let mut child = GuestCommand::new(&guest) 4908 .args(["--cpus", "boot=2,max=4"]) 4909 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4910 .args(["--kernel", kernel_path.to_str().unwrap()]) 4911 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4912 .default_disks() 4913 .default_net() 4914 .args(["--balloon", "size=0"]) 4915 .args(["--api-socket", &api_socket]) 4916 .capture_output() 4917 .spawn() 4918 .unwrap(); 4919 4920 let r = std::panic::catch_unwind(|| { 4921 guest.wait_vm_boot(None).unwrap(); 4922 4923 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4924 4925 guest.enable_memory_hotplug(); 4926 4927 // Add RAM to the VM 4928 let desired_ram = 1024 << 20; 4929 resize_command(&api_socket, None, Some(desired_ram), None, None); 4930 4931 thread::sleep(std::time::Duration::new(10, 0)); 4932 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4933 4934 // Use balloon to remove RAM from the VM 4935 let desired_balloon = 512 << 20; 4936 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4937 4938 thread::sleep(std::time::Duration::new(10, 0)); 4939 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4940 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4941 4942 guest.reboot_linux(0, None); 4943 4944 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4945 4946 // Use balloon add RAM to the VM 4947 let desired_balloon = 0; 4948 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4949 4950 thread::sleep(std::time::Duration::new(10, 0)); 4951 4952 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4953 4954 guest.enable_memory_hotplug(); 4955 4956 // Add RAM to the VM 4957 let desired_ram = 2048 << 20; 4958 resize_command(&api_socket, None, Some(desired_ram), None, None); 4959 4960 thread::sleep(std::time::Duration::new(10, 0)); 4961 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4962 4963 // Remove RAM to the VM (only applies after reboot) 4964 let desired_ram = 1024 << 20; 4965 resize_command(&api_socket, None, Some(desired_ram), None, None); 4966 4967 guest.reboot_linux(1, None); 4968 4969 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4970 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4971 }); 4972 4973 let _ = child.kill(); 4974 let output = child.wait_with_output().unwrap(); 4975 4976 handle_child_output(r, &output); 4977 } 4978 4979 #[test] 4980 #[cfg(not(feature = "mshv"))] 4981 fn test_virtio_mem() { 4982 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4983 let guest = Guest::new(Box::new(focal)); 4984 let api_socket = temp_api_path(&guest.tmp_dir); 4985 4986 let kernel_path = direct_kernel_boot_path(); 4987 4988 let mut child = GuestCommand::new(&guest) 4989 .args(["--cpus", "boot=2,max=4"]) 4990 .args([ 4991 "--memory", 4992 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4993 ]) 4994 .args(["--kernel", kernel_path.to_str().unwrap()]) 4995 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4996 .default_disks() 4997 .default_net() 4998 .args(["--api-socket", &api_socket]) 4999 .capture_output() 5000 .spawn() 5001 .unwrap(); 5002 5003 let r = std::panic::catch_unwind(|| { 5004 guest.wait_vm_boot(None).unwrap(); 5005 5006 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5007 5008 guest.enable_memory_hotplug(); 5009 5010 // Add RAM to the VM 5011 let desired_ram = 1024 << 20; 5012 resize_command(&api_socket, None, Some(desired_ram), None, None); 5013 5014 thread::sleep(std::time::Duration::new(10, 0)); 5015 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5016 5017 // Add RAM to the VM 5018 let desired_ram = 2048 << 20; 5019 resize_command(&api_socket, None, Some(desired_ram), None, None); 5020 5021 thread::sleep(std::time::Duration::new(10, 0)); 5022 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 5023 5024 // Remove RAM from the VM 5025 let desired_ram = 1024 << 20; 5026 resize_command(&api_socket, None, Some(desired_ram), None, None); 5027 5028 thread::sleep(std::time::Duration::new(10, 0)); 5029 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5030 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5031 5032 guest.reboot_linux(0, None); 5033 5034 // Check the amount of memory after reboot is 1GiB 5035 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5036 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5037 5038 // Check we can still resize to 512MiB 5039 let desired_ram = 512 << 20; 5040 resize_command(&api_socket, None, Some(desired_ram), None, None); 5041 thread::sleep(std::time::Duration::new(10, 0)); 5042 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5043 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 5044 }); 5045 5046 let _ = child.kill(); 5047 let output = child.wait_with_output().unwrap(); 5048 5049 handle_child_output(r, &output); 5050 } 5051 5052 #[test] 5053 #[cfg(target_arch = "x86_64")] 5054 #[cfg(not(feature = "mshv"))] 5055 // Test both vCPU and memory resizing together 5056 fn test_resize() { 5057 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5058 let guest = Guest::new(Box::new(focal)); 5059 let api_socket = temp_api_path(&guest.tmp_dir); 5060 5061 let kernel_path = direct_kernel_boot_path(); 5062 5063 let mut child = GuestCommand::new(&guest) 5064 .args(["--cpus", "boot=2,max=4"]) 5065 .args(["--memory", "size=512M,hotplug_size=8192M"]) 5066 .args(["--kernel", kernel_path.to_str().unwrap()]) 5067 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5068 .default_disks() 5069 .default_net() 5070 .args(["--api-socket", &api_socket]) 5071 .capture_output() 5072 .spawn() 5073 .unwrap(); 5074 5075 let r = std::panic::catch_unwind(|| { 5076 guest.wait_vm_boot(None).unwrap(); 5077 5078 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 5079 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5080 5081 guest.enable_memory_hotplug(); 5082 5083 // Resize the VM 5084 let desired_vcpus = 4; 5085 let desired_ram = 1024 << 20; 5086 resize_command( 5087 &api_socket, 5088 Some(desired_vcpus), 5089 Some(desired_ram), 5090 None, 5091 None, 5092 ); 5093 5094 guest 5095 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 5096 .unwrap(); 5097 guest 5098 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 5099 .unwrap(); 5100 thread::sleep(std::time::Duration::new(10, 0)); 5101 assert_eq!( 5102 guest.get_cpu_count().unwrap_or_default(), 5103 u32::from(desired_vcpus) 5104 ); 5105 5106 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5107 }); 5108 5109 let _ = child.kill(); 5110 let output = child.wait_with_output().unwrap(); 5111 5112 handle_child_output(r, &output); 5113 } 5114 5115 #[test] 5116 fn test_memory_overhead() { 5117 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5118 let guest = Guest::new(Box::new(focal)); 5119 5120 let kernel_path = direct_kernel_boot_path(); 5121 5122 let guest_memory_size_kb = 512 * 1024; 5123 5124 let mut child = GuestCommand::new(&guest) 5125 .args(["--cpus", "boot=1"]) 5126 .args(["--memory", format!("size={guest_memory_size_kb}K").as_str()]) 5127 .args(["--kernel", kernel_path.to_str().unwrap()]) 5128 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5129 .default_net() 5130 .default_disks() 5131 .capture_output() 5132 .spawn() 5133 .unwrap(); 5134 5135 guest.wait_vm_boot(None).unwrap(); 5136 5137 let r = std::panic::catch_unwind(|| { 5138 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 5139 eprintln!("Guest memory overhead: {overhead} vs {MAXIMUM_VMM_OVERHEAD_KB}"); 5140 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 5141 }); 5142 5143 let _ = child.kill(); 5144 let output = child.wait_with_output().unwrap(); 5145 5146 handle_child_output(r, &output); 5147 } 5148 5149 #[test] 5150 fn test_disk_hotplug() { 5151 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5152 let guest = Guest::new(Box::new(focal)); 5153 5154 #[cfg(target_arch = "x86_64")] 5155 let kernel_path = direct_kernel_boot_path(); 5156 #[cfg(target_arch = "aarch64")] 5157 let kernel_path = edk2_path(); 5158 5159 let api_socket = temp_api_path(&guest.tmp_dir); 5160 5161 let mut child = GuestCommand::new(&guest) 5162 .args(["--api-socket", &api_socket]) 5163 .args(["--cpus", "boot=1"]) 5164 .args(["--memory", "size=512M"]) 5165 .args(["--kernel", kernel_path.to_str().unwrap()]) 5166 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5167 .default_disks() 5168 .default_net() 5169 .capture_output() 5170 .spawn() 5171 .unwrap(); 5172 5173 let r = std::panic::catch_unwind(|| { 5174 guest.wait_vm_boot(None).unwrap(); 5175 5176 // Check /dev/vdc is not there 5177 assert_eq!( 5178 guest 5179 .ssh_command("lsblk | grep -c vdc.*16M || true") 5180 .unwrap() 5181 .trim() 5182 .parse::<u32>() 5183 .unwrap_or(1), 5184 0 5185 ); 5186 5187 // Now let's add the extra disk. 5188 let mut blk_file_path = dirs::home_dir().unwrap(); 5189 blk_file_path.push("workloads"); 5190 blk_file_path.push("blk.img"); 5191 let (cmd_success, cmd_output) = remote_command_w_output( 5192 &api_socket, 5193 "add-disk", 5194 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5195 ); 5196 assert!(cmd_success); 5197 assert!(String::from_utf8_lossy(&cmd_output) 5198 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5199 5200 thread::sleep(std::time::Duration::new(10, 0)); 5201 5202 // Check that /dev/vdc exists and the block size is 16M. 5203 assert_eq!( 5204 guest 5205 .ssh_command("lsblk | grep vdc | grep -c 16M") 5206 .unwrap() 5207 .trim() 5208 .parse::<u32>() 5209 .unwrap_or_default(), 5210 1 5211 ); 5212 // And check the block device can be read. 5213 guest 5214 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5215 .unwrap(); 5216 5217 // Let's remove it the extra disk. 5218 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5219 thread::sleep(std::time::Duration::new(5, 0)); 5220 // And check /dev/vdc is not there 5221 assert_eq!( 5222 guest 5223 .ssh_command("lsblk | grep -c vdc.*16M || true") 5224 .unwrap() 5225 .trim() 5226 .parse::<u32>() 5227 .unwrap_or(1), 5228 0 5229 ); 5230 5231 // And add it back to validate unplug did work correctly. 5232 let (cmd_success, cmd_output) = remote_command_w_output( 5233 &api_socket, 5234 "add-disk", 5235 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5236 ); 5237 assert!(cmd_success); 5238 assert!(String::from_utf8_lossy(&cmd_output) 5239 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5240 5241 thread::sleep(std::time::Duration::new(10, 0)); 5242 5243 // Check that /dev/vdc exists and the block size is 16M. 5244 assert_eq!( 5245 guest 5246 .ssh_command("lsblk | grep vdc | grep -c 16M") 5247 .unwrap() 5248 .trim() 5249 .parse::<u32>() 5250 .unwrap_or_default(), 5251 1 5252 ); 5253 // And check the block device can be read. 5254 guest 5255 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5256 .unwrap(); 5257 5258 // Reboot the VM. 5259 guest.reboot_linux(0, None); 5260 5261 // Check still there after reboot 5262 assert_eq!( 5263 guest 5264 .ssh_command("lsblk | grep vdc | grep -c 16M") 5265 .unwrap() 5266 .trim() 5267 .parse::<u32>() 5268 .unwrap_or_default(), 5269 1 5270 ); 5271 5272 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5273 5274 thread::sleep(std::time::Duration::new(20, 0)); 5275 5276 // Check device has gone away 5277 assert_eq!( 5278 guest 5279 .ssh_command("lsblk | grep -c vdc.*16M || true") 5280 .unwrap() 5281 .trim() 5282 .parse::<u32>() 5283 .unwrap_or(1), 5284 0 5285 ); 5286 5287 guest.reboot_linux(1, None); 5288 5289 // Check device still absent 5290 assert_eq!( 5291 guest 5292 .ssh_command("lsblk | grep -c vdc.*16M || true") 5293 .unwrap() 5294 .trim() 5295 .parse::<u32>() 5296 .unwrap_or(1), 5297 0 5298 ); 5299 }); 5300 5301 let _ = child.kill(); 5302 let output = child.wait_with_output().unwrap(); 5303 5304 handle_child_output(r, &output); 5305 } 5306 5307 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 5308 const LOOP_CONFIGURE: u64 = 0x4c0a; 5309 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 5310 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 5311 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 5312 5313 #[repr(C)] 5314 struct LoopInfo64 { 5315 lo_device: u64, 5316 lo_inode: u64, 5317 lo_rdevice: u64, 5318 lo_offset: u64, 5319 lo_sizelimit: u64, 5320 lo_number: u32, 5321 lo_encrypt_type: u32, 5322 lo_encrypt_key_size: u32, 5323 lo_flags: u32, 5324 lo_file_name: [u8; 64], 5325 lo_crypt_name: [u8; 64], 5326 lo_encrypt_key: [u8; 32], 5327 lo_init: [u64; 2], 5328 } 5329 5330 impl Default for LoopInfo64 { 5331 fn default() -> Self { 5332 LoopInfo64 { 5333 lo_device: 0, 5334 lo_inode: 0, 5335 lo_rdevice: 0, 5336 lo_offset: 0, 5337 lo_sizelimit: 0, 5338 lo_number: 0, 5339 lo_encrypt_type: 0, 5340 lo_encrypt_key_size: 0, 5341 lo_flags: 0, 5342 lo_file_name: [0; 64], 5343 lo_crypt_name: [0; 64], 5344 lo_encrypt_key: [0; 32], 5345 lo_init: [0; 2], 5346 } 5347 } 5348 } 5349 5350 #[derive(Default)] 5351 #[repr(C)] 5352 struct LoopConfig { 5353 fd: u32, 5354 block_size: u32, 5355 info: LoopInfo64, 5356 _reserved: [u64; 8], 5357 } 5358 5359 // Open loop-control device 5360 let loop_ctl_file = OpenOptions::new() 5361 .read(true) 5362 .write(true) 5363 .open(LOOP_CTL_PATH) 5364 .unwrap(); 5365 5366 // Request a free loop device 5367 let loop_device_number = 5368 unsafe { libc::ioctl(loop_ctl_file.as_raw_fd(), LOOP_CTL_GET_FREE as _) }; 5369 5370 if loop_device_number < 0 { 5371 panic!("Couldn't find a free loop device"); 5372 } 5373 5374 // Create loop device path 5375 let loop_device_path = format!("{LOOP_DEVICE_PREFIX}{loop_device_number}"); 5376 5377 // Open loop device 5378 let loop_device_file = OpenOptions::new() 5379 .read(true) 5380 .write(true) 5381 .open(&loop_device_path) 5382 .unwrap(); 5383 5384 // Open backing file 5385 let backing_file = OpenOptions::new() 5386 .read(true) 5387 .write(true) 5388 .open(backing_file_path) 5389 .unwrap(); 5390 5391 let loop_config = LoopConfig { 5392 fd: backing_file.as_raw_fd() as u32, 5393 block_size, 5394 ..Default::default() 5395 }; 5396 5397 for i in 0..num_retries { 5398 let ret = unsafe { 5399 libc::ioctl( 5400 loop_device_file.as_raw_fd(), 5401 LOOP_CONFIGURE as _, 5402 &loop_config, 5403 ) 5404 }; 5405 if ret != 0 { 5406 if i < num_retries - 1 { 5407 println!( 5408 "Iteration {}: Failed to configure the loop device {}: {}", 5409 i, 5410 loop_device_path, 5411 std::io::Error::last_os_error() 5412 ); 5413 } else { 5414 panic!( 5415 "Failed {} times trying to configure the loop device {}: {}", 5416 num_retries, 5417 loop_device_path, 5418 std::io::Error::last_os_error() 5419 ); 5420 } 5421 } else { 5422 break; 5423 } 5424 5425 // Wait for a bit before retrying 5426 thread::sleep(std::time::Duration::new(5, 0)); 5427 } 5428 5429 loop_device_path 5430 } 5431 5432 #[test] 5433 fn test_virtio_block_topology() { 5434 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5435 let guest = Guest::new(Box::new(focal)); 5436 5437 let kernel_path = direct_kernel_boot_path(); 5438 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5439 5440 let output = exec_host_command_output( 5441 format!( 5442 "qemu-img create -f raw {} 16M", 5443 test_disk_path.to_str().unwrap() 5444 ) 5445 .as_str(), 5446 ); 5447 if !output.status.success() { 5448 let stdout = String::from_utf8_lossy(&output.stdout); 5449 let stderr = String::from_utf8_lossy(&output.stderr); 5450 panic!("qemu-img command failed\nstdout\n{stdout}\nstderr\n{stderr}"); 5451 } 5452 5453 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5454 5455 let mut child = GuestCommand::new(&guest) 5456 .args(["--cpus", "boot=1"]) 5457 .args(["--memory", "size=512M"]) 5458 .args(["--kernel", kernel_path.to_str().unwrap()]) 5459 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5460 .args([ 5461 "--disk", 5462 format!( 5463 "path={}", 5464 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5465 ) 5466 .as_str(), 5467 format!( 5468 "path={}", 5469 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5470 ) 5471 .as_str(), 5472 format!("path={}", &loop_dev).as_str(), 5473 ]) 5474 .default_net() 5475 .capture_output() 5476 .spawn() 5477 .unwrap(); 5478 5479 let r = std::panic::catch_unwind(|| { 5480 guest.wait_vm_boot(None).unwrap(); 5481 5482 // MIN-IO column 5483 assert_eq!( 5484 guest 5485 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5486 .unwrap() 5487 .trim() 5488 .parse::<u32>() 5489 .unwrap_or_default(), 5490 4096 5491 ); 5492 // PHY-SEC column 5493 assert_eq!( 5494 guest 5495 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5496 .unwrap() 5497 .trim() 5498 .parse::<u32>() 5499 .unwrap_or_default(), 5500 4096 5501 ); 5502 // LOG-SEC column 5503 assert_eq!( 5504 guest 5505 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5506 .unwrap() 5507 .trim() 5508 .parse::<u32>() 5509 .unwrap_or_default(), 5510 4096 5511 ); 5512 }); 5513 5514 let _ = child.kill(); 5515 let output = child.wait_with_output().unwrap(); 5516 5517 handle_child_output(r, &output); 5518 5519 Command::new("losetup") 5520 .args(["-d", &loop_dev]) 5521 .output() 5522 .expect("loop device not found"); 5523 } 5524 5525 #[test] 5526 fn test_virtio_balloon_deflate_on_oom() { 5527 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5528 let guest = Guest::new(Box::new(focal)); 5529 5530 let kernel_path = direct_kernel_boot_path(); 5531 5532 let api_socket = temp_api_path(&guest.tmp_dir); 5533 5534 //Let's start a 4G guest with balloon occupied 2G memory 5535 let mut child = GuestCommand::new(&guest) 5536 .args(["--api-socket", &api_socket]) 5537 .args(["--cpus", "boot=1"]) 5538 .args(["--memory", "size=4G"]) 5539 .args(["--kernel", kernel_path.to_str().unwrap()]) 5540 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5541 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5542 .default_disks() 5543 .default_net() 5544 .capture_output() 5545 .spawn() 5546 .unwrap(); 5547 5548 let r = std::panic::catch_unwind(|| { 5549 guest.wait_vm_boot(None).unwrap(); 5550 5551 // Wait for balloon memory's initialization and check its size. 5552 // The virtio-balloon driver might take a few seconds to report the 5553 // balloon effective size back to the VMM. 5554 thread::sleep(std::time::Duration::new(20, 0)); 5555 5556 let orig_balloon = balloon_size(&api_socket); 5557 println!("The original balloon memory size is {orig_balloon} bytes"); 5558 assert!(orig_balloon == 2147483648); 5559 5560 // Two steps to verify if the 'deflate_on_oom' parameter works. 5561 // 1st: run a command to trigger an OOM in the guest. 5562 guest 5563 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5564 .unwrap(); 5565 5566 // Give some time for the OOM to happen in the guest and be reported 5567 // back to the host. 5568 thread::sleep(std::time::Duration::new(20, 0)); 5569 5570 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5571 let deflated_balloon = balloon_size(&api_socket); 5572 println!("After deflating, balloon memory size is {deflated_balloon} bytes"); 5573 // Verify the balloon size deflated 5574 assert!(deflated_balloon < 2147483648); 5575 }); 5576 5577 let _ = child.kill(); 5578 let output = child.wait_with_output().unwrap(); 5579 5580 handle_child_output(r, &output); 5581 } 5582 5583 #[test] 5584 #[cfg(not(feature = "mshv"))] 5585 fn test_virtio_balloon_free_page_reporting() { 5586 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5587 let guest = Guest::new(Box::new(focal)); 5588 5589 //Let's start a 4G guest with balloon occupied 2G memory 5590 let mut child = GuestCommand::new(&guest) 5591 .args(["--cpus", "boot=1"]) 5592 .args(["--memory", "size=4G"]) 5593 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5594 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5595 .args(["--balloon", "size=0,free_page_reporting=on"]) 5596 .default_disks() 5597 .default_net() 5598 .capture_output() 5599 .spawn() 5600 .unwrap(); 5601 5602 let pid = child.id(); 5603 let r = std::panic::catch_unwind(|| { 5604 guest.wait_vm_boot(None).unwrap(); 5605 5606 // Check the initial RSS is less than 1GiB 5607 let rss = process_rss_kib(pid); 5608 println!("RSS {rss} < 1048576"); 5609 assert!(rss < 1048576); 5610 5611 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5612 // seconds 5613 let guest_ip = guest.network.guest_ip.clone(); 5614 thread::spawn(move || { 5615 ssh_command_ip( 5616 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5617 &guest_ip, 5618 DEFAULT_SSH_RETRIES, 5619 DEFAULT_SSH_TIMEOUT, 5620 ) 5621 .unwrap(); 5622 }); 5623 5624 // Wait for 50 seconds to make sure the stress command is consuming 5625 // the expected amount of memory. 5626 thread::sleep(std::time::Duration::new(50, 0)); 5627 let rss = process_rss_kib(pid); 5628 println!("RSS {rss} >= 2097152"); 5629 assert!(rss >= 2097152); 5630 5631 // Wait for an extra minute to make sure the stress command has 5632 // completed and that the guest reported the free pages to the VMM 5633 // through the virtio-balloon device. We expect the RSS to be under 5634 // 2GiB. 5635 thread::sleep(std::time::Duration::new(60, 0)); 5636 let rss = process_rss_kib(pid); 5637 println!("RSS {rss} < 2097152"); 5638 assert!(rss < 2097152); 5639 }); 5640 5641 let _ = child.kill(); 5642 let output = child.wait_with_output().unwrap(); 5643 5644 handle_child_output(r, &output); 5645 } 5646 5647 #[test] 5648 fn test_pmem_hotplug() { 5649 _test_pmem_hotplug(None) 5650 } 5651 5652 #[test] 5653 fn test_pmem_multi_segment_hotplug() { 5654 _test_pmem_hotplug(Some(15)) 5655 } 5656 5657 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5658 #[cfg(target_arch = "aarch64")] 5659 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5660 #[cfg(target_arch = "x86_64")] 5661 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5662 let focal = UbuntuDiskConfig::new(focal_image); 5663 let guest = Guest::new(Box::new(focal)); 5664 5665 #[cfg(target_arch = "x86_64")] 5666 let kernel_path = direct_kernel_boot_path(); 5667 #[cfg(target_arch = "aarch64")] 5668 let kernel_path = edk2_path(); 5669 5670 let api_socket = temp_api_path(&guest.tmp_dir); 5671 5672 let mut cmd = GuestCommand::new(&guest); 5673 5674 cmd.args(["--api-socket", &api_socket]) 5675 .args(["--cpus", "boot=1"]) 5676 .args(["--memory", "size=512M"]) 5677 .args(["--kernel", kernel_path.to_str().unwrap()]) 5678 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5679 .default_disks() 5680 .default_net() 5681 .capture_output(); 5682 5683 if pci_segment.is_some() { 5684 cmd.args([ 5685 "--platform", 5686 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5687 ]); 5688 } 5689 5690 let mut child = cmd.spawn().unwrap(); 5691 5692 let r = std::panic::catch_unwind(|| { 5693 guest.wait_vm_boot(None).unwrap(); 5694 5695 // Check /dev/pmem0 is not there 5696 assert_eq!( 5697 guest 5698 .ssh_command("lsblk | grep -c pmem0 || true") 5699 .unwrap() 5700 .trim() 5701 .parse::<u32>() 5702 .unwrap_or(1), 5703 0 5704 ); 5705 5706 let pmem_temp_file = TempFile::new().unwrap(); 5707 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5708 let (cmd_success, cmd_output) = remote_command_w_output( 5709 &api_socket, 5710 "add-pmem", 5711 Some(&format!( 5712 "file={},id=test0{}", 5713 pmem_temp_file.as_path().to_str().unwrap(), 5714 if let Some(pci_segment) = pci_segment { 5715 format!(",pci_segment={pci_segment}") 5716 } else { 5717 "".to_owned() 5718 } 5719 )), 5720 ); 5721 assert!(cmd_success); 5722 if let Some(pci_segment) = pci_segment { 5723 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5724 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5725 ))); 5726 } else { 5727 assert!(String::from_utf8_lossy(&cmd_output) 5728 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5729 } 5730 5731 // Check that /dev/pmem0 exists and the block size is 128M 5732 assert_eq!( 5733 guest 5734 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5735 .unwrap() 5736 .trim() 5737 .parse::<u32>() 5738 .unwrap_or_default(), 5739 1 5740 ); 5741 5742 guest.reboot_linux(0, None); 5743 5744 // Check still there after reboot 5745 assert_eq!( 5746 guest 5747 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5748 .unwrap() 5749 .trim() 5750 .parse::<u32>() 5751 .unwrap_or_default(), 5752 1 5753 ); 5754 5755 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5756 5757 thread::sleep(std::time::Duration::new(20, 0)); 5758 5759 // Check device has gone away 5760 assert_eq!( 5761 guest 5762 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5763 .unwrap() 5764 .trim() 5765 .parse::<u32>() 5766 .unwrap_or(1), 5767 0 5768 ); 5769 5770 guest.reboot_linux(1, None); 5771 5772 // Check still absent after reboot 5773 assert_eq!( 5774 guest 5775 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5776 .unwrap() 5777 .trim() 5778 .parse::<u32>() 5779 .unwrap_or(1), 5780 0 5781 ); 5782 }); 5783 5784 let _ = child.kill(); 5785 let output = child.wait_with_output().unwrap(); 5786 5787 handle_child_output(r, &output); 5788 } 5789 5790 #[test] 5791 fn test_net_hotplug() { 5792 _test_net_hotplug(None) 5793 } 5794 5795 #[test] 5796 fn test_net_multi_segment_hotplug() { 5797 _test_net_hotplug(Some(15)) 5798 } 5799 5800 fn _test_net_hotplug(pci_segment: Option<u16>) { 5801 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5802 let guest = Guest::new(Box::new(focal)); 5803 5804 #[cfg(target_arch = "x86_64")] 5805 let kernel_path = direct_kernel_boot_path(); 5806 #[cfg(target_arch = "aarch64")] 5807 let kernel_path = edk2_path(); 5808 5809 let api_socket = temp_api_path(&guest.tmp_dir); 5810 5811 // Boot without network 5812 let mut cmd = GuestCommand::new(&guest); 5813 5814 cmd.args(["--api-socket", &api_socket]) 5815 .args(["--cpus", "boot=1"]) 5816 .args(["--memory", "size=512M"]) 5817 .args(["--kernel", kernel_path.to_str().unwrap()]) 5818 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5819 .default_disks() 5820 .capture_output(); 5821 5822 if pci_segment.is_some() { 5823 cmd.args([ 5824 "--platform", 5825 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5826 ]); 5827 } 5828 5829 let mut child = cmd.spawn().unwrap(); 5830 5831 thread::sleep(std::time::Duration::new(20, 0)); 5832 5833 let r = std::panic::catch_unwind(|| { 5834 // Add network 5835 let (cmd_success, cmd_output) = remote_command_w_output( 5836 &api_socket, 5837 "add-net", 5838 Some( 5839 format!( 5840 "{}{},id=test0", 5841 guest.default_net_string(), 5842 if let Some(pci_segment) = pci_segment { 5843 format!(",pci_segment={pci_segment}") 5844 } else { 5845 "".to_owned() 5846 } 5847 ) 5848 .as_str(), 5849 ), 5850 ); 5851 assert!(cmd_success); 5852 5853 if let Some(pci_segment) = pci_segment { 5854 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5855 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5856 ))); 5857 } else { 5858 assert!(String::from_utf8_lossy(&cmd_output) 5859 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5860 } 5861 5862 thread::sleep(std::time::Duration::new(5, 0)); 5863 5864 // 1 network interfaces + default localhost ==> 2 interfaces 5865 assert_eq!( 5866 guest 5867 .ssh_command("ip -o link | wc -l") 5868 .unwrap() 5869 .trim() 5870 .parse::<u32>() 5871 .unwrap_or_default(), 5872 2 5873 ); 5874 5875 // Remove network 5876 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5877 thread::sleep(std::time::Duration::new(5, 0)); 5878 5879 let (cmd_success, cmd_output) = remote_command_w_output( 5880 &api_socket, 5881 "add-net", 5882 Some( 5883 format!( 5884 "{}{},id=test1", 5885 guest.default_net_string(), 5886 if let Some(pci_segment) = pci_segment { 5887 format!(",pci_segment={pci_segment}") 5888 } else { 5889 "".to_owned() 5890 } 5891 ) 5892 .as_str(), 5893 ), 5894 ); 5895 assert!(cmd_success); 5896 5897 if let Some(pci_segment) = pci_segment { 5898 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5899 "{{\"id\":\"test1\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5900 ))); 5901 } else { 5902 assert!(String::from_utf8_lossy(&cmd_output) 5903 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5904 } 5905 5906 thread::sleep(std::time::Duration::new(5, 0)); 5907 5908 // 1 network interfaces + default localhost ==> 2 interfaces 5909 assert_eq!( 5910 guest 5911 .ssh_command("ip -o link | wc -l") 5912 .unwrap() 5913 .trim() 5914 .parse::<u32>() 5915 .unwrap_or_default(), 5916 2 5917 ); 5918 5919 guest.reboot_linux(0, None); 5920 5921 // Check still there after reboot 5922 // 1 network interfaces + default localhost ==> 2 interfaces 5923 assert_eq!( 5924 guest 5925 .ssh_command("ip -o link | wc -l") 5926 .unwrap() 5927 .trim() 5928 .parse::<u32>() 5929 .unwrap_or_default(), 5930 2 5931 ); 5932 }); 5933 5934 let _ = child.kill(); 5935 let output = child.wait_with_output().unwrap(); 5936 5937 handle_child_output(r, &output); 5938 } 5939 5940 #[test] 5941 fn test_initramfs() { 5942 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5943 let guest = Guest::new(Box::new(focal)); 5944 let mut workload_path = dirs::home_dir().unwrap(); 5945 workload_path.push("workloads"); 5946 5947 #[cfg(target_arch = "x86_64")] 5948 let mut kernels = vec![direct_kernel_boot_path()]; 5949 #[cfg(target_arch = "aarch64")] 5950 let kernels = [direct_kernel_boot_path()]; 5951 5952 #[cfg(target_arch = "x86_64")] 5953 { 5954 let mut pvh_kernel_path = workload_path.clone(); 5955 pvh_kernel_path.push("vmlinux"); 5956 kernels.push(pvh_kernel_path); 5957 } 5958 5959 let mut initramfs_path = workload_path; 5960 initramfs_path.push("alpine_initramfs.img"); 5961 5962 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5963 let cmdline = format!("console=hvc0 quiet TEST_STRING={test_string}"); 5964 5965 kernels.iter().for_each(|k_path| { 5966 let mut child = GuestCommand::new(&guest) 5967 .args(["--kernel", k_path.to_str().unwrap()]) 5968 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 5969 .args(["--cmdline", &cmdline]) 5970 .capture_output() 5971 .spawn() 5972 .unwrap(); 5973 5974 thread::sleep(std::time::Duration::new(20, 0)); 5975 5976 let _ = child.kill(); 5977 let output = child.wait_with_output().unwrap(); 5978 5979 let r = std::panic::catch_unwind(|| { 5980 let s = String::from_utf8_lossy(&output.stdout); 5981 5982 assert_ne!(s.lines().position(|line| line == test_string), None); 5983 }); 5984 5985 handle_child_output(r, &output); 5986 }); 5987 } 5988 5989 // One thing to note about this test. The virtio-net device is heavily used 5990 // through each ssh command. There's no need to perform a dedicated test to 5991 // verify the migration went well for virtio-net. 5992 #[test] 5993 #[cfg(not(feature = "mshv"))] 5994 fn test_snapshot_restore_hotplug_virtiomem() { 5995 _test_snapshot_restore(true); 5996 } 5997 5998 #[test] 5999 fn test_snapshot_restore_basic() { 6000 _test_snapshot_restore(false); 6001 } 6002 6003 fn _test_snapshot_restore(use_hotplug: bool) { 6004 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6005 let guest = Guest::new(Box::new(focal)); 6006 let kernel_path = direct_kernel_boot_path(); 6007 6008 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 6009 6010 let net_id = "net123"; 6011 let net_params = format!( 6012 "id={},tap=,mac={},ip={},mask=255.255.255.0", 6013 net_id, guest.network.guest_mac, guest.network.host_ip 6014 ); 6015 let mut mem_params = "size=4G"; 6016 6017 if use_hotplug { 6018 mem_params = "size=4G,hotplug_method=virtio-mem,hotplug_size=32G" 6019 } 6020 6021 let cloudinit_params = format!( 6022 "path={},iommu=on", 6023 guest.disk_config.disk(DiskType::CloudInit).unwrap() 6024 ); 6025 6026 let socket = temp_vsock_path(&guest.tmp_dir); 6027 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6028 6029 let mut child = GuestCommand::new(&guest) 6030 .args(["--api-socket", &api_socket_source]) 6031 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6032 .args(["--cpus", "boot=4"]) 6033 .args(["--memory", mem_params]) 6034 .args(["--balloon", "size=0"]) 6035 .args(["--kernel", kernel_path.to_str().unwrap()]) 6036 .args([ 6037 "--disk", 6038 format!( 6039 "path={}", 6040 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 6041 ) 6042 .as_str(), 6043 cloudinit_params.as_str(), 6044 ]) 6045 .args(["--net", net_params.as_str()]) 6046 .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) 6047 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6048 .capture_output() 6049 .spawn() 6050 .unwrap(); 6051 6052 let console_text = String::from("On a branch floating down river a cricket, singing."); 6053 // Create the snapshot directory 6054 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 6055 6056 let r = std::panic::catch_unwind(|| { 6057 guest.wait_vm_boot(None).unwrap(); 6058 6059 // Check the number of vCPUs 6060 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 6061 // Check the guest RAM 6062 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6063 if use_hotplug { 6064 // Increase guest RAM with virtio-mem 6065 resize_command( 6066 &api_socket_source, 6067 None, 6068 Some(6 << 30), 6069 None, 6070 Some(&event_path), 6071 ); 6072 thread::sleep(std::time::Duration::new(5, 0)); 6073 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6074 // Use balloon to remove RAM from the VM 6075 resize_command( 6076 &api_socket_source, 6077 None, 6078 None, 6079 Some(1 << 30), 6080 Some(&event_path), 6081 ); 6082 thread::sleep(std::time::Duration::new(5, 0)); 6083 let total_memory = guest.get_total_memory().unwrap_or_default(); 6084 assert!(total_memory > 4_800_000); 6085 assert!(total_memory < 5_760_000); 6086 } 6087 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 6088 guest.check_devices_common(Some(&socket), Some(&console_text), None); 6089 6090 // x86_64: We check that removing and adding back the virtio-net device 6091 // does not break the snapshot/restore support for virtio-pci. 6092 // This is an important thing to test as the hotplug will 6093 // trigger a PCI BAR reprogramming, which is a good way of 6094 // checking if the stored resources are correctly restored. 6095 // Unplug the virtio-net device 6096 // AArch64: Device hotplug is currently not supported, skipping here. 6097 #[cfg(target_arch = "x86_64")] 6098 { 6099 assert!(remote_command( 6100 &api_socket_source, 6101 "remove-device", 6102 Some(net_id), 6103 )); 6104 thread::sleep(std::time::Duration::new(10, 0)); 6105 let latest_events = [&MetaEvent { 6106 event: "device-removed".to_string(), 6107 device_id: Some(net_id.to_string()), 6108 }]; 6109 // See: #5938 6110 thread::sleep(std::time::Duration::new(1, 0)); 6111 assert!(check_latest_events_exact(&latest_events, &event_path)); 6112 6113 // Plug the virtio-net device again 6114 assert!(remote_command( 6115 &api_socket_source, 6116 "add-net", 6117 Some(net_params.as_str()), 6118 )); 6119 thread::sleep(std::time::Duration::new(10, 0)); 6120 } 6121 6122 // Pause the VM 6123 assert!(remote_command(&api_socket_source, "pause", None)); 6124 let latest_events = [ 6125 &MetaEvent { 6126 event: "pausing".to_string(), 6127 device_id: None, 6128 }, 6129 &MetaEvent { 6130 event: "paused".to_string(), 6131 device_id: None, 6132 }, 6133 ]; 6134 // See: #5938 6135 thread::sleep(std::time::Duration::new(1, 0)); 6136 assert!(check_latest_events_exact(&latest_events, &event_path)); 6137 6138 // Take a snapshot from the VM 6139 assert!(remote_command( 6140 &api_socket_source, 6141 "snapshot", 6142 Some(format!("file://{snapshot_dir}").as_str()), 6143 )); 6144 6145 // Wait to make sure the snapshot is completed 6146 thread::sleep(std::time::Duration::new(10, 0)); 6147 6148 let latest_events = [ 6149 &MetaEvent { 6150 event: "snapshotting".to_string(), 6151 device_id: None, 6152 }, 6153 &MetaEvent { 6154 event: "snapshotted".to_string(), 6155 device_id: None, 6156 }, 6157 ]; 6158 // See: #5938 6159 thread::sleep(std::time::Duration::new(1, 0)); 6160 assert!(check_latest_events_exact(&latest_events, &event_path)); 6161 }); 6162 6163 // Shutdown the source VM and check console output 6164 let _ = child.kill(); 6165 let output = child.wait_with_output().unwrap(); 6166 handle_child_output(r, &output); 6167 6168 let r = std::panic::catch_unwind(|| { 6169 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 6170 }); 6171 6172 handle_child_output(r, &output); 6173 6174 // Remove the vsock socket file. 6175 Command::new("rm") 6176 .arg("-f") 6177 .arg(socket.as_str()) 6178 .output() 6179 .unwrap(); 6180 6181 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 6182 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 6183 6184 // Restore the VM from the snapshot 6185 let mut child = GuestCommand::new(&guest) 6186 .args(["--api-socket", &api_socket_restored]) 6187 .args([ 6188 "--event-monitor", 6189 format!("path={event_path_restored}").as_str(), 6190 ]) 6191 .args([ 6192 "--restore", 6193 format!("source_url=file://{snapshot_dir}").as_str(), 6194 ]) 6195 .capture_output() 6196 .spawn() 6197 .unwrap(); 6198 6199 // Wait for the VM to be restored 6200 thread::sleep(std::time::Duration::new(20, 0)); 6201 let expected_events = [ 6202 &MetaEvent { 6203 event: "starting".to_string(), 6204 device_id: None, 6205 }, 6206 &MetaEvent { 6207 event: "activated".to_string(), 6208 device_id: Some("__console".to_string()), 6209 }, 6210 &MetaEvent { 6211 event: "activated".to_string(), 6212 device_id: Some("__rng".to_string()), 6213 }, 6214 &MetaEvent { 6215 event: "restoring".to_string(), 6216 device_id: None, 6217 }, 6218 ]; 6219 assert!(check_sequential_events( 6220 &expected_events, 6221 &event_path_restored 6222 )); 6223 let latest_events = [&MetaEvent { 6224 event: "restored".to_string(), 6225 device_id: None, 6226 }]; 6227 assert!(check_latest_events_exact( 6228 &latest_events, 6229 &event_path_restored 6230 )); 6231 6232 let r = std::panic::catch_unwind(|| { 6233 // Resume the VM 6234 assert!(remote_command(&api_socket_restored, "resume", None)); 6235 // There is no way that we can ensure the 'write()' to the 6236 // event file is completed when the 'resume' request is 6237 // returned successfully, because the 'write()' was done 6238 // asynchronously from a different thread of Cloud 6239 // Hypervisor (e.g. the event-monitor thread). 6240 thread::sleep(std::time::Duration::new(1, 0)); 6241 let latest_events = [ 6242 &MetaEvent { 6243 event: "resuming".to_string(), 6244 device_id: None, 6245 }, 6246 &MetaEvent { 6247 event: "resumed".to_string(), 6248 device_id: None, 6249 }, 6250 ]; 6251 assert!(check_latest_events_exact( 6252 &latest_events, 6253 &event_path_restored 6254 )); 6255 6256 // Perform same checks to validate VM has been properly restored 6257 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 6258 let total_memory = guest.get_total_memory().unwrap_or_default(); 6259 if !use_hotplug { 6260 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6261 } else { 6262 assert!(total_memory > 4_800_000); 6263 assert!(total_memory < 5_760_000); 6264 // Deflate balloon to restore entire RAM to the VM 6265 resize_command(&api_socket_restored, None, None, Some(0), None); 6266 thread::sleep(std::time::Duration::new(5, 0)); 6267 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6268 // Decrease guest RAM with virtio-mem 6269 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 6270 thread::sleep(std::time::Duration::new(5, 0)); 6271 let total_memory = guest.get_total_memory().unwrap_or_default(); 6272 assert!(total_memory > 4_800_000); 6273 assert!(total_memory < 5_760_000); 6274 } 6275 6276 guest.check_devices_common(Some(&socket), Some(&console_text), None); 6277 }); 6278 // Shutdown the target VM and check console output 6279 let _ = child.kill(); 6280 let output = child.wait_with_output().unwrap(); 6281 handle_child_output(r, &output); 6282 6283 let r = std::panic::catch_unwind(|| { 6284 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 6285 }); 6286 6287 handle_child_output(r, &output); 6288 } 6289 6290 #[test] 6291 fn test_counters() { 6292 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6293 let guest = Guest::new(Box::new(focal)); 6294 let api_socket = temp_api_path(&guest.tmp_dir); 6295 6296 let mut cmd = GuestCommand::new(&guest); 6297 cmd.args(["--cpus", "boot=1"]) 6298 .args(["--memory", "size=512M"]) 6299 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 6300 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6301 .default_disks() 6302 .args(["--net", guest.default_net_string().as_str()]) 6303 .args(["--api-socket", &api_socket]) 6304 .capture_output(); 6305 6306 let mut child = cmd.spawn().unwrap(); 6307 6308 let r = std::panic::catch_unwind(|| { 6309 guest.wait_vm_boot(None).unwrap(); 6310 6311 let orig_counters = get_counters(&api_socket); 6312 guest 6313 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 6314 .unwrap(); 6315 6316 let new_counters = get_counters(&api_socket); 6317 6318 // Check that all the counters have increased 6319 assert!(new_counters > orig_counters); 6320 }); 6321 6322 let _ = child.kill(); 6323 let output = child.wait_with_output().unwrap(); 6324 6325 handle_child_output(r, &output); 6326 } 6327 6328 #[test] 6329 #[cfg(feature = "guest_debug")] 6330 fn test_coredump() { 6331 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6332 let guest = Guest::new(Box::new(focal)); 6333 let api_socket = temp_api_path(&guest.tmp_dir); 6334 6335 let mut cmd = GuestCommand::new(&guest); 6336 cmd.args(["--cpus", "boot=4"]) 6337 .args(["--memory", "size=4G"]) 6338 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6339 .default_disks() 6340 .args(["--net", guest.default_net_string().as_str()]) 6341 .args(["--api-socket", &api_socket]) 6342 .capture_output(); 6343 6344 let mut child = cmd.spawn().unwrap(); 6345 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6346 6347 let r = std::panic::catch_unwind(|| { 6348 guest.wait_vm_boot(None).unwrap(); 6349 6350 assert!(remote_command(&api_socket, "pause", None)); 6351 6352 assert!(remote_command( 6353 &api_socket, 6354 "coredump", 6355 Some(format!("file://{vmcore_file}").as_str()), 6356 )); 6357 6358 // the num of CORE notes should equals to vcpu 6359 let readelf_core_num_cmd = 6360 format!("readelf --all {vmcore_file} |grep CORE |grep -v Type |wc -l"); 6361 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 6362 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 6363 6364 // the num of QEMU notes should equals to vcpu 6365 let readelf_vmm_num_cmd = format!("readelf --all {vmcore_file} |grep QEMU |wc -l"); 6366 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 6367 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 6368 }); 6369 6370 let _ = child.kill(); 6371 let output = child.wait_with_output().unwrap(); 6372 6373 handle_child_output(r, &output); 6374 } 6375 6376 #[test] 6377 #[cfg(feature = "guest_debug")] 6378 fn test_coredump_no_pause() { 6379 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6380 let guest = Guest::new(Box::new(focal)); 6381 let api_socket = temp_api_path(&guest.tmp_dir); 6382 6383 let mut cmd = GuestCommand::new(&guest); 6384 cmd.args(["--cpus", "boot=4"]) 6385 .args(["--memory", "size=4G"]) 6386 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6387 .default_disks() 6388 .args(["--net", guest.default_net_string().as_str()]) 6389 .args(["--api-socket", &api_socket]) 6390 .capture_output(); 6391 6392 let mut child = cmd.spawn().unwrap(); 6393 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6394 6395 let r = std::panic::catch_unwind(|| { 6396 guest.wait_vm_boot(None).unwrap(); 6397 6398 assert!(remote_command( 6399 &api_socket, 6400 "coredump", 6401 Some(format!("file://{vmcore_file}").as_str()), 6402 )); 6403 6404 assert_eq!(vm_state(&api_socket), "Running"); 6405 }); 6406 6407 let _ = child.kill(); 6408 let output = child.wait_with_output().unwrap(); 6409 6410 handle_child_output(r, &output); 6411 } 6412 6413 #[test] 6414 fn test_watchdog() { 6415 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6416 let guest = Guest::new(Box::new(focal)); 6417 let api_socket = temp_api_path(&guest.tmp_dir); 6418 6419 let kernel_path = direct_kernel_boot_path(); 6420 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6421 6422 let mut cmd = GuestCommand::new(&guest); 6423 cmd.args(["--cpus", "boot=1"]) 6424 .args(["--memory", "size=512M"]) 6425 .args(["--kernel", kernel_path.to_str().unwrap()]) 6426 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6427 .default_disks() 6428 .args(["--net", guest.default_net_string().as_str()]) 6429 .args(["--watchdog"]) 6430 .args(["--api-socket", &api_socket]) 6431 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6432 .capture_output(); 6433 6434 let mut child = cmd.spawn().unwrap(); 6435 6436 let r = std::panic::catch_unwind(|| { 6437 guest.wait_vm_boot(None).unwrap(); 6438 6439 let mut expected_reboot_count = 1; 6440 6441 // Enable the watchdog with a 15s timeout 6442 enable_guest_watchdog(&guest, 15); 6443 6444 // Reboot and check that systemd has activated the watchdog 6445 guest.ssh_command("sudo reboot").unwrap(); 6446 guest.wait_vm_boot(None).unwrap(); 6447 expected_reboot_count += 1; 6448 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6449 assert_eq!( 6450 guest 6451 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6452 .unwrap() 6453 .trim() 6454 .parse::<u32>() 6455 .unwrap_or_default(), 6456 2 6457 ); 6458 6459 // Allow some normal time to elapse to check we don't get spurious reboots 6460 thread::sleep(std::time::Duration::new(40, 0)); 6461 // Check no reboot 6462 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6463 6464 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6465 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6466 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6467 guest.wait_vm_boot(Some(50)).unwrap(); 6468 // Check a reboot is triggered by the watchdog 6469 expected_reboot_count += 1; 6470 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6471 6472 #[cfg(target_arch = "x86_64")] 6473 { 6474 // Now pause the VM and remain offline for 30s 6475 assert!(remote_command(&api_socket, "pause", None)); 6476 let latest_events = [ 6477 &MetaEvent { 6478 event: "pausing".to_string(), 6479 device_id: None, 6480 }, 6481 &MetaEvent { 6482 event: "paused".to_string(), 6483 device_id: None, 6484 }, 6485 ]; 6486 assert!(check_latest_events_exact(&latest_events, &event_path)); 6487 assert!(remote_command(&api_socket, "resume", None)); 6488 6489 // Check no reboot 6490 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6491 } 6492 }); 6493 6494 let _ = child.kill(); 6495 let output = child.wait_with_output().unwrap(); 6496 6497 handle_child_output(r, &output); 6498 } 6499 6500 #[test] 6501 fn test_pvpanic() { 6502 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 6503 let guest = Guest::new(Box::new(jammy)); 6504 let api_socket = temp_api_path(&guest.tmp_dir); 6505 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6506 6507 let kernel_path = direct_kernel_boot_path(); 6508 6509 let mut cmd = GuestCommand::new(&guest); 6510 cmd.args(["--cpus", "boot=1"]) 6511 .args(["--memory", "size=512M"]) 6512 .args(["--kernel", kernel_path.to_str().unwrap()]) 6513 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6514 .default_disks() 6515 .args(["--net", guest.default_net_string().as_str()]) 6516 .args(["--pvpanic"]) 6517 .args(["--api-socket", &api_socket]) 6518 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6519 .capture_output(); 6520 6521 let mut child = cmd.spawn().unwrap(); 6522 6523 let r = std::panic::catch_unwind(|| { 6524 guest.wait_vm_boot(None).unwrap(); 6525 6526 // Trigger guest a panic 6527 make_guest_panic(&guest); 6528 6529 // Wait a while for guest 6530 thread::sleep(std::time::Duration::new(10, 0)); 6531 6532 let expected_sequential_events = [&MetaEvent { 6533 event: "panic".to_string(), 6534 device_id: None, 6535 }]; 6536 assert!(check_latest_events_exact( 6537 &expected_sequential_events, 6538 &event_path 6539 )); 6540 }); 6541 6542 let _ = child.kill(); 6543 let output = child.wait_with_output().unwrap(); 6544 6545 handle_child_output(r, &output); 6546 } 6547 6548 #[test] 6549 fn test_tap_from_fd() { 6550 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6551 let guest = Guest::new(Box::new(focal)); 6552 let kernel_path = direct_kernel_boot_path(); 6553 6554 // Create a TAP interface with multi-queue enabled 6555 let num_queue_pairs: usize = 2; 6556 6557 use std::str::FromStr; 6558 let taps = net_util::open_tap( 6559 Some("chtap0"), 6560 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6561 None, 6562 &mut None, 6563 None, 6564 num_queue_pairs, 6565 Some(libc::O_RDWR | libc::O_NONBLOCK), 6566 ) 6567 .unwrap(); 6568 6569 let mut child = GuestCommand::new(&guest) 6570 .args(["--cpus", &format!("boot={num_queue_pairs}")]) 6571 .args(["--memory", "size=512M"]) 6572 .args(["--kernel", kernel_path.to_str().unwrap()]) 6573 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6574 .default_disks() 6575 .args([ 6576 "--net", 6577 &format!( 6578 "fd=[{},{}],mac={},num_queues={}", 6579 taps[0].as_raw_fd(), 6580 taps[1].as_raw_fd(), 6581 guest.network.guest_mac, 6582 num_queue_pairs * 2 6583 ), 6584 ]) 6585 .capture_output() 6586 .spawn() 6587 .unwrap(); 6588 6589 let r = std::panic::catch_unwind(|| { 6590 guest.wait_vm_boot(None).unwrap(); 6591 6592 assert_eq!( 6593 guest 6594 .ssh_command("ip -o link | wc -l") 6595 .unwrap() 6596 .trim() 6597 .parse::<u32>() 6598 .unwrap_or_default(), 6599 2 6600 ); 6601 6602 guest.reboot_linux(0, None); 6603 6604 assert_eq!( 6605 guest 6606 .ssh_command("ip -o link | wc -l") 6607 .unwrap() 6608 .trim() 6609 .parse::<u32>() 6610 .unwrap_or_default(), 6611 2 6612 ); 6613 }); 6614 6615 let _ = child.kill(); 6616 let output = child.wait_with_output().unwrap(); 6617 6618 handle_child_output(r, &output); 6619 } 6620 6621 // By design, a guest VM won't be able to connect to the host 6622 // machine when using a macvtap network interface (while it can 6623 // communicate externally). As a workaround, this integration 6624 // test creates two macvtap interfaces in 'bridge' mode on the 6625 // same physical net interface, one for the guest and one for 6626 // the host. With additional setup on the IP address and the 6627 // routing table, it enables the communications between the 6628 // guest VM and the host machine. 6629 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6630 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6631 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6632 let guest = Guest::new(Box::new(focal)); 6633 let api_socket = temp_api_path(&guest.tmp_dir); 6634 6635 #[cfg(target_arch = "x86_64")] 6636 let kernel_path = direct_kernel_boot_path(); 6637 #[cfg(target_arch = "aarch64")] 6638 let kernel_path = edk2_path(); 6639 6640 let phy_net = "eth0"; 6641 6642 // Create a macvtap interface for the guest VM to use 6643 assert!(exec_host_command_status(&format!( 6644 "sudo ip link add link {phy_net} name {guest_macvtap_name} type macvtap mod bridge" 6645 )) 6646 .success()); 6647 assert!(exec_host_command_status(&format!( 6648 "sudo ip link set {} address {} up", 6649 guest_macvtap_name, guest.network.guest_mac 6650 )) 6651 .success()); 6652 assert!( 6653 exec_host_command_status(&format!("sudo ip link show {guest_macvtap_name}")).success() 6654 ); 6655 6656 let tap_index = 6657 fs::read_to_string(format!("/sys/class/net/{guest_macvtap_name}/ifindex")).unwrap(); 6658 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6659 6660 assert!(exec_host_command_status(&format!("sudo chown $UID.$UID {tap_device}")).success()); 6661 6662 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6663 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6664 assert!(tap_fd1 > 0); 6665 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6666 assert!(tap_fd2 > 0); 6667 6668 // Create a macvtap on the same physical net interface for 6669 // the host machine to use 6670 assert!(exec_host_command_status(&format!( 6671 "sudo ip link add link {phy_net} name {host_macvtap_name} type macvtap mod bridge" 6672 )) 6673 .success()); 6674 // Use default mask "255.255.255.0" 6675 assert!(exec_host_command_status(&format!( 6676 "sudo ip address add {}/24 dev {}", 6677 guest.network.host_ip, host_macvtap_name 6678 )) 6679 .success()); 6680 assert!( 6681 exec_host_command_status(&format!("sudo ip link set dev {host_macvtap_name} up")) 6682 .success() 6683 ); 6684 6685 let mut guest_command = GuestCommand::new(&guest); 6686 guest_command 6687 .args(["--cpus", "boot=2"]) 6688 .args(["--memory", "size=512M"]) 6689 .args(["--kernel", kernel_path.to_str().unwrap()]) 6690 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6691 .default_disks() 6692 .args(["--api-socket", &api_socket]); 6693 6694 let net_params = format!( 6695 "fd=[{},{}],mac={},num_queues=4", 6696 tap_fd1, tap_fd2, guest.network.guest_mac 6697 ); 6698 6699 if !hotplug { 6700 guest_command.args(["--net", &net_params]); 6701 } 6702 6703 let mut child = guest_command.capture_output().spawn().unwrap(); 6704 6705 if hotplug { 6706 // Give some time to the VMM process to listen to the API 6707 // socket. This is the only requirement to avoid the following 6708 // call to ch-remote from failing. 6709 thread::sleep(std::time::Duration::new(10, 0)); 6710 // Hotplug the virtio-net device 6711 let (cmd_success, cmd_output) = 6712 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6713 assert!(cmd_success); 6714 #[cfg(target_arch = "x86_64")] 6715 assert!(String::from_utf8_lossy(&cmd_output) 6716 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6717 #[cfg(target_arch = "aarch64")] 6718 assert!(String::from_utf8_lossy(&cmd_output) 6719 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6720 } 6721 6722 // The functional connectivity provided by the virtio-net device 6723 // gets tested through wait_vm_boot() as it expects to receive a 6724 // HTTP request, and through the SSH command as well. 6725 let r = std::panic::catch_unwind(|| { 6726 guest.wait_vm_boot(None).unwrap(); 6727 6728 assert_eq!( 6729 guest 6730 .ssh_command("ip -o link | wc -l") 6731 .unwrap() 6732 .trim() 6733 .parse::<u32>() 6734 .unwrap_or_default(), 6735 2 6736 ); 6737 6738 guest.reboot_linux(0, None); 6739 6740 assert_eq!( 6741 guest 6742 .ssh_command("ip -o link | wc -l") 6743 .unwrap() 6744 .trim() 6745 .parse::<u32>() 6746 .unwrap_or_default(), 6747 2 6748 ); 6749 }); 6750 6751 let _ = child.kill(); 6752 6753 exec_host_command_status(&format!("sudo ip link del {guest_macvtap_name}")); 6754 exec_host_command_status(&format!("sudo ip link del {host_macvtap_name}")); 6755 6756 let output = child.wait_with_output().unwrap(); 6757 6758 handle_child_output(r, &output); 6759 } 6760 6761 #[test] 6762 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6763 fn test_macvtap() { 6764 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6765 } 6766 6767 #[test] 6768 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6769 fn test_macvtap_hotplug() { 6770 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6771 } 6772 6773 #[test] 6774 #[cfg(not(feature = "mshv"))] 6775 fn test_ovs_dpdk() { 6776 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6777 let guest1 = Guest::new(Box::new(focal1)); 6778 6779 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6780 let guest2 = Guest::new(Box::new(focal2)); 6781 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6782 6783 let (mut child1, mut child2) = 6784 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6785 6786 // Create the snapshot directory 6787 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6788 6789 let r = std::panic::catch_unwind(|| { 6790 // Remove one of the two ports from the OVS bridge 6791 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6792 6793 // Spawn a new netcat listener in the first VM 6794 let guest_ip = guest1.network.guest_ip.clone(); 6795 thread::spawn(move || { 6796 ssh_command_ip( 6797 "nc -l 12345", 6798 &guest_ip, 6799 DEFAULT_SSH_RETRIES, 6800 DEFAULT_SSH_TIMEOUT, 6801 ) 6802 .unwrap(); 6803 }); 6804 6805 // Wait for the server to be listening 6806 thread::sleep(std::time::Duration::new(5, 0)); 6807 6808 // Check the connection fails this time 6809 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6810 6811 // Add the OVS port back 6812 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6813 6814 // And finally check the connection is functional again 6815 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6816 6817 // Pause the VM 6818 assert!(remote_command(&api_socket_source, "pause", None)); 6819 6820 // Take a snapshot from the VM 6821 assert!(remote_command( 6822 &api_socket_source, 6823 "snapshot", 6824 Some(format!("file://{snapshot_dir}").as_str()), 6825 )); 6826 6827 // Wait to make sure the snapshot is completed 6828 thread::sleep(std::time::Duration::new(10, 0)); 6829 }); 6830 6831 // Shutdown the source VM 6832 let _ = child2.kill(); 6833 let output = child2.wait_with_output().unwrap(); 6834 handle_child_output(r, &output); 6835 6836 // Remove the vhost-user socket file. 6837 Command::new("rm") 6838 .arg("-f") 6839 .arg("/tmp/dpdkvhostclient2") 6840 .output() 6841 .unwrap(); 6842 6843 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6844 // Restore the VM from the snapshot 6845 let mut child2 = GuestCommand::new(&guest2) 6846 .args(["--api-socket", &api_socket_restored]) 6847 .args([ 6848 "--restore", 6849 format!("source_url=file://{snapshot_dir}").as_str(), 6850 ]) 6851 .capture_output() 6852 .spawn() 6853 .unwrap(); 6854 6855 // Wait for the VM to be restored 6856 thread::sleep(std::time::Duration::new(10, 0)); 6857 6858 let r = std::panic::catch_unwind(|| { 6859 // Resume the VM 6860 assert!(remote_command(&api_socket_restored, "resume", None)); 6861 6862 // Spawn a new netcat listener in the first VM 6863 let guest_ip = guest1.network.guest_ip.clone(); 6864 thread::spawn(move || { 6865 ssh_command_ip( 6866 "nc -l 12345", 6867 &guest_ip, 6868 DEFAULT_SSH_RETRIES, 6869 DEFAULT_SSH_TIMEOUT, 6870 ) 6871 .unwrap(); 6872 }); 6873 6874 // Wait for the server to be listening 6875 thread::sleep(std::time::Duration::new(5, 0)); 6876 6877 // And check the connection is still functional after restore 6878 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6879 }); 6880 6881 let _ = child1.kill(); 6882 let _ = child2.kill(); 6883 6884 let output = child1.wait_with_output().unwrap(); 6885 child2.wait().unwrap(); 6886 6887 cleanup_ovs_dpdk(); 6888 6889 handle_child_output(r, &output); 6890 } 6891 6892 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6893 cleanup_spdk_nvme(); 6894 6895 assert!(exec_host_command_status(&format!( 6896 "mkdir -p {}", 6897 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6898 )) 6899 .success()); 6900 assert!(exec_host_command_status(&format!( 6901 "truncate {} -s 128M", 6902 nvme_dir.join("test-disk.raw").to_str().unwrap() 6903 )) 6904 .success()); 6905 assert!(exec_host_command_status(&format!( 6906 "mkfs.ext4 {}", 6907 nvme_dir.join("test-disk.raw").to_str().unwrap() 6908 )) 6909 .success()); 6910 6911 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6912 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6913 .args(["-i", "0", "-m", "0x1"]) 6914 .spawn() 6915 .unwrap(); 6916 thread::sleep(std::time::Duration::new(2, 0)); 6917 6918 assert!(exec_host_command_with_retries( 6919 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER", 6920 3, 6921 std::time::Duration::new(5, 0), 6922 )); 6923 assert!(exec_host_command_status(&format!( 6924 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6925 nvme_dir.join("test-disk.raw").to_str().unwrap() 6926 )) 6927 .success()); 6928 assert!(exec_host_command_status( 6929 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6930 ) 6931 .success()); 6932 assert!(exec_host_command_status( 6933 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6934 ) 6935 .success()); 6936 assert!(exec_host_command_status(&format!( 6937 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6938 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6939 )) 6940 .success()); 6941 } 6942 6943 fn cleanup_spdk_nvme() { 6944 exec_host_command_status("pkill -f nvmf_tgt"); 6945 } 6946 6947 #[test] 6948 fn test_vfio_user() { 6949 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6950 let jammy = UbuntuDiskConfig::new(jammy_image); 6951 let guest = Guest::new(Box::new(jammy)); 6952 6953 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6954 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6955 6956 let api_socket = temp_api_path(&guest.tmp_dir); 6957 let mut child = GuestCommand::new(&guest) 6958 .args(["--api-socket", &api_socket]) 6959 .args(["--cpus", "boot=1"]) 6960 .args(["--memory", "size=512M,shared=on,hugepages=on"]) 6961 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6962 .args(["--serial", "tty", "--console", "off"]) 6963 .default_disks() 6964 .default_net() 6965 .capture_output() 6966 .spawn() 6967 .unwrap(); 6968 6969 let r = std::panic::catch_unwind(|| { 6970 guest.wait_vm_boot(None).unwrap(); 6971 6972 // Hotplug the SPDK-NVMe device to the VM 6973 let (cmd_success, cmd_output) = remote_command_w_output( 6974 &api_socket, 6975 "add-user-device", 6976 Some(&format!( 6977 "socket={},id=vfio_user0", 6978 spdk_nvme_dir 6979 .as_path() 6980 .join("nvme-vfio-user/cntrl") 6981 .to_str() 6982 .unwrap(), 6983 )), 6984 ); 6985 assert!(cmd_success); 6986 assert!(String::from_utf8_lossy(&cmd_output) 6987 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6988 6989 thread::sleep(std::time::Duration::new(10, 0)); 6990 6991 // Check both if /dev/nvme exists and if the block size is 128M. 6992 assert_eq!( 6993 guest 6994 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6995 .unwrap() 6996 .trim() 6997 .parse::<u32>() 6998 .unwrap_or_default(), 6999 1 7000 ); 7001 7002 // Check changes persist after reboot 7003 assert_eq!( 7004 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 7005 "" 7006 ); 7007 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 7008 guest 7009 .ssh_command("echo test123 | sudo tee /mnt/test") 7010 .unwrap(); 7011 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 7012 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 7013 7014 guest.reboot_linux(0, None); 7015 assert_eq!( 7016 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 7017 "" 7018 ); 7019 assert_eq!( 7020 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 7021 "test123" 7022 ); 7023 }); 7024 7025 cleanup_spdk_nvme(); 7026 7027 let _ = child.kill(); 7028 let output = child.wait_with_output().unwrap(); 7029 7030 handle_child_output(r, &output); 7031 } 7032 7033 #[test] 7034 #[cfg(target_arch = "x86_64")] 7035 fn test_vdpa_block() { 7036 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 7037 assert!(exec_host_command_status("lsmod | grep vdpa_sim_blk").success()); 7038 7039 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7040 let guest = Guest::new(Box::new(focal)); 7041 let api_socket = temp_api_path(&guest.tmp_dir); 7042 7043 let kernel_path = direct_kernel_boot_path(); 7044 7045 let mut child = GuestCommand::new(&guest) 7046 .args(["--cpus", "boot=2"]) 7047 .args(["--memory", "size=512M,hugepages=on"]) 7048 .args(["--kernel", kernel_path.to_str().unwrap()]) 7049 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7050 .default_disks() 7051 .default_net() 7052 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 7053 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 7054 .args(["--api-socket", &api_socket]) 7055 .capture_output() 7056 .spawn() 7057 .unwrap(); 7058 7059 let r = std::panic::catch_unwind(|| { 7060 guest.wait_vm_boot(None).unwrap(); 7061 7062 // Check both if /dev/vdc exists and if the block size is 128M. 7063 assert_eq!( 7064 guest 7065 .ssh_command("lsblk | grep vdc | grep -c 128M") 7066 .unwrap() 7067 .trim() 7068 .parse::<u32>() 7069 .unwrap_or_default(), 7070 1 7071 ); 7072 7073 // Check the content of the block device after we wrote to it. 7074 // The vpda-sim-blk should let us read what we previously wrote. 7075 guest 7076 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 7077 .unwrap(); 7078 assert_eq!( 7079 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 7080 "foobar" 7081 ); 7082 7083 // Hotplug an extra vDPA block device behind the vIOMMU 7084 // Add a new vDPA device to the VM 7085 let (cmd_success, cmd_output) = remote_command_w_output( 7086 &api_socket, 7087 "add-vdpa", 7088 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 7089 ); 7090 assert!(cmd_success); 7091 assert!(String::from_utf8_lossy(&cmd_output) 7092 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 7093 7094 thread::sleep(std::time::Duration::new(10, 0)); 7095 7096 // Check IOMMU setup 7097 assert!(guest 7098 .does_device_vendor_pair_match("0x1057", "0x1af4") 7099 .unwrap_or_default()); 7100 assert_eq!( 7101 guest 7102 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 7103 .unwrap() 7104 .trim(), 7105 "0001:00:01.0" 7106 ); 7107 7108 // Check both if /dev/vdd exists and if the block size is 128M. 7109 assert_eq!( 7110 guest 7111 .ssh_command("lsblk | grep vdd | grep -c 128M") 7112 .unwrap() 7113 .trim() 7114 .parse::<u32>() 7115 .unwrap_or_default(), 7116 1 7117 ); 7118 7119 // Write some content to the block device we've just plugged. 7120 guest 7121 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 7122 .unwrap(); 7123 7124 // Check we can read the content back. 7125 assert_eq!( 7126 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 7127 "foobar" 7128 ); 7129 7130 // Unplug the device 7131 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 7132 assert!(cmd_success); 7133 thread::sleep(std::time::Duration::new(10, 0)); 7134 7135 // Check /dev/vdd doesn't exist anymore 7136 assert_eq!( 7137 guest 7138 .ssh_command("lsblk | grep -c vdd || true") 7139 .unwrap() 7140 .trim() 7141 .parse::<u32>() 7142 .unwrap_or(1), 7143 0 7144 ); 7145 }); 7146 7147 let _ = child.kill(); 7148 let output = child.wait_with_output().unwrap(); 7149 7150 handle_child_output(r, &output); 7151 } 7152 7153 #[test] 7154 #[cfg(target_arch = "x86_64")] 7155 #[ignore = "See #5756"] 7156 fn test_vdpa_net() { 7157 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 7158 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 7159 return; 7160 } 7161 7162 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7163 let guest = Guest::new(Box::new(focal)); 7164 7165 let kernel_path = direct_kernel_boot_path(); 7166 7167 let mut child = GuestCommand::new(&guest) 7168 .args(["--cpus", "boot=2"]) 7169 .args(["--memory", "size=512M,hugepages=on"]) 7170 .args(["--kernel", kernel_path.to_str().unwrap()]) 7171 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7172 .default_disks() 7173 .default_net() 7174 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 7175 .capture_output() 7176 .spawn() 7177 .unwrap(); 7178 7179 let r = std::panic::catch_unwind(|| { 7180 guest.wait_vm_boot(None).unwrap(); 7181 7182 // Check we can find network interface related to vDPA device 7183 assert_eq!( 7184 guest 7185 .ssh_command("ip -o link | grep -c ens6") 7186 .unwrap() 7187 .trim() 7188 .parse::<u32>() 7189 .unwrap_or(0), 7190 1 7191 ); 7192 7193 guest 7194 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 7195 .unwrap(); 7196 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 7197 7198 // Check there is no packet yet on both TX/RX of the network interface 7199 assert_eq!( 7200 guest 7201 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 7202 .unwrap() 7203 .trim() 7204 .parse::<u32>() 7205 .unwrap_or(0), 7206 2 7207 ); 7208 7209 // Send 6 packets with ping command 7210 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 7211 7212 // Check we can find 6 packets on both TX/RX of the network interface 7213 assert_eq!( 7214 guest 7215 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 7216 .unwrap() 7217 .trim() 7218 .parse::<u32>() 7219 .unwrap_or(0), 7220 2 7221 ); 7222 7223 // No need to check for hotplug as we already tested it through 7224 // test_vdpa_block() 7225 }); 7226 7227 let _ = child.kill(); 7228 let output = child.wait_with_output().unwrap(); 7229 7230 handle_child_output(r, &output); 7231 } 7232 7233 #[test] 7234 #[cfg(target_arch = "x86_64")] 7235 fn test_tpm() { 7236 let focal = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7237 let guest = Guest::new(Box::new(focal)); 7238 7239 let (mut swtpm_command, swtpm_socket_path) = prepare_swtpm_daemon(&guest.tmp_dir); 7240 7241 let mut guest_cmd = GuestCommand::new(&guest); 7242 guest_cmd 7243 .args(["--cpus", "boot=1"]) 7244 .args(["--memory", "size=512M"]) 7245 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7246 .args(["--tpm", &format!("socket={swtpm_socket_path}")]) 7247 .capture_output() 7248 .default_disks() 7249 .default_net(); 7250 7251 // Start swtpm daemon 7252 let mut swtpm_child = swtpm_command.spawn().unwrap(); 7253 thread::sleep(std::time::Duration::new(10, 0)); 7254 let mut child = guest_cmd.spawn().unwrap(); 7255 let r = std::panic::catch_unwind(|| { 7256 guest.wait_vm_boot(None).unwrap(); 7257 assert_eq!( 7258 guest.ssh_command("ls /dev/tpm0").unwrap().trim(), 7259 "/dev/tpm0" 7260 ); 7261 guest.ssh_command("sudo tpm2_selftest -f").unwrap(); 7262 guest 7263 .ssh_command("echo 'hello' > /tmp/checksum_test; ") 7264 .unwrap(); 7265 guest.ssh_command("cmp <(sudo tpm2_pcrevent /tmp/checksum_test | grep sha256 | awk '{print $2}') <(sha256sum /tmp/checksum_test| awk '{print $1}')").unwrap(); 7266 }); 7267 7268 let _ = swtpm_child.kill(); 7269 let _d_out = swtpm_child.wait_with_output().unwrap(); 7270 7271 let _ = child.kill(); 7272 let output = child.wait_with_output().unwrap(); 7273 7274 handle_child_output(r, &output); 7275 } 7276 7277 #[test] 7278 #[cfg(target_arch = "x86_64")] 7279 fn test_double_tty() { 7280 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7281 let guest = Guest::new(Box::new(focal)); 7282 let mut cmd = GuestCommand::new(&guest); 7283 let api_socket = temp_api_path(&guest.tmp_dir); 7284 let tty_str: &str = "console=hvc0 earlyprintk=ttyS0 "; 7285 // linux printk module enable console log. 7286 let con_dis_str: &str = "console [hvc0] enabled"; 7287 // linux printk module disable console log. 7288 let con_enb_str: &str = "bootconsole [earlyser0] disabled"; 7289 7290 let kernel_path = direct_kernel_boot_path(); 7291 7292 cmd.args(["--cpus", "boot=1"]) 7293 .args(["--memory", "size=512M"]) 7294 .args(["--kernel", kernel_path.to_str().unwrap()]) 7295 .args([ 7296 "--cmdline", 7297 DIRECT_KERNEL_BOOT_CMDLINE 7298 .replace("console=hvc0 ", tty_str) 7299 .as_str(), 7300 ]) 7301 .capture_output() 7302 .default_disks() 7303 .default_net() 7304 .args(["--serial", "tty"]) 7305 .args(["--console", "tty"]) 7306 .args(["--api-socket", &api_socket]); 7307 7308 let mut child = cmd.spawn().unwrap(); 7309 7310 let mut r = std::panic::catch_unwind(|| { 7311 guest.wait_vm_boot(None).unwrap(); 7312 }); 7313 7314 let _ = child.kill(); 7315 let output = child.wait_with_output().unwrap(); 7316 7317 if r.is_ok() { 7318 r = std::panic::catch_unwind(|| { 7319 let s = String::from_utf8_lossy(&output.stdout); 7320 assert!(s.contains(tty_str)); 7321 assert!(s.contains(con_dis_str)); 7322 assert!(s.contains(con_enb_str)); 7323 }); 7324 } 7325 7326 handle_child_output(r, &output); 7327 } 7328 7329 #[test] 7330 #[cfg(target_arch = "x86_64")] 7331 fn test_nmi() { 7332 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7333 let guest = Guest::new(Box::new(jammy)); 7334 let api_socket = temp_api_path(&guest.tmp_dir); 7335 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7336 7337 let kernel_path = direct_kernel_boot_path(); 7338 let cmd_line = format!("{} {}", DIRECT_KERNEL_BOOT_CMDLINE, "unknown_nmi_panic=1"); 7339 7340 let mut cmd = GuestCommand::new(&guest); 7341 cmd.args(["--cpus", "boot=4"]) 7342 .args(["--memory", "size=512M"]) 7343 .args(["--kernel", kernel_path.to_str().unwrap()]) 7344 .args(["--cmdline", cmd_line.as_str()]) 7345 .default_disks() 7346 .args(["--net", guest.default_net_string().as_str()]) 7347 .args(["--pvpanic"]) 7348 .args(["--api-socket", &api_socket]) 7349 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7350 .capture_output(); 7351 7352 let mut child = cmd.spawn().unwrap(); 7353 7354 let r = std::panic::catch_unwind(|| { 7355 guest.wait_vm_boot(None).unwrap(); 7356 7357 assert!(remote_command(&api_socket, "nmi", None)); 7358 7359 // Wait a while for guest 7360 thread::sleep(std::time::Duration::new(3, 0)); 7361 7362 let expected_sequential_events = [&MetaEvent { 7363 event: "panic".to_string(), 7364 device_id: None, 7365 }]; 7366 assert!(check_latest_events_exact( 7367 &expected_sequential_events, 7368 &event_path 7369 )); 7370 }); 7371 7372 let _ = child.kill(); 7373 let output = child.wait_with_output().unwrap(); 7374 7375 handle_child_output(r, &output); 7376 } 7377 } 7378 7379 mod dbus_api { 7380 use crate::*; 7381 7382 // Start cloud-hypervisor with no VM parameters, running both the HTTP 7383 // and DBus APIs. Alternate calls to the external APIs (HTTP and DBus) 7384 // to create a VM, boot it, and verify that it can be shut down and then 7385 // booted again. 7386 #[test] 7387 fn test_api_dbus_and_http_interleaved() { 7388 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7389 let guest = Guest::new(Box::new(focal)); 7390 let dbus_api = TargetApi::new_dbus_api(&guest.tmp_dir); 7391 let http_api = TargetApi::new_http_api(&guest.tmp_dir); 7392 7393 let mut child = GuestCommand::new(&guest) 7394 .args(dbus_api.guest_args()) 7395 .args(http_api.guest_args()) 7396 .capture_output() 7397 .spawn() 7398 .unwrap(); 7399 7400 thread::sleep(std::time::Duration::new(1, 0)); 7401 7402 // Verify API servers are running 7403 assert!(dbus_api.remote_command("ping", None)); 7404 assert!(http_api.remote_command("ping", None)); 7405 7406 // Create the VM first 7407 let cpu_count: u8 = 4; 7408 let request_body = guest.api_create_body( 7409 cpu_count, 7410 direct_kernel_boot_path().to_str().unwrap(), 7411 DIRECT_KERNEL_BOOT_CMDLINE, 7412 ); 7413 7414 let temp_config_path = guest.tmp_dir.as_path().join("config"); 7415 std::fs::write(&temp_config_path, request_body).unwrap(); 7416 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 7417 7418 let r = std::panic::catch_unwind(|| { 7419 // Create the VM 7420 assert!(dbus_api.remote_command("create", Some(create_config),)); 7421 7422 // Then boot it 7423 assert!(http_api.remote_command("boot", None)); 7424 guest.wait_vm_boot(None).unwrap(); 7425 7426 // Check that the VM booted as expected 7427 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7428 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7429 7430 // Sync and shutdown without powering off to prevent filesystem 7431 // corruption. 7432 guest.ssh_command("sync").unwrap(); 7433 guest.ssh_command("sudo shutdown -H now").unwrap(); 7434 7435 // Wait for the guest to be fully shutdown 7436 thread::sleep(std::time::Duration::new(20, 0)); 7437 7438 // Then shutdown the VM 7439 assert!(dbus_api.remote_command("shutdown", None)); 7440 7441 // Then boot it again 7442 assert!(http_api.remote_command("boot", None)); 7443 guest.wait_vm_boot(None).unwrap(); 7444 7445 // Check that the VM booted as expected 7446 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7447 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7448 }); 7449 7450 let _ = child.kill(); 7451 let output = child.wait_with_output().unwrap(); 7452 7453 handle_child_output(r, &output); 7454 } 7455 7456 #[test] 7457 fn test_api_dbus_create_boot() { 7458 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7459 let guest = Guest::new(Box::new(focal)); 7460 7461 _test_api_create_boot(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7462 } 7463 7464 #[test] 7465 fn test_api_dbus_shutdown() { 7466 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7467 let guest = Guest::new(Box::new(focal)); 7468 7469 _test_api_shutdown(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7470 } 7471 7472 #[test] 7473 fn test_api_dbus_delete() { 7474 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7475 let guest = Guest::new(Box::new(focal)); 7476 7477 _test_api_delete(TargetApi::new_dbus_api(&guest.tmp_dir), guest); 7478 } 7479 7480 #[test] 7481 fn test_api_dbus_pause_resume() { 7482 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7483 let guest = Guest::new(Box::new(focal)); 7484 7485 _test_api_pause_resume(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7486 } 7487 } 7488 7489 mod common_sequential { 7490 #[cfg(not(feature = "mshv"))] 7491 use crate::*; 7492 7493 #[test] 7494 #[cfg(not(feature = "mshv"))] 7495 fn test_memory_mergeable_on() { 7496 test_memory_mergeable(true) 7497 } 7498 } 7499 7500 mod windows { 7501 use crate::*; 7502 use once_cell::sync::Lazy; 7503 7504 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 7505 7506 struct WindowsGuest { 7507 guest: Guest, 7508 auth: PasswordAuth, 7509 } 7510 7511 trait FsType { 7512 const FS_FAT: u8; 7513 const FS_NTFS: u8; 7514 } 7515 impl FsType for WindowsGuest { 7516 const FS_FAT: u8 = 0; 7517 const FS_NTFS: u8 = 1; 7518 } 7519 7520 impl WindowsGuest { 7521 fn new() -> Self { 7522 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 7523 let guest = Guest::new(Box::new(disk)); 7524 let auth = PasswordAuth { 7525 username: String::from("administrator"), 7526 password: String::from("Admin123"), 7527 }; 7528 7529 WindowsGuest { guest, auth } 7530 } 7531 7532 fn guest(&self) -> &Guest { 7533 &self.guest 7534 } 7535 7536 fn ssh_cmd(&self, cmd: &str) -> String { 7537 ssh_command_ip_with_auth( 7538 cmd, 7539 &self.auth, 7540 &self.guest.network.guest_ip, 7541 DEFAULT_SSH_RETRIES, 7542 DEFAULT_SSH_TIMEOUT, 7543 ) 7544 .unwrap() 7545 } 7546 7547 fn cpu_count(&self) -> u8 { 7548 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 7549 .trim() 7550 .parse::<u8>() 7551 .unwrap_or(0) 7552 } 7553 7554 fn ram_size(&self) -> usize { 7555 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 7556 .trim() 7557 .parse::<usize>() 7558 .unwrap_or(0) 7559 } 7560 7561 fn netdev_count(&self) -> u8 { 7562 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7563 .trim() 7564 .parse::<u8>() 7565 .unwrap_or(0) 7566 } 7567 7568 fn disk_count(&self) -> u8 { 7569 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7570 .trim() 7571 .parse::<u8>() 7572 .unwrap_or(0) 7573 } 7574 7575 fn reboot(&self) { 7576 let _ = self.ssh_cmd("shutdown /r /t 0"); 7577 } 7578 7579 fn shutdown(&self) { 7580 let _ = self.ssh_cmd("shutdown /s /t 0"); 7581 } 7582 7583 fn run_dnsmasq(&self) -> std::process::Child { 7584 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 7585 let dhcp_host = format!( 7586 "--dhcp-host={},{}", 7587 self.guest.network.guest_mac, self.guest.network.guest_ip 7588 ); 7589 let dhcp_range = format!( 7590 "--dhcp-range=eth,{},{}", 7591 self.guest.network.guest_ip, self.guest.network.guest_ip 7592 ); 7593 7594 Command::new("dnsmasq") 7595 .arg("--no-daemon") 7596 .arg("--log-queries") 7597 .arg(listen_address.as_str()) 7598 .arg("--except-interface=lo") 7599 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 7600 .arg("--conf-file=/dev/null") 7601 .arg(dhcp_host.as_str()) 7602 .arg(dhcp_range.as_str()) 7603 .spawn() 7604 .unwrap() 7605 } 7606 7607 // TODO Cleanup image file explicitly after test, if there's some space issues. 7608 fn disk_new(&self, fs: u8, sz: usize) -> String { 7609 let mut guard = NEXT_DISK_ID.lock().unwrap(); 7610 let id = *guard; 7611 *guard = id + 1; 7612 7613 let img = PathBuf::from(format!("/tmp/test-hotplug-{id}.raw")); 7614 let _ = fs::remove_file(&img); 7615 7616 // Create an image file 7617 let out = Command::new("qemu-img") 7618 .args([ 7619 "create", 7620 "-f", 7621 "raw", 7622 img.to_str().unwrap(), 7623 format!("{sz}m").as_str(), 7624 ]) 7625 .output() 7626 .expect("qemu-img command failed") 7627 .stdout; 7628 println!("{out:?}"); 7629 7630 // Associate image to a loop device 7631 let out = Command::new("losetup") 7632 .args(["--show", "-f", img.to_str().unwrap()]) 7633 .output() 7634 .expect("failed to create loop device") 7635 .stdout; 7636 let _tmp = String::from_utf8_lossy(&out); 7637 let loop_dev = _tmp.trim(); 7638 println!("{out:?}"); 7639 7640 // Create a partition table 7641 // echo 'type=7' | sudo sfdisk "${LOOP}" 7642 let mut child = Command::new("sfdisk") 7643 .args([loop_dev]) 7644 .stdin(Stdio::piped()) 7645 .spawn() 7646 .unwrap(); 7647 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 7648 stdin 7649 .write_all("type=7".as_bytes()) 7650 .expect("failed to write stdin"); 7651 let out = child.wait_with_output().expect("sfdisk failed").stdout; 7652 println!("{out:?}"); 7653 7654 // Disengage the loop device 7655 let out = Command::new("losetup") 7656 .args(["-d", loop_dev]) 7657 .output() 7658 .expect("loop device not found") 7659 .stdout; 7660 println!("{out:?}"); 7661 7662 // Re-associate loop device pointing to the partition only 7663 let out = Command::new("losetup") 7664 .args([ 7665 "--show", 7666 "--offset", 7667 (512 * 2048).to_string().as_str(), 7668 "-f", 7669 img.to_str().unwrap(), 7670 ]) 7671 .output() 7672 .expect("failed to create loop device") 7673 .stdout; 7674 let _tmp = String::from_utf8_lossy(&out); 7675 let loop_dev = _tmp.trim(); 7676 println!("{out:?}"); 7677 7678 // Create filesystem. 7679 let fs_cmd = match fs { 7680 WindowsGuest::FS_FAT => "mkfs.msdos", 7681 WindowsGuest::FS_NTFS => "mkfs.ntfs", 7682 _ => panic!("Unknown filesystem type '{fs}'"), 7683 }; 7684 let out = Command::new(fs_cmd) 7685 .args([&loop_dev]) 7686 .output() 7687 .unwrap_or_else(|_| panic!("{fs_cmd} failed")) 7688 .stdout; 7689 println!("{out:?}"); 7690 7691 // Disengage the loop device 7692 let out = Command::new("losetup") 7693 .args(["-d", loop_dev]) 7694 .output() 7695 .unwrap_or_else(|_| panic!("loop device '{loop_dev}' not found")) 7696 .stdout; 7697 println!("{out:?}"); 7698 7699 img.to_str().unwrap().to_string() 7700 } 7701 7702 fn disks_set_rw(&self) { 7703 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 7704 } 7705 7706 fn disks_online(&self) { 7707 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 7708 } 7709 7710 fn disk_file_put(&self, fname: &str, data: &str) { 7711 let _ = self.ssh_cmd(&format!( 7712 "powershell -Command \"'{data}' | Set-Content -Path {fname}\"" 7713 )); 7714 } 7715 7716 fn disk_file_read(&self, fname: &str) -> String { 7717 self.ssh_cmd(&format!( 7718 "powershell -Command \"Get-Content -Path {fname}\"" 7719 )) 7720 } 7721 7722 fn wait_for_boot(&self) -> bool { 7723 let cmd = "dir /b c:\\ | find \"Windows\""; 7724 let tmo_max = 180; 7725 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 7726 // interval must be small. 7727 let tmo_int = 2; 7728 let out = ssh_command_ip_with_auth( 7729 cmd, 7730 &self.auth, 7731 &self.guest.network.guest_ip, 7732 { 7733 let mut ret = 1; 7734 let mut tmo_acc = 0; 7735 loop { 7736 tmo_acc += tmo_int * ret; 7737 if tmo_acc >= tmo_max { 7738 break; 7739 } 7740 ret += 1; 7741 } 7742 ret 7743 }, 7744 tmo_int, 7745 ) 7746 .unwrap(); 7747 7748 if "Windows" == out.trim() { 7749 return true; 7750 } 7751 7752 false 7753 } 7754 } 7755 7756 fn vcpu_threads_count(pid: u32) -> u8 { 7757 // ps -T -p 12345 | grep vcpu | wc -l 7758 let out = Command::new("ps") 7759 .args(["-T", "-p", format!("{pid}").as_str()]) 7760 .output() 7761 .expect("ps command failed") 7762 .stdout; 7763 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 7764 } 7765 7766 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 7767 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 7768 let out = Command::new("ps") 7769 .args(["-T", "-p", format!("{pid}").as_str()]) 7770 .output() 7771 .expect("ps command failed") 7772 .stdout; 7773 let mut n = 0; 7774 String::from_utf8_lossy(&out) 7775 .split_whitespace() 7776 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 7777 n 7778 } 7779 7780 fn disk_ctrl_threads_count(pid: u32) -> u8 { 7781 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 7782 let out = Command::new("ps") 7783 .args(["-T", "-p", format!("{pid}").as_str()]) 7784 .output() 7785 .expect("ps command failed") 7786 .stdout; 7787 let mut n = 0; 7788 String::from_utf8_lossy(&out) 7789 .split_whitespace() 7790 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 7791 n 7792 } 7793 7794 #[test] 7795 fn test_windows_guest() { 7796 let windows_guest = WindowsGuest::new(); 7797 7798 let mut child = GuestCommand::new(windows_guest.guest()) 7799 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7800 .args(["--memory", "size=4G"]) 7801 .args(["--kernel", edk2_path().to_str().unwrap()]) 7802 .args(["--serial", "tty"]) 7803 .args(["--console", "off"]) 7804 .default_disks() 7805 .default_net() 7806 .capture_output() 7807 .spawn() 7808 .unwrap(); 7809 7810 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7811 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7812 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7813 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7814 7815 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7816 7817 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7818 7819 let r = std::panic::catch_unwind(|| { 7820 // Wait to make sure Windows boots up 7821 assert!(windows_guest.wait_for_boot()); 7822 7823 windows_guest.shutdown(); 7824 }); 7825 7826 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7827 let _ = child.kill(); 7828 let output = child.wait_with_output().unwrap(); 7829 7830 let _ = child_dnsmasq.kill(); 7831 let _ = child_dnsmasq.wait(); 7832 7833 handle_child_output(r, &output); 7834 } 7835 7836 #[test] 7837 fn test_windows_guest_multiple_queues() { 7838 let windows_guest = WindowsGuest::new(); 7839 7840 let mut ovmf_path = dirs::home_dir().unwrap(); 7841 ovmf_path.push("workloads"); 7842 ovmf_path.push(OVMF_NAME); 7843 7844 let mut child = GuestCommand::new(windows_guest.guest()) 7845 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 7846 .args(["--memory", "size=4G"]) 7847 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7848 .args(["--serial", "tty"]) 7849 .args(["--console", "off"]) 7850 .args([ 7851 "--disk", 7852 format!( 7853 "path={},num_queues=4", 7854 windows_guest 7855 .guest() 7856 .disk_config 7857 .disk(DiskType::OperatingSystem) 7858 .unwrap() 7859 ) 7860 .as_str(), 7861 ]) 7862 .args([ 7863 "--net", 7864 format!( 7865 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 7866 windows_guest.guest().network.guest_mac, 7867 windows_guest.guest().network.host_ip 7868 ) 7869 .as_str(), 7870 ]) 7871 .capture_output() 7872 .spawn() 7873 .unwrap(); 7874 7875 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7876 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7877 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7878 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7879 7880 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7881 7882 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7883 7884 let r = std::panic::catch_unwind(|| { 7885 // Wait to make sure Windows boots up 7886 assert!(windows_guest.wait_for_boot()); 7887 7888 windows_guest.shutdown(); 7889 }); 7890 7891 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7892 let _ = child.kill(); 7893 let output = child.wait_with_output().unwrap(); 7894 7895 let _ = child_dnsmasq.kill(); 7896 let _ = child_dnsmasq.wait(); 7897 7898 handle_child_output(r, &output); 7899 } 7900 7901 #[test] 7902 #[cfg(not(feature = "mshv"))] 7903 #[ignore = "See #4327"] 7904 fn test_windows_guest_snapshot_restore() { 7905 let windows_guest = WindowsGuest::new(); 7906 7907 let mut ovmf_path = dirs::home_dir().unwrap(); 7908 ovmf_path.push("workloads"); 7909 ovmf_path.push(OVMF_NAME); 7910 7911 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7912 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 7913 7914 let mut child = GuestCommand::new(windows_guest.guest()) 7915 .args(["--api-socket", &api_socket_source]) 7916 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7917 .args(["--memory", "size=4G"]) 7918 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7919 .args(["--serial", "tty"]) 7920 .args(["--console", "off"]) 7921 .default_disks() 7922 .default_net() 7923 .capture_output() 7924 .spawn() 7925 .unwrap(); 7926 7927 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7928 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7929 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7930 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7931 7932 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7933 7934 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7935 7936 // Wait to make sure Windows boots up 7937 assert!(windows_guest.wait_for_boot()); 7938 7939 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 7940 7941 // Pause the VM 7942 assert!(remote_command(&api_socket_source, "pause", None)); 7943 7944 // Take a snapshot from the VM 7945 assert!(remote_command( 7946 &api_socket_source, 7947 "snapshot", 7948 Some(format!("file://{snapshot_dir}").as_str()), 7949 )); 7950 7951 // Wait to make sure the snapshot is completed 7952 thread::sleep(std::time::Duration::new(30, 0)); 7953 7954 let _ = child.kill(); 7955 child.wait().unwrap(); 7956 7957 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 7958 7959 // Restore the VM from the snapshot 7960 let mut child = GuestCommand::new(windows_guest.guest()) 7961 .args(["--api-socket", &api_socket_restored]) 7962 .args([ 7963 "--restore", 7964 format!("source_url=file://{snapshot_dir}").as_str(), 7965 ]) 7966 .capture_output() 7967 .spawn() 7968 .unwrap(); 7969 7970 // Wait for the VM to be restored 7971 thread::sleep(std::time::Duration::new(20, 0)); 7972 7973 let r = std::panic::catch_unwind(|| { 7974 // Resume the VM 7975 assert!(remote_command(&api_socket_restored, "resume", None)); 7976 7977 windows_guest.shutdown(); 7978 }); 7979 7980 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7981 let _ = child.kill(); 7982 let output = child.wait_with_output().unwrap(); 7983 7984 let _ = child_dnsmasq.kill(); 7985 let _ = child_dnsmasq.wait(); 7986 7987 handle_child_output(r, &output); 7988 } 7989 7990 #[test] 7991 #[cfg(not(feature = "mshv"))] 7992 #[cfg(not(target_arch = "aarch64"))] 7993 fn test_windows_guest_cpu_hotplug() { 7994 let windows_guest = WindowsGuest::new(); 7995 7996 let mut ovmf_path = dirs::home_dir().unwrap(); 7997 ovmf_path.push("workloads"); 7998 ovmf_path.push(OVMF_NAME); 7999 8000 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8001 let api_socket = temp_api_path(&tmp_dir); 8002 8003 let mut child = GuestCommand::new(windows_guest.guest()) 8004 .args(["--api-socket", &api_socket]) 8005 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 8006 .args(["--memory", "size=4G"]) 8007 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8008 .args(["--serial", "tty"]) 8009 .args(["--console", "off"]) 8010 .default_disks() 8011 .default_net() 8012 .capture_output() 8013 .spawn() 8014 .unwrap(); 8015 8016 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8017 8018 let r = std::panic::catch_unwind(|| { 8019 // Wait to make sure Windows boots up 8020 assert!(windows_guest.wait_for_boot()); 8021 8022 let vcpu_num = 2; 8023 // Check the initial number of CPUs the guest sees 8024 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8025 // Check the initial number of vcpu threads in the CH process 8026 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8027 8028 let vcpu_num = 6; 8029 // Hotplug some CPUs 8030 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8031 // Wait to make sure CPUs are added 8032 thread::sleep(std::time::Duration::new(10, 0)); 8033 // Check the guest sees the correct number 8034 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8035 // Check the CH process has the correct number of vcpu threads 8036 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8037 8038 let vcpu_num = 4; 8039 // Remove some CPUs. Note that Windows doesn't support hot-remove. 8040 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8041 // Wait to make sure CPUs are removed 8042 thread::sleep(std::time::Duration::new(10, 0)); 8043 // Reboot to let Windows catch up 8044 windows_guest.reboot(); 8045 // Wait to make sure Windows completely rebooted 8046 thread::sleep(std::time::Duration::new(60, 0)); 8047 // Check the guest sees the correct number 8048 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8049 // Check the CH process has the correct number of vcpu threads 8050 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8051 8052 windows_guest.shutdown(); 8053 }); 8054 8055 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8056 let _ = child.kill(); 8057 let output = child.wait_with_output().unwrap(); 8058 8059 let _ = child_dnsmasq.kill(); 8060 let _ = child_dnsmasq.wait(); 8061 8062 handle_child_output(r, &output); 8063 } 8064 8065 #[test] 8066 #[cfg(not(feature = "mshv"))] 8067 #[cfg(not(target_arch = "aarch64"))] 8068 fn test_windows_guest_ram_hotplug() { 8069 let windows_guest = WindowsGuest::new(); 8070 8071 let mut ovmf_path = dirs::home_dir().unwrap(); 8072 ovmf_path.push("workloads"); 8073 ovmf_path.push(OVMF_NAME); 8074 8075 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8076 let api_socket = temp_api_path(&tmp_dir); 8077 8078 let mut child = GuestCommand::new(windows_guest.guest()) 8079 .args(["--api-socket", &api_socket]) 8080 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8081 .args(["--memory", "size=2G,hotplug_size=5G"]) 8082 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8083 .args(["--serial", "tty"]) 8084 .args(["--console", "off"]) 8085 .default_disks() 8086 .default_net() 8087 .capture_output() 8088 .spawn() 8089 .unwrap(); 8090 8091 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8092 8093 let r = std::panic::catch_unwind(|| { 8094 // Wait to make sure Windows boots up 8095 assert!(windows_guest.wait_for_boot()); 8096 8097 let ram_size = 2 * 1024 * 1024 * 1024; 8098 // Check the initial number of RAM the guest sees 8099 let current_ram_size = windows_guest.ram_size(); 8100 // This size seems to be reserved by the system and thus the 8101 // reported amount differs by this constant value. 8102 let reserved_ram_size = ram_size - current_ram_size; 8103 // Verify that there's not more than 4mb constant diff wasted 8104 // by the reserved ram. 8105 assert!(reserved_ram_size < 4 * 1024 * 1024); 8106 8107 let ram_size = 4 * 1024 * 1024 * 1024; 8108 // Hotplug some RAM 8109 resize_command(&api_socket, None, Some(ram_size), None, None); 8110 // Wait to make sure RAM has been added 8111 thread::sleep(std::time::Duration::new(10, 0)); 8112 // Check the guest sees the correct number 8113 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8114 8115 let ram_size = 3 * 1024 * 1024 * 1024; 8116 // Unplug some RAM. Note that hot-remove most likely won't work. 8117 resize_command(&api_socket, None, Some(ram_size), None, None); 8118 // Wait to make sure RAM has been added 8119 thread::sleep(std::time::Duration::new(10, 0)); 8120 // Reboot to let Windows catch up 8121 windows_guest.reboot(); 8122 // Wait to make sure guest completely rebooted 8123 thread::sleep(std::time::Duration::new(60, 0)); 8124 // Check the guest sees the correct number 8125 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8126 8127 windows_guest.shutdown(); 8128 }); 8129 8130 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8131 let _ = child.kill(); 8132 let output = child.wait_with_output().unwrap(); 8133 8134 let _ = child_dnsmasq.kill(); 8135 let _ = child_dnsmasq.wait(); 8136 8137 handle_child_output(r, &output); 8138 } 8139 8140 #[test] 8141 #[cfg(not(feature = "mshv"))] 8142 fn test_windows_guest_netdev_hotplug() { 8143 let windows_guest = WindowsGuest::new(); 8144 8145 let mut ovmf_path = dirs::home_dir().unwrap(); 8146 ovmf_path.push("workloads"); 8147 ovmf_path.push(OVMF_NAME); 8148 8149 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8150 let api_socket = temp_api_path(&tmp_dir); 8151 8152 let mut child = GuestCommand::new(windows_guest.guest()) 8153 .args(["--api-socket", &api_socket]) 8154 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8155 .args(["--memory", "size=4G"]) 8156 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8157 .args(["--serial", "tty"]) 8158 .args(["--console", "off"]) 8159 .default_disks() 8160 .default_net() 8161 .capture_output() 8162 .spawn() 8163 .unwrap(); 8164 8165 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8166 8167 let r = std::panic::catch_unwind(|| { 8168 // Wait to make sure Windows boots up 8169 assert!(windows_guest.wait_for_boot()); 8170 8171 // Initially present network device 8172 let netdev_num = 1; 8173 assert_eq!(windows_guest.netdev_count(), netdev_num); 8174 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8175 8176 // Hotplug network device 8177 let (cmd_success, cmd_output) = remote_command_w_output( 8178 &api_socket, 8179 "add-net", 8180 Some(windows_guest.guest().default_net_string().as_str()), 8181 ); 8182 assert!(cmd_success); 8183 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 8184 thread::sleep(std::time::Duration::new(5, 0)); 8185 // Verify the device is on the system 8186 let netdev_num = 2; 8187 assert_eq!(windows_guest.netdev_count(), netdev_num); 8188 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8189 8190 // Remove network device 8191 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 8192 assert!(cmd_success); 8193 thread::sleep(std::time::Duration::new(5, 0)); 8194 // Verify the device has been removed 8195 let netdev_num = 1; 8196 assert_eq!(windows_guest.netdev_count(), netdev_num); 8197 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8198 8199 windows_guest.shutdown(); 8200 }); 8201 8202 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8203 let _ = child.kill(); 8204 let output = child.wait_with_output().unwrap(); 8205 8206 let _ = child_dnsmasq.kill(); 8207 let _ = child_dnsmasq.wait(); 8208 8209 handle_child_output(r, &output); 8210 } 8211 8212 #[test] 8213 #[ignore = "See #6037"] 8214 #[cfg(not(feature = "mshv"))] 8215 #[cfg(not(target_arch = "aarch64"))] 8216 fn test_windows_guest_disk_hotplug() { 8217 let windows_guest = WindowsGuest::new(); 8218 8219 let mut ovmf_path = dirs::home_dir().unwrap(); 8220 ovmf_path.push("workloads"); 8221 ovmf_path.push(OVMF_NAME); 8222 8223 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8224 let api_socket = temp_api_path(&tmp_dir); 8225 8226 let mut child = GuestCommand::new(windows_guest.guest()) 8227 .args(["--api-socket", &api_socket]) 8228 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8229 .args(["--memory", "size=4G"]) 8230 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8231 .args(["--serial", "tty"]) 8232 .args(["--console", "off"]) 8233 .default_disks() 8234 .default_net() 8235 .capture_output() 8236 .spawn() 8237 .unwrap(); 8238 8239 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8240 8241 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 8242 8243 let r = std::panic::catch_unwind(|| { 8244 // Wait to make sure Windows boots up 8245 assert!(windows_guest.wait_for_boot()); 8246 8247 // Initially present disk device 8248 let disk_num = 1; 8249 assert_eq!(windows_guest.disk_count(), disk_num); 8250 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8251 8252 // Hotplug disk device 8253 let (cmd_success, cmd_output) = remote_command_w_output( 8254 &api_socket, 8255 "add-disk", 8256 Some(format!("path={disk},readonly=off").as_str()), 8257 ); 8258 assert!(cmd_success); 8259 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 8260 thread::sleep(std::time::Duration::new(5, 0)); 8261 // Online disk device 8262 windows_guest.disks_set_rw(); 8263 windows_guest.disks_online(); 8264 // Verify the device is on the system 8265 let disk_num = 2; 8266 assert_eq!(windows_guest.disk_count(), disk_num); 8267 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8268 8269 let data = "hello"; 8270 let fname = "d:\\world"; 8271 windows_guest.disk_file_put(fname, data); 8272 8273 // Unmount disk device 8274 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 8275 assert!(cmd_success); 8276 thread::sleep(std::time::Duration::new(5, 0)); 8277 // Verify the device has been removed 8278 let disk_num = 1; 8279 assert_eq!(windows_guest.disk_count(), disk_num); 8280 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8281 8282 // Remount and check the file exists with the expected contents 8283 let (cmd_success, _cmd_output) = remote_command_w_output( 8284 &api_socket, 8285 "add-disk", 8286 Some(format!("path={disk},readonly=off").as_str()), 8287 ); 8288 assert!(cmd_success); 8289 thread::sleep(std::time::Duration::new(5, 0)); 8290 let out = windows_guest.disk_file_read(fname); 8291 assert_eq!(data, out.trim()); 8292 8293 // Intentionally no unmount, it'll happen at shutdown. 8294 8295 windows_guest.shutdown(); 8296 }); 8297 8298 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8299 let _ = child.kill(); 8300 let output = child.wait_with_output().unwrap(); 8301 8302 let _ = child_dnsmasq.kill(); 8303 let _ = child_dnsmasq.wait(); 8304 8305 handle_child_output(r, &output); 8306 } 8307 8308 #[test] 8309 #[ignore = "See #6037"] 8310 #[cfg(not(feature = "mshv"))] 8311 #[cfg(not(target_arch = "aarch64"))] 8312 fn test_windows_guest_disk_hotplug_multi() { 8313 let windows_guest = WindowsGuest::new(); 8314 8315 let mut ovmf_path = dirs::home_dir().unwrap(); 8316 ovmf_path.push("workloads"); 8317 ovmf_path.push(OVMF_NAME); 8318 8319 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8320 let api_socket = temp_api_path(&tmp_dir); 8321 8322 let mut child = GuestCommand::new(windows_guest.guest()) 8323 .args(["--api-socket", &api_socket]) 8324 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8325 .args(["--memory", "size=2G"]) 8326 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8327 .args(["--serial", "tty"]) 8328 .args(["--console", "off"]) 8329 .default_disks() 8330 .default_net() 8331 .capture_output() 8332 .spawn() 8333 .unwrap(); 8334 8335 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8336 8337 // Predefined data to used at various test stages 8338 let disk_test_data: [[String; 4]; 2] = [ 8339 [ 8340 "_disk2".to_string(), 8341 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 8342 "d:\\world".to_string(), 8343 "hello".to_string(), 8344 ], 8345 [ 8346 "_disk3".to_string(), 8347 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 8348 "e:\\hello".to_string(), 8349 "world".to_string(), 8350 ], 8351 ]; 8352 8353 let r = std::panic::catch_unwind(|| { 8354 // Wait to make sure Windows boots up 8355 assert!(windows_guest.wait_for_boot()); 8356 8357 // Initially present disk device 8358 let disk_num = 1; 8359 assert_eq!(windows_guest.disk_count(), disk_num); 8360 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8361 8362 for it in &disk_test_data { 8363 let disk_id = it[0].as_str(); 8364 let disk = it[1].as_str(); 8365 // Hotplug disk device 8366 let (cmd_success, cmd_output) = remote_command_w_output( 8367 &api_socket, 8368 "add-disk", 8369 Some(format!("path={disk},readonly=off").as_str()), 8370 ); 8371 assert!(cmd_success); 8372 assert!(String::from_utf8_lossy(&cmd_output) 8373 .contains(format!("\"id\":\"{disk_id}\"").as_str())); 8374 thread::sleep(std::time::Duration::new(5, 0)); 8375 // Online disk devices 8376 windows_guest.disks_set_rw(); 8377 windows_guest.disks_online(); 8378 } 8379 // Verify the devices are on the system 8380 let disk_num = (disk_test_data.len() + 1) as u8; 8381 assert_eq!(windows_guest.disk_count(), disk_num); 8382 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8383 8384 // Put test data 8385 for it in &disk_test_data { 8386 let fname = it[2].as_str(); 8387 let data = it[3].as_str(); 8388 windows_guest.disk_file_put(fname, data); 8389 } 8390 8391 // Unmount disk devices 8392 for it in &disk_test_data { 8393 let disk_id = it[0].as_str(); 8394 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 8395 assert!(cmd_success); 8396 thread::sleep(std::time::Duration::new(5, 0)); 8397 } 8398 8399 // Verify the devices have been removed 8400 let disk_num = 1; 8401 assert_eq!(windows_guest.disk_count(), disk_num); 8402 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8403 8404 // Remount 8405 for it in &disk_test_data { 8406 let disk = it[1].as_str(); 8407 let (cmd_success, _cmd_output) = remote_command_w_output( 8408 &api_socket, 8409 "add-disk", 8410 Some(format!("path={disk},readonly=off").as_str()), 8411 ); 8412 assert!(cmd_success); 8413 thread::sleep(std::time::Duration::new(5, 0)); 8414 } 8415 8416 // Check the files exists with the expected contents 8417 for it in &disk_test_data { 8418 let fname = it[2].as_str(); 8419 let data = it[3].as_str(); 8420 let out = windows_guest.disk_file_read(fname); 8421 assert_eq!(data, out.trim()); 8422 } 8423 8424 // Intentionally no unmount, it'll happen at shutdown. 8425 8426 windows_guest.shutdown(); 8427 }); 8428 8429 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8430 let _ = child.kill(); 8431 let output = child.wait_with_output().unwrap(); 8432 8433 let _ = child_dnsmasq.kill(); 8434 let _ = child_dnsmasq.wait(); 8435 8436 handle_child_output(r, &output); 8437 } 8438 8439 #[test] 8440 #[cfg(not(feature = "mshv"))] 8441 #[cfg(not(target_arch = "aarch64"))] 8442 fn test_windows_guest_netdev_multi() { 8443 let windows_guest = WindowsGuest::new(); 8444 8445 let mut ovmf_path = dirs::home_dir().unwrap(); 8446 ovmf_path.push("workloads"); 8447 ovmf_path.push(OVMF_NAME); 8448 8449 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8450 let api_socket = temp_api_path(&tmp_dir); 8451 8452 let mut child = GuestCommand::new(windows_guest.guest()) 8453 .args(["--api-socket", &api_socket]) 8454 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8455 .args(["--memory", "size=4G"]) 8456 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8457 .args(["--serial", "tty"]) 8458 .args(["--console", "off"]) 8459 .default_disks() 8460 // The multi net dev config is borrowed from test_multiple_network_interfaces 8461 .args([ 8462 "--net", 8463 windows_guest.guest().default_net_string().as_str(), 8464 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 8465 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 8466 ]) 8467 .capture_output() 8468 .spawn() 8469 .unwrap(); 8470 8471 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8472 8473 let r = std::panic::catch_unwind(|| { 8474 // Wait to make sure Windows boots up 8475 assert!(windows_guest.wait_for_boot()); 8476 8477 let netdev_num = 3; 8478 assert_eq!(windows_guest.netdev_count(), netdev_num); 8479 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8480 8481 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 8482 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 8483 8484 windows_guest.shutdown(); 8485 }); 8486 8487 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8488 let _ = child.kill(); 8489 let output = child.wait_with_output().unwrap(); 8490 8491 let _ = child_dnsmasq.kill(); 8492 let _ = child_dnsmasq.wait(); 8493 8494 handle_child_output(r, &output); 8495 } 8496 } 8497 8498 #[cfg(target_arch = "x86_64")] 8499 mod sgx { 8500 use crate::*; 8501 8502 #[test] 8503 fn test_sgx() { 8504 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 8505 let jammy = UbuntuDiskConfig::new(jammy_image); 8506 let guest = Guest::new(Box::new(jammy)); 8507 8508 let mut child = GuestCommand::new(&guest) 8509 .args(["--cpus", "boot=1"]) 8510 .args(["--memory", "size=512M"]) 8511 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8512 .default_disks() 8513 .default_net() 8514 .args(["--sgx-epc", "id=epc0,size=64M"]) 8515 .capture_output() 8516 .spawn() 8517 .unwrap(); 8518 8519 let r = std::panic::catch_unwind(|| { 8520 guest.wait_vm_boot(None).unwrap(); 8521 8522 // Check if SGX is correctly detected in the guest. 8523 guest.check_sgx_support().unwrap(); 8524 8525 // Validate the SGX EPC section is 64MiB. 8526 assert_eq!( 8527 guest 8528 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 8529 .unwrap() 8530 .trim(), 8531 "0x0000000004000000" 8532 ); 8533 }); 8534 8535 let _ = child.kill(); 8536 let output = child.wait_with_output().unwrap(); 8537 8538 handle_child_output(r, &output); 8539 } 8540 } 8541 8542 #[cfg(target_arch = "x86_64")] 8543 mod vfio { 8544 use crate::*; 8545 8546 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 8547 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8548 let guest = Guest::new(Box::new(jammy)); 8549 let api_socket = temp_api_path(&guest.tmp_dir); 8550 8551 let mut child = GuestCommand::new(&guest) 8552 .args(["--cpus", "boot=4"]) 8553 .args([ 8554 "--memory", 8555 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(), 8556 ]) 8557 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8558 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8559 .args(["--api-socket", &api_socket]) 8560 .default_disks() 8561 .default_net() 8562 .capture_output() 8563 .spawn() 8564 .unwrap(); 8565 8566 let r = std::panic::catch_unwind(|| { 8567 guest.wait_vm_boot(None).unwrap(); 8568 8569 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8570 8571 guest.enable_memory_hotplug(); 8572 8573 // Add RAM to the VM 8574 let desired_ram = 6 << 30; 8575 resize_command(&api_socket, None, Some(desired_ram), None, None); 8576 thread::sleep(std::time::Duration::new(30, 0)); 8577 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8578 8579 // Check the VFIO device works when RAM is increased to 6GiB 8580 guest.check_nvidia_gpu(); 8581 }); 8582 8583 let _ = child.kill(); 8584 let output = child.wait_with_output().unwrap(); 8585 8586 handle_child_output(r, &output); 8587 } 8588 8589 #[test] 8590 fn test_nvidia_card_memory_hotplug_acpi() { 8591 test_nvidia_card_memory_hotplug("acpi") 8592 } 8593 8594 #[test] 8595 fn test_nvidia_card_memory_hotplug_virtio_mem() { 8596 test_nvidia_card_memory_hotplug("virtio-mem") 8597 } 8598 8599 #[test] 8600 fn test_nvidia_card_pci_hotplug() { 8601 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8602 let guest = Guest::new(Box::new(jammy)); 8603 let api_socket = temp_api_path(&guest.tmp_dir); 8604 8605 let mut child = GuestCommand::new(&guest) 8606 .args(["--cpus", "boot=4"]) 8607 .args(["--memory", "size=4G"]) 8608 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8609 .args(["--api-socket", &api_socket]) 8610 .default_disks() 8611 .default_net() 8612 .capture_output() 8613 .spawn() 8614 .unwrap(); 8615 8616 let r = std::panic::catch_unwind(|| { 8617 guest.wait_vm_boot(None).unwrap(); 8618 8619 // Hotplug the card to the VM 8620 let (cmd_success, cmd_output) = remote_command_w_output( 8621 &api_socket, 8622 "add-device", 8623 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 8624 ); 8625 assert!(cmd_success); 8626 assert!(String::from_utf8_lossy(&cmd_output) 8627 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 8628 8629 thread::sleep(std::time::Duration::new(10, 0)); 8630 8631 // Check the VFIO device works after hotplug 8632 guest.check_nvidia_gpu(); 8633 }); 8634 8635 let _ = child.kill(); 8636 let output = child.wait_with_output().unwrap(); 8637 8638 handle_child_output(r, &output); 8639 } 8640 8641 #[test] 8642 fn test_nvidia_card_reboot() { 8643 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8644 let guest = Guest::new(Box::new(jammy)); 8645 let api_socket = temp_api_path(&guest.tmp_dir); 8646 8647 let mut child = GuestCommand::new(&guest) 8648 .args(["--cpus", "boot=4"]) 8649 .args(["--memory", "size=4G"]) 8650 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8651 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8652 .args(["--api-socket", &api_socket]) 8653 .default_disks() 8654 .default_net() 8655 .capture_output() 8656 .spawn() 8657 .unwrap(); 8658 8659 let r = std::panic::catch_unwind(|| { 8660 guest.wait_vm_boot(None).unwrap(); 8661 8662 // Check the VFIO device works after boot 8663 guest.check_nvidia_gpu(); 8664 8665 guest.reboot_linux(0, None); 8666 8667 // Check the VFIO device works after reboot 8668 guest.check_nvidia_gpu(); 8669 }); 8670 8671 let _ = child.kill(); 8672 let output = child.wait_with_output().unwrap(); 8673 8674 handle_child_output(r, &output); 8675 } 8676 } 8677 8678 mod live_migration { 8679 use crate::*; 8680 8681 fn start_live_migration( 8682 migration_socket: &str, 8683 src_api_socket: &str, 8684 dest_api_socket: &str, 8685 local: bool, 8686 ) -> bool { 8687 // Start to receive migration from the destintion VM 8688 let mut receive_migration = Command::new(clh_command("ch-remote")) 8689 .args([ 8690 &format!("--api-socket={dest_api_socket}"), 8691 "receive-migration", 8692 &format! {"unix:{migration_socket}"}, 8693 ]) 8694 .stderr(Stdio::piped()) 8695 .stdout(Stdio::piped()) 8696 .spawn() 8697 .unwrap(); 8698 // Give it '1s' to make sure the 'migration_socket' file is properly created 8699 thread::sleep(std::time::Duration::new(1, 0)); 8700 // Start to send migration from the source VM 8701 8702 let mut args = [ 8703 format!("--api-socket={}", &src_api_socket), 8704 "send-migration".to_string(), 8705 format! {"unix:{migration_socket}"}, 8706 ] 8707 .to_vec(); 8708 8709 if local { 8710 args.insert(2, "--local".to_string()); 8711 } 8712 8713 let mut send_migration = Command::new(clh_command("ch-remote")) 8714 .args(&args) 8715 .stderr(Stdio::piped()) 8716 .stdout(Stdio::piped()) 8717 .spawn() 8718 .unwrap(); 8719 8720 // The 'send-migration' command should be executed successfully within the given timeout 8721 let send_success = if let Some(status) = send_migration 8722 .wait_timeout(std::time::Duration::from_secs(30)) 8723 .unwrap() 8724 { 8725 status.success() 8726 } else { 8727 false 8728 }; 8729 8730 if !send_success { 8731 let _ = send_migration.kill(); 8732 let output = send_migration.wait_with_output().unwrap(); 8733 eprintln!( 8734 "\n\n==== Start 'send_migration' output ==== \ 8735 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8736 \n\n==== End 'send_migration' output ====\n\n", 8737 String::from_utf8_lossy(&output.stdout), 8738 String::from_utf8_lossy(&output.stderr) 8739 ); 8740 } 8741 8742 // The 'receive-migration' command should be executed successfully within the given timeout 8743 let receive_success = if let Some(status) = receive_migration 8744 .wait_timeout(std::time::Duration::from_secs(30)) 8745 .unwrap() 8746 { 8747 status.success() 8748 } else { 8749 false 8750 }; 8751 8752 if !receive_success { 8753 let _ = receive_migration.kill(); 8754 let output = receive_migration.wait_with_output().unwrap(); 8755 eprintln!( 8756 "\n\n==== Start 'receive_migration' output ==== \ 8757 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8758 \n\n==== End 'receive_migration' output ====\n\n", 8759 String::from_utf8_lossy(&output.stdout), 8760 String::from_utf8_lossy(&output.stderr) 8761 ); 8762 } 8763 8764 send_success && receive_success 8765 } 8766 8767 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 8768 let mut src_vm = src_vm; 8769 let mut dest_vm = dest_vm; 8770 8771 let _ = src_vm.kill(); 8772 let src_output = src_vm.wait_with_output().unwrap(); 8773 eprintln!( 8774 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8775 String::from_utf8_lossy(&src_output.stdout) 8776 ); 8777 eprintln!( 8778 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8779 String::from_utf8_lossy(&src_output.stderr) 8780 ); 8781 let _ = dest_vm.kill(); 8782 let dest_output = dest_vm.wait_with_output().unwrap(); 8783 eprintln!( 8784 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8785 String::from_utf8_lossy(&dest_output.stdout) 8786 ); 8787 eprintln!( 8788 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8789 String::from_utf8_lossy(&dest_output.stderr) 8790 ); 8791 8792 if let Some(ovs_vm) = ovs_vm { 8793 let mut ovs_vm = ovs_vm; 8794 let _ = ovs_vm.kill(); 8795 let ovs_output = ovs_vm.wait_with_output().unwrap(); 8796 eprintln!( 8797 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 8798 String::from_utf8_lossy(&ovs_output.stdout) 8799 ); 8800 eprintln!( 8801 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 8802 String::from_utf8_lossy(&ovs_output.stderr) 8803 ); 8804 8805 cleanup_ovs_dpdk(); 8806 } 8807 8808 panic!("Test failed: {message}") 8809 } 8810 8811 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 8812 // same host. It ensures the following behaviors: 8813 // 1. The source VM is up and functional (including various virtio-devices are working properly); 8814 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 8815 // 3. The source VM terminated gracefully after live migration; 8816 // 4. The destination VM is functional (including various virtio-devices are working properly) after 8817 // live migration; 8818 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 8819 fn _test_live_migration(upgrade_test: bool, local: bool) { 8820 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8821 let guest = Guest::new(Box::new(focal)); 8822 let kernel_path = direct_kernel_boot_path(); 8823 let console_text = String::from("On a branch floating down river a cricket, singing."); 8824 let net_id = "net123"; 8825 let net_params = format!( 8826 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8827 net_id, guest.network.guest_mac, guest.network.host_ip 8828 ); 8829 8830 let memory_param: &[&str] = if local { 8831 &["--memory", "size=4G,shared=on"] 8832 } else { 8833 &["--memory", "size=4G"] 8834 }; 8835 8836 let boot_vcpus = 2; 8837 let max_vcpus = 4; 8838 8839 let pmem_temp_file = TempFile::new().unwrap(); 8840 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8841 std::process::Command::new("mkfs.ext4") 8842 .arg(pmem_temp_file.as_path()) 8843 .output() 8844 .expect("Expect creating disk image to succeed"); 8845 let pmem_path = String::from("/dev/pmem0"); 8846 8847 // Start the source VM 8848 let src_vm_path = if !upgrade_test { 8849 clh_command("cloud-hypervisor") 8850 } else { 8851 cloud_hypervisor_release_path() 8852 }; 8853 let src_api_socket = temp_api_path(&guest.tmp_dir); 8854 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8855 src_vm_cmd 8856 .args([ 8857 "--cpus", 8858 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8859 ]) 8860 .args(memory_param) 8861 .args(["--kernel", kernel_path.to_str().unwrap()]) 8862 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8863 .default_disks() 8864 .args(["--net", net_params.as_str()]) 8865 .args(["--api-socket", &src_api_socket]) 8866 .args([ 8867 "--pmem", 8868 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8869 ]); 8870 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8871 8872 // Start the destination VM 8873 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8874 dest_api_socket.push_str(".dest"); 8875 let mut dest_child = GuestCommand::new(&guest) 8876 .args(["--api-socket", &dest_api_socket]) 8877 .capture_output() 8878 .spawn() 8879 .unwrap(); 8880 8881 let r = std::panic::catch_unwind(|| { 8882 guest.wait_vm_boot(None).unwrap(); 8883 8884 // Make sure the source VM is functaionl 8885 // Check the number of vCPUs 8886 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8887 8888 // Check the guest RAM 8889 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8890 8891 // Check the guest virtio-devices, e.g. block, rng, console, and net 8892 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8893 8894 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8895 // to make sure that removing and adding back the virtio-net device does 8896 // not break the live-migration support for virtio-pci. 8897 #[cfg(target_arch = "x86_64")] 8898 { 8899 assert!(remote_command( 8900 &src_api_socket, 8901 "remove-device", 8902 Some(net_id), 8903 )); 8904 thread::sleep(std::time::Duration::new(10, 0)); 8905 8906 // Plug the virtio-net device again 8907 assert!(remote_command( 8908 &src_api_socket, 8909 "add-net", 8910 Some(net_params.as_str()), 8911 )); 8912 thread::sleep(std::time::Duration::new(10, 0)); 8913 } 8914 8915 // Start the live-migration 8916 let migration_socket = String::from( 8917 guest 8918 .tmp_dir 8919 .as_path() 8920 .join("live-migration.sock") 8921 .to_str() 8922 .unwrap(), 8923 ); 8924 8925 assert!( 8926 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8927 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8928 ); 8929 }); 8930 8931 // Check and report any errors occurred during the live-migration 8932 if r.is_err() { 8933 print_and_panic( 8934 src_child, 8935 dest_child, 8936 None, 8937 "Error occurred during live-migration", 8938 ); 8939 } 8940 8941 // Check the source vm has been terminated successful (give it '3s' to settle) 8942 thread::sleep(std::time::Duration::new(3, 0)); 8943 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8944 print_and_panic( 8945 src_child, 8946 dest_child, 8947 None, 8948 "source VM was not terminated successfully.", 8949 ); 8950 }; 8951 8952 // Post live-migration check to make sure the destination VM is funcational 8953 let r = std::panic::catch_unwind(|| { 8954 // Perform same checks to validate VM has been properly migrated 8955 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8956 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8957 8958 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8959 }); 8960 8961 // Clean-up the destination VM and make sure it terminated correctly 8962 let _ = dest_child.kill(); 8963 let dest_output = dest_child.wait_with_output().unwrap(); 8964 handle_child_output(r, &dest_output); 8965 8966 // Check the destination VM has the expected 'concole_text' from its output 8967 let r = std::panic::catch_unwind(|| { 8968 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8969 }); 8970 handle_child_output(r, &dest_output); 8971 } 8972 8973 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 8974 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8975 let guest = Guest::new(Box::new(focal)); 8976 let kernel_path = direct_kernel_boot_path(); 8977 let console_text = String::from("On a branch floating down river a cricket, singing."); 8978 let net_id = "net123"; 8979 let net_params = format!( 8980 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8981 net_id, guest.network.guest_mac, guest.network.host_ip 8982 ); 8983 8984 let memory_param: &[&str] = if local { 8985 &[ 8986 "--memory", 8987 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 8988 "--balloon", 8989 "size=0", 8990 ] 8991 } else { 8992 &[ 8993 "--memory", 8994 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 8995 "--balloon", 8996 "size=0", 8997 ] 8998 }; 8999 9000 let boot_vcpus = 2; 9001 let max_vcpus = 4; 9002 9003 let pmem_temp_file = TempFile::new().unwrap(); 9004 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9005 std::process::Command::new("mkfs.ext4") 9006 .arg(pmem_temp_file.as_path()) 9007 .output() 9008 .expect("Expect creating disk image to succeed"); 9009 let pmem_path = String::from("/dev/pmem0"); 9010 9011 // Start the source VM 9012 let src_vm_path = if !upgrade_test { 9013 clh_command("cloud-hypervisor") 9014 } else { 9015 cloud_hypervisor_release_path() 9016 }; 9017 let src_api_socket = temp_api_path(&guest.tmp_dir); 9018 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9019 src_vm_cmd 9020 .args([ 9021 "--cpus", 9022 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9023 ]) 9024 .args(memory_param) 9025 .args(["--kernel", kernel_path.to_str().unwrap()]) 9026 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9027 .default_disks() 9028 .args(["--net", net_params.as_str()]) 9029 .args(["--api-socket", &src_api_socket]) 9030 .args([ 9031 "--pmem", 9032 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9033 ]); 9034 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9035 9036 // Start the destination VM 9037 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9038 dest_api_socket.push_str(".dest"); 9039 let mut dest_child = GuestCommand::new(&guest) 9040 .args(["--api-socket", &dest_api_socket]) 9041 .capture_output() 9042 .spawn() 9043 .unwrap(); 9044 9045 let r = std::panic::catch_unwind(|| { 9046 guest.wait_vm_boot(None).unwrap(); 9047 9048 // Make sure the source VM is functaionl 9049 // Check the number of vCPUs 9050 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9051 9052 // Check the guest RAM 9053 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9054 // Increase the guest RAM 9055 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 9056 thread::sleep(std::time::Duration::new(5, 0)); 9057 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9058 // Use balloon to remove RAM from the VM 9059 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 9060 thread::sleep(std::time::Duration::new(5, 0)); 9061 let total_memory = guest.get_total_memory().unwrap_or_default(); 9062 assert!(total_memory > 4_800_000); 9063 assert!(total_memory < 5_760_000); 9064 9065 // Check the guest virtio-devices, e.g. block, rng, console, and net 9066 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9067 9068 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9069 // to make sure that removing and adding back the virtio-net device does 9070 // not break the live-migration support for virtio-pci. 9071 #[cfg(target_arch = "x86_64")] 9072 { 9073 assert!(remote_command( 9074 &src_api_socket, 9075 "remove-device", 9076 Some(net_id), 9077 )); 9078 thread::sleep(std::time::Duration::new(10, 0)); 9079 9080 // Plug the virtio-net device again 9081 assert!(remote_command( 9082 &src_api_socket, 9083 "add-net", 9084 Some(net_params.as_str()), 9085 )); 9086 thread::sleep(std::time::Duration::new(10, 0)); 9087 } 9088 9089 // Start the live-migration 9090 let migration_socket = String::from( 9091 guest 9092 .tmp_dir 9093 .as_path() 9094 .join("live-migration.sock") 9095 .to_str() 9096 .unwrap(), 9097 ); 9098 9099 assert!( 9100 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9101 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9102 ); 9103 }); 9104 9105 // Check and report any errors occurred during the live-migration 9106 if r.is_err() { 9107 print_and_panic( 9108 src_child, 9109 dest_child, 9110 None, 9111 "Error occurred during live-migration", 9112 ); 9113 } 9114 9115 // Check the source vm has been terminated successful (give it '3s' to settle) 9116 thread::sleep(std::time::Duration::new(3, 0)); 9117 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9118 print_and_panic( 9119 src_child, 9120 dest_child, 9121 None, 9122 "source VM was not terminated successfully.", 9123 ); 9124 }; 9125 9126 // Post live-migration check to make sure the destination VM is funcational 9127 let r = std::panic::catch_unwind(|| { 9128 // Perform same checks to validate VM has been properly migrated 9129 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9130 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9131 9132 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9133 9134 // Perform checks on guest RAM using balloon 9135 let total_memory = guest.get_total_memory().unwrap_or_default(); 9136 assert!(total_memory > 4_800_000); 9137 assert!(total_memory < 5_760_000); 9138 // Deflate balloon to restore entire RAM to the VM 9139 resize_command(&dest_api_socket, None, None, Some(0), None); 9140 thread::sleep(std::time::Duration::new(5, 0)); 9141 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9142 // Decrease guest RAM with virtio-mem 9143 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 9144 thread::sleep(std::time::Duration::new(5, 0)); 9145 let total_memory = guest.get_total_memory().unwrap_or_default(); 9146 assert!(total_memory > 4_800_000); 9147 assert!(total_memory < 5_760_000); 9148 }); 9149 9150 // Clean-up the destination VM and make sure it terminated correctly 9151 let _ = dest_child.kill(); 9152 let dest_output = dest_child.wait_with_output().unwrap(); 9153 handle_child_output(r, &dest_output); 9154 9155 // Check the destination VM has the expected 'concole_text' from its output 9156 let r = std::panic::catch_unwind(|| { 9157 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9158 }); 9159 handle_child_output(r, &dest_output); 9160 } 9161 9162 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 9163 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9164 let guest = Guest::new(Box::new(focal)); 9165 let kernel_path = direct_kernel_boot_path(); 9166 let console_text = String::from("On a branch floating down river a cricket, singing."); 9167 let net_id = "net123"; 9168 let net_params = format!( 9169 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9170 net_id, guest.network.guest_mac, guest.network.host_ip 9171 ); 9172 9173 let memory_param: &[&str] = if local { 9174 &[ 9175 "--memory", 9176 "size=0,hotplug_method=virtio-mem,shared=on", 9177 "--memory-zone", 9178 "id=mem0,size=1G,hotplug_size=4G,shared=on", 9179 "id=mem1,size=1G,hotplug_size=4G,shared=on", 9180 "id=mem2,size=2G,hotplug_size=4G,shared=on", 9181 "--numa", 9182 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9183 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9184 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9185 ] 9186 } else { 9187 &[ 9188 "--memory", 9189 "size=0,hotplug_method=virtio-mem", 9190 "--memory-zone", 9191 "id=mem0,size=1G,hotplug_size=4G", 9192 "id=mem1,size=1G,hotplug_size=4G", 9193 "id=mem2,size=2G,hotplug_size=4G", 9194 "--numa", 9195 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9196 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9197 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9198 ] 9199 }; 9200 9201 let boot_vcpus = 6; 9202 let max_vcpus = 12; 9203 9204 let pmem_temp_file = TempFile::new().unwrap(); 9205 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9206 std::process::Command::new("mkfs.ext4") 9207 .arg(pmem_temp_file.as_path()) 9208 .output() 9209 .expect("Expect creating disk image to succeed"); 9210 let pmem_path = String::from("/dev/pmem0"); 9211 9212 // Start the source VM 9213 let src_vm_path = if !upgrade_test { 9214 clh_command("cloud-hypervisor") 9215 } else { 9216 cloud_hypervisor_release_path() 9217 }; 9218 let src_api_socket = temp_api_path(&guest.tmp_dir); 9219 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9220 src_vm_cmd 9221 .args([ 9222 "--cpus", 9223 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9224 ]) 9225 .args(memory_param) 9226 .args(["--kernel", kernel_path.to_str().unwrap()]) 9227 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9228 .default_disks() 9229 .args(["--net", net_params.as_str()]) 9230 .args(["--api-socket", &src_api_socket]) 9231 .args([ 9232 "--pmem", 9233 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9234 ]); 9235 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9236 9237 // Start the destination VM 9238 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9239 dest_api_socket.push_str(".dest"); 9240 let mut dest_child = GuestCommand::new(&guest) 9241 .args(["--api-socket", &dest_api_socket]) 9242 .capture_output() 9243 .spawn() 9244 .unwrap(); 9245 9246 let r = std::panic::catch_unwind(|| { 9247 guest.wait_vm_boot(None).unwrap(); 9248 9249 // Make sure the source VM is functaionl 9250 // Check the number of vCPUs 9251 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9252 9253 // Check the guest RAM 9254 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 9255 9256 // Check the guest virtio-devices, e.g. block, rng, console, and net 9257 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9258 9259 // Check the NUMA parameters are applied correctly and resize 9260 // each zone to test the case where we migrate a VM with the 9261 // virtio-mem regions being used. 9262 { 9263 guest.check_numa_common( 9264 Some(&[960_000, 960_000, 1_920_000]), 9265 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9266 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9267 ); 9268 9269 // AArch64 currently does not support hotplug, and therefore we only 9270 // test hotplug-related function on x86_64 here. 9271 #[cfg(target_arch = "x86_64")] 9272 { 9273 guest.enable_memory_hotplug(); 9274 9275 // Resize every memory zone and check each associated NUMA node 9276 // has been assigned the right amount of memory. 9277 resize_zone_command(&src_api_socket, "mem0", "2G"); 9278 resize_zone_command(&src_api_socket, "mem1", "2G"); 9279 resize_zone_command(&src_api_socket, "mem2", "3G"); 9280 thread::sleep(std::time::Duration::new(5, 0)); 9281 9282 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 9283 } 9284 } 9285 9286 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9287 // to make sure that removing and adding back the virtio-net device does 9288 // not break the live-migration support for virtio-pci. 9289 #[cfg(target_arch = "x86_64")] 9290 { 9291 assert!(remote_command( 9292 &src_api_socket, 9293 "remove-device", 9294 Some(net_id), 9295 )); 9296 thread::sleep(std::time::Duration::new(10, 0)); 9297 9298 // Plug the virtio-net device again 9299 assert!(remote_command( 9300 &src_api_socket, 9301 "add-net", 9302 Some(net_params.as_str()), 9303 )); 9304 thread::sleep(std::time::Duration::new(10, 0)); 9305 } 9306 9307 // Start the live-migration 9308 let migration_socket = String::from( 9309 guest 9310 .tmp_dir 9311 .as_path() 9312 .join("live-migration.sock") 9313 .to_str() 9314 .unwrap(), 9315 ); 9316 9317 assert!( 9318 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9319 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9320 ); 9321 }); 9322 9323 // Check and report any errors occurred during the live-migration 9324 if r.is_err() { 9325 print_and_panic( 9326 src_child, 9327 dest_child, 9328 None, 9329 "Error occurred during live-migration", 9330 ); 9331 } 9332 9333 // Check the source vm has been terminated successful (give it '3s' to settle) 9334 thread::sleep(std::time::Duration::new(3, 0)); 9335 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9336 print_and_panic( 9337 src_child, 9338 dest_child, 9339 None, 9340 "source VM was not terminated successfully.", 9341 ); 9342 }; 9343 9344 // Post live-migration check to make sure the destination VM is funcational 9345 let r = std::panic::catch_unwind(|| { 9346 // Perform same checks to validate VM has been properly migrated 9347 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9348 #[cfg(target_arch = "x86_64")] 9349 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 9350 #[cfg(target_arch = "aarch64")] 9351 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9352 9353 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9354 9355 // Perform NUMA related checks 9356 { 9357 #[cfg(target_arch = "aarch64")] 9358 { 9359 guest.check_numa_common( 9360 Some(&[960_000, 960_000, 1_920_000]), 9361 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9362 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9363 ); 9364 } 9365 9366 // AArch64 currently does not support hotplug, and therefore we only 9367 // test hotplug-related function on x86_64 here. 9368 #[cfg(target_arch = "x86_64")] 9369 { 9370 guest.check_numa_common( 9371 Some(&[1_920_000, 1_920_000, 2_880_000]), 9372 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9373 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9374 ); 9375 9376 guest.enable_memory_hotplug(); 9377 9378 // Resize every memory zone and check each associated NUMA node 9379 // has been assigned the right amount of memory. 9380 resize_zone_command(&dest_api_socket, "mem0", "4G"); 9381 resize_zone_command(&dest_api_socket, "mem1", "4G"); 9382 resize_zone_command(&dest_api_socket, "mem2", "4G"); 9383 // Resize to the maximum amount of CPUs and check each NUMA 9384 // node has been assigned the right CPUs set. 9385 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 9386 thread::sleep(std::time::Duration::new(5, 0)); 9387 9388 guest.check_numa_common( 9389 Some(&[3_840_000, 3_840_000, 3_840_000]), 9390 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 9391 None, 9392 ); 9393 } 9394 } 9395 }); 9396 9397 // Clean-up the destination VM and make sure it terminated correctly 9398 let _ = dest_child.kill(); 9399 let dest_output = dest_child.wait_with_output().unwrap(); 9400 handle_child_output(r, &dest_output); 9401 9402 // Check the destination VM has the expected 'concole_text' from its output 9403 let r = std::panic::catch_unwind(|| { 9404 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9405 }); 9406 handle_child_output(r, &dest_output); 9407 } 9408 9409 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 9410 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9411 let guest = Guest::new(Box::new(focal)); 9412 let kernel_path = direct_kernel_boot_path(); 9413 let console_text = String::from("On a branch floating down river a cricket, singing."); 9414 let net_id = "net123"; 9415 let net_params = format!( 9416 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9417 net_id, guest.network.guest_mac, guest.network.host_ip 9418 ); 9419 9420 let memory_param: &[&str] = if local { 9421 &["--memory", "size=4G,shared=on"] 9422 } else { 9423 &["--memory", "size=4G"] 9424 }; 9425 9426 let boot_vcpus = 2; 9427 let max_vcpus = 4; 9428 9429 let pmem_temp_file = TempFile::new().unwrap(); 9430 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9431 std::process::Command::new("mkfs.ext4") 9432 .arg(pmem_temp_file.as_path()) 9433 .output() 9434 .expect("Expect creating disk image to succeed"); 9435 let pmem_path = String::from("/dev/pmem0"); 9436 9437 // Start the source VM 9438 let src_vm_path = if !upgrade_test { 9439 clh_command("cloud-hypervisor") 9440 } else { 9441 cloud_hypervisor_release_path() 9442 }; 9443 let src_api_socket = temp_api_path(&guest.tmp_dir); 9444 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9445 src_vm_cmd 9446 .args([ 9447 "--cpus", 9448 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9449 ]) 9450 .args(memory_param) 9451 .args(["--kernel", kernel_path.to_str().unwrap()]) 9452 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9453 .default_disks() 9454 .args(["--net", net_params.as_str()]) 9455 .args(["--api-socket", &src_api_socket]) 9456 .args([ 9457 "--pmem", 9458 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9459 ]) 9460 .args(["--watchdog"]); 9461 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9462 9463 // Start the destination VM 9464 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9465 dest_api_socket.push_str(".dest"); 9466 let mut dest_child = GuestCommand::new(&guest) 9467 .args(["--api-socket", &dest_api_socket]) 9468 .capture_output() 9469 .spawn() 9470 .unwrap(); 9471 9472 let r = std::panic::catch_unwind(|| { 9473 guest.wait_vm_boot(None).unwrap(); 9474 9475 // Make sure the source VM is functaionl 9476 // Check the number of vCPUs 9477 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9478 // Check the guest RAM 9479 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9480 // Check the guest virtio-devices, e.g. block, rng, console, and net 9481 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9482 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9483 // to make sure that removing and adding back the virtio-net device does 9484 // not break the live-migration support for virtio-pci. 9485 #[cfg(target_arch = "x86_64")] 9486 { 9487 assert!(remote_command( 9488 &src_api_socket, 9489 "remove-device", 9490 Some(net_id), 9491 )); 9492 thread::sleep(std::time::Duration::new(10, 0)); 9493 9494 // Plug the virtio-net device again 9495 assert!(remote_command( 9496 &src_api_socket, 9497 "add-net", 9498 Some(net_params.as_str()), 9499 )); 9500 thread::sleep(std::time::Duration::new(10, 0)); 9501 } 9502 9503 // Enable watchdog and ensure its functional 9504 let mut expected_reboot_count = 1; 9505 // Enable the watchdog with a 15s timeout 9506 enable_guest_watchdog(&guest, 15); 9507 // Reboot and check that systemd has activated the watchdog 9508 guest.ssh_command("sudo reboot").unwrap(); 9509 guest.wait_vm_boot(None).unwrap(); 9510 expected_reboot_count += 1; 9511 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9512 assert_eq!( 9513 guest 9514 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 9515 .unwrap() 9516 .trim() 9517 .parse::<u32>() 9518 .unwrap_or_default(), 9519 2 9520 ); 9521 // Allow some normal time to elapse to check we don't get spurious reboots 9522 thread::sleep(std::time::Duration::new(40, 0)); 9523 // Check no reboot 9524 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9525 9526 // Start the live-migration 9527 let migration_socket = String::from( 9528 guest 9529 .tmp_dir 9530 .as_path() 9531 .join("live-migration.sock") 9532 .to_str() 9533 .unwrap(), 9534 ); 9535 9536 assert!( 9537 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9538 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9539 ); 9540 }); 9541 9542 // Check and report any errors occurred during the live-migration 9543 if r.is_err() { 9544 print_and_panic( 9545 src_child, 9546 dest_child, 9547 None, 9548 "Error occurred during live-migration", 9549 ); 9550 } 9551 9552 // Check the source vm has been terminated successful (give it '3s' to settle) 9553 thread::sleep(std::time::Duration::new(3, 0)); 9554 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9555 print_and_panic( 9556 src_child, 9557 dest_child, 9558 None, 9559 "source VM was not terminated successfully.", 9560 ); 9561 }; 9562 9563 // Post live-migration check to make sure the destination VM is funcational 9564 let r = std::panic::catch_unwind(|| { 9565 // Perform same checks to validate VM has been properly migrated 9566 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9567 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9568 9569 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9570 9571 // Perform checks on watchdog 9572 let mut expected_reboot_count = 2; 9573 9574 // Allow some normal time to elapse to check we don't get spurious reboots 9575 thread::sleep(std::time::Duration::new(40, 0)); 9576 // Check no reboot 9577 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9578 9579 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 9580 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 9581 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 9582 guest.wait_vm_boot(Some(50)).unwrap(); 9583 // Check a reboot is triggered by the watchdog 9584 expected_reboot_count += 1; 9585 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9586 9587 #[cfg(target_arch = "x86_64")] 9588 { 9589 // Now pause the VM and remain offline for 30s 9590 assert!(remote_command(&dest_api_socket, "pause", None)); 9591 thread::sleep(std::time::Duration::new(30, 0)); 9592 assert!(remote_command(&dest_api_socket, "resume", None)); 9593 9594 // Check no reboot 9595 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9596 } 9597 }); 9598 9599 // Clean-up the destination VM and make sure it terminated correctly 9600 let _ = dest_child.kill(); 9601 let dest_output = dest_child.wait_with_output().unwrap(); 9602 handle_child_output(r, &dest_output); 9603 9604 // Check the destination VM has the expected 'concole_text' from its output 9605 let r = std::panic::catch_unwind(|| { 9606 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9607 }); 9608 handle_child_output(r, &dest_output); 9609 } 9610 9611 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 9612 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9613 let ovs_guest = Guest::new(Box::new(ovs_focal)); 9614 9615 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9616 let migration_guest = Guest::new(Box::new(migration_focal)); 9617 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 9618 9619 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 9620 let (mut ovs_child, mut src_child) = 9621 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 9622 9623 // Start the destination VM 9624 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 9625 dest_api_socket.push_str(".dest"); 9626 let mut dest_child = GuestCommand::new(&migration_guest) 9627 .args(["--api-socket", &dest_api_socket]) 9628 .capture_output() 9629 .spawn() 9630 .unwrap(); 9631 9632 let r = std::panic::catch_unwind(|| { 9633 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 9634 thread::sleep(std::time::Duration::new(1, 0)); 9635 9636 // Start the live-migration 9637 let migration_socket = String::from( 9638 migration_guest 9639 .tmp_dir 9640 .as_path() 9641 .join("live-migration.sock") 9642 .to_str() 9643 .unwrap(), 9644 ); 9645 9646 assert!( 9647 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9648 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9649 ); 9650 }); 9651 9652 // Check and report any errors occurred during the live-migration 9653 if r.is_err() { 9654 print_and_panic( 9655 src_child, 9656 dest_child, 9657 Some(ovs_child), 9658 "Error occurred during live-migration", 9659 ); 9660 } 9661 9662 // Check the source vm has been terminated successful (give it '3s' to settle) 9663 thread::sleep(std::time::Duration::new(3, 0)); 9664 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9665 print_and_panic( 9666 src_child, 9667 dest_child, 9668 Some(ovs_child), 9669 "source VM was not terminated successfully.", 9670 ); 9671 }; 9672 9673 // Post live-migration check to make sure the destination VM is funcational 9674 let r = std::panic::catch_unwind(|| { 9675 // Perform same checks to validate VM has been properly migrated 9676 // Spawn a new netcat listener in the OVS VM 9677 let guest_ip = ovs_guest.network.guest_ip.clone(); 9678 thread::spawn(move || { 9679 ssh_command_ip( 9680 "nc -l 12345", 9681 &guest_ip, 9682 DEFAULT_SSH_RETRIES, 9683 DEFAULT_SSH_TIMEOUT, 9684 ) 9685 .unwrap(); 9686 }); 9687 9688 // Wait for the server to be listening 9689 thread::sleep(std::time::Duration::new(5, 0)); 9690 9691 // And check the connection is still functional after live-migration 9692 migration_guest 9693 .ssh_command("nc -vz 172.100.0.1 12345") 9694 .unwrap(); 9695 }); 9696 9697 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 9698 let _ = dest_child.kill(); 9699 let _ = ovs_child.kill(); 9700 let dest_output = dest_child.wait_with_output().unwrap(); 9701 let ovs_output = ovs_child.wait_with_output().unwrap(); 9702 9703 cleanup_ovs_dpdk(); 9704 9705 handle_child_output(r, &dest_output); 9706 handle_child_output(Ok(()), &ovs_output); 9707 } 9708 9709 mod live_migration_parallel { 9710 use super::*; 9711 #[test] 9712 fn test_live_migration_basic() { 9713 _test_live_migration(false, false) 9714 } 9715 9716 #[test] 9717 fn test_live_migration_local() { 9718 _test_live_migration(false, true) 9719 } 9720 9721 #[test] 9722 #[cfg(not(feature = "mshv"))] 9723 fn test_live_migration_numa() { 9724 _test_live_migration_numa(false, false) 9725 } 9726 9727 #[test] 9728 #[cfg(not(feature = "mshv"))] 9729 fn test_live_migration_numa_local() { 9730 _test_live_migration_numa(false, true) 9731 } 9732 9733 #[test] 9734 fn test_live_migration_watchdog() { 9735 _test_live_migration_watchdog(false, false) 9736 } 9737 9738 #[test] 9739 fn test_live_migration_watchdog_local() { 9740 _test_live_migration_watchdog(false, true) 9741 } 9742 9743 #[test] 9744 fn test_live_migration_balloon() { 9745 _test_live_migration_balloon(false, false) 9746 } 9747 9748 #[test] 9749 fn test_live_migration_balloon_local() { 9750 _test_live_migration_balloon(false, true) 9751 } 9752 9753 #[test] 9754 #[ignore = "See #6134"] 9755 fn test_live_upgrade_basic() { 9756 _test_live_migration(true, false) 9757 } 9758 9759 #[test] 9760 #[ignore = "See #6134"] 9761 fn test_live_upgrade_local() { 9762 _test_live_migration(true, true) 9763 } 9764 9765 #[test] 9766 #[ignore = "See #6134"] 9767 #[cfg(not(feature = "mshv"))] 9768 fn test_live_upgrade_numa() { 9769 _test_live_migration_numa(true, false) 9770 } 9771 9772 #[test] 9773 #[ignore = "See #6134"] 9774 #[cfg(not(feature = "mshv"))] 9775 fn test_live_upgrade_numa_local() { 9776 _test_live_migration_numa(true, true) 9777 } 9778 9779 #[test] 9780 #[ignore = "See #6134"] 9781 fn test_live_upgrade_watchdog() { 9782 _test_live_migration_watchdog(true, false) 9783 } 9784 9785 #[test] 9786 #[ignore = "See #6134"] 9787 fn test_live_upgrade_watchdog_local() { 9788 _test_live_migration_watchdog(true, true) 9789 } 9790 9791 #[test] 9792 #[ignore = "See #6134"] 9793 fn test_live_upgrade_balloon() { 9794 _test_live_migration_balloon(true, false) 9795 } 9796 9797 #[test] 9798 #[ignore = "See #6134"] 9799 fn test_live_upgrade_balloon_local() { 9800 _test_live_migration_balloon(true, true) 9801 } 9802 } 9803 9804 mod live_migration_sequential { 9805 #[cfg(target_arch = "x86_64")] 9806 #[cfg(not(feature = "mshv"))] 9807 use super::*; 9808 9809 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 9810 #[test] 9811 #[ignore = "See #5532"] 9812 #[cfg(target_arch = "x86_64")] 9813 #[cfg(not(feature = "mshv"))] 9814 fn test_live_migration_ovs_dpdk() { 9815 _test_live_migration_ovs_dpdk(false, false); 9816 } 9817 9818 #[test] 9819 #[cfg(target_arch = "x86_64")] 9820 #[cfg(not(feature = "mshv"))] 9821 fn test_live_migration_ovs_dpdk_local() { 9822 _test_live_migration_ovs_dpdk(false, true); 9823 } 9824 9825 #[test] 9826 #[ignore = "See #5532"] 9827 #[cfg(target_arch = "x86_64")] 9828 #[cfg(not(feature = "mshv"))] 9829 fn test_live_upgrade_ovs_dpdk() { 9830 _test_live_migration_ovs_dpdk(true, false); 9831 } 9832 9833 #[test] 9834 #[ignore = "See #5532"] 9835 #[cfg(target_arch = "x86_64")] 9836 #[cfg(not(feature = "mshv"))] 9837 fn test_live_upgrade_ovs_dpdk_local() { 9838 _test_live_migration_ovs_dpdk(true, true); 9839 } 9840 } 9841 } 9842 9843 #[cfg(target_arch = "aarch64")] 9844 mod aarch64_acpi { 9845 use crate::*; 9846 9847 #[test] 9848 fn test_simple_launch_acpi() { 9849 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9850 9851 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 9852 let guest = Guest::new(disk_config); 9853 9854 let mut child = GuestCommand::new(&guest) 9855 .args(["--cpus", "boot=1"]) 9856 .args(["--memory", "size=512M"]) 9857 .args(["--kernel", edk2_path().to_str().unwrap()]) 9858 .default_disks() 9859 .default_net() 9860 .args(["--serial", "tty", "--console", "off"]) 9861 .capture_output() 9862 .spawn() 9863 .unwrap(); 9864 9865 let r = std::panic::catch_unwind(|| { 9866 guest.wait_vm_boot(Some(120)).unwrap(); 9867 9868 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 9869 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 9870 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 9871 }); 9872 9873 let _ = child.kill(); 9874 let output = child.wait_with_output().unwrap(); 9875 9876 handle_child_output(r, &output); 9877 }); 9878 } 9879 9880 #[test] 9881 fn test_guest_numa_nodes_acpi() { 9882 _test_guest_numa_nodes(true); 9883 } 9884 9885 #[test] 9886 fn test_cpu_topology_421_acpi() { 9887 test_cpu_topology(4, 2, 1, true); 9888 } 9889 9890 #[test] 9891 fn test_cpu_topology_142_acpi() { 9892 test_cpu_topology(1, 4, 2, true); 9893 } 9894 9895 #[test] 9896 fn test_cpu_topology_262_acpi() { 9897 test_cpu_topology(2, 6, 2, true); 9898 } 9899 9900 #[test] 9901 fn test_power_button_acpi() { 9902 _test_power_button(true); 9903 } 9904 9905 #[test] 9906 fn test_virtio_iommu() { 9907 _test_virtio_iommu(true) 9908 } 9909 } 9910 9911 mod rate_limiter { 9912 use super::*; 9913 9914 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 9915 // compared to given 'limit' rate. 9916 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 9917 let upper_limit = limit * (1_f64 + difference); 9918 let lower_limit = limit * (1_f64 - difference); 9919 9920 if measured > lower_limit && measured < upper_limit { 9921 return true; 9922 } 9923 9924 eprintln!( 9925 "\n\n==== Start 'check_rate_limit' failed ==== \ 9926 \n\nmeasured={measured}, , lower_limit={lower_limit}, upper_limit={upper_limit} \ 9927 \n\n==== End 'check_rate_limit' failed ====\n\n" 9928 ); 9929 9930 false 9931 } 9932 9933 fn _test_rate_limiter_net(rx: bool) { 9934 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9935 let guest = Guest::new(Box::new(focal)); 9936 9937 let test_timeout = 10; 9938 let num_queues = 2; 9939 let queue_size = 256; 9940 let bw_size = 10485760_u64; // bytes 9941 let bw_refill_time = 100; // ms 9942 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 9943 9944 let net_params = format!( 9945 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 9946 guest.network.guest_mac, 9947 guest.network.host_ip, 9948 num_queues, 9949 queue_size, 9950 bw_size, 9951 bw_refill_time, 9952 ); 9953 9954 let mut child = GuestCommand::new(&guest) 9955 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 9956 .args(["--memory", "size=4G"]) 9957 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9958 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9959 .default_disks() 9960 .args(["--net", net_params.as_str()]) 9961 .capture_output() 9962 .spawn() 9963 .unwrap(); 9964 9965 let r = std::panic::catch_unwind(|| { 9966 guest.wait_vm_boot(None).unwrap(); 9967 let measured_bps = 9968 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, true) 9969 .unwrap(); 9970 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 9971 }); 9972 9973 let _ = child.kill(); 9974 let output = child.wait_with_output().unwrap(); 9975 handle_child_output(r, &output); 9976 } 9977 9978 #[test] 9979 fn test_rate_limiter_net_rx() { 9980 _test_rate_limiter_net(true); 9981 } 9982 9983 #[test] 9984 fn test_rate_limiter_net_tx() { 9985 _test_rate_limiter_net(false); 9986 } 9987 9988 fn _test_rate_limiter_block(bandwidth: bool, num_queues: u32) { 9989 let test_timeout = 10; 9990 let fio_ops = FioOps::RandRW; 9991 9992 let bw_size = if bandwidth { 9993 10485760_u64 // bytes 9994 } else { 9995 100_u64 // I/O 9996 }; 9997 let bw_refill_time = 100; // ms 9998 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 9999 10000 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10001 let guest = Guest::new(Box::new(focal)); 10002 let api_socket = temp_api_path(&guest.tmp_dir); 10003 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10004 let blk_rate_limiter_test_img = 10005 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 10006 10007 // Create the test block image 10008 assert!(exec_host_command_output(&format!( 10009 "dd if=/dev/zero of={blk_rate_limiter_test_img} bs=1M count=1024" 10010 )) 10011 .status 10012 .success()); 10013 10014 let test_blk_params = if bandwidth { 10015 format!( 10016 "path={blk_rate_limiter_test_img},num_queues={num_queues},bw_size={bw_size},bw_refill_time={bw_refill_time}" 10017 ) 10018 } else { 10019 format!( 10020 "path={blk_rate_limiter_test_img},num_queues={num_queues},ops_size={bw_size},ops_refill_time={bw_refill_time}" 10021 ) 10022 }; 10023 10024 let mut child = GuestCommand::new(&guest) 10025 .args(["--cpus", &format!("boot={num_queues}")]) 10026 .args(["--memory", "size=4G"]) 10027 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10028 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10029 .args([ 10030 "--disk", 10031 format!( 10032 "path={}", 10033 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10034 ) 10035 .as_str(), 10036 format!( 10037 "path={}", 10038 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10039 ) 10040 .as_str(), 10041 test_blk_params.as_str(), 10042 ]) 10043 .default_net() 10044 .args(["--api-socket", &api_socket]) 10045 .capture_output() 10046 .spawn() 10047 .unwrap(); 10048 10049 let r = std::panic::catch_unwind(|| { 10050 guest.wait_vm_boot(None).unwrap(); 10051 10052 let fio_command = format!( 10053 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 10054 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10055 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10056 ); 10057 let output = guest.ssh_command(&fio_command).unwrap(); 10058 10059 // Parse fio output 10060 let measured_rate = if bandwidth { 10061 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 10062 } else { 10063 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 10064 }; 10065 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10066 }); 10067 10068 let _ = child.kill(); 10069 let output = child.wait_with_output().unwrap(); 10070 handle_child_output(r, &output); 10071 } 10072 10073 fn _test_rate_limiter_group_block(bandwidth: bool, num_queues: u32, num_disks: u32) { 10074 let test_timeout = 10; 10075 let fio_ops = FioOps::RandRW; 10076 10077 let bw_size = if bandwidth { 10078 10485760_u64 // bytes 10079 } else { 10080 100_u64 // I/O 10081 }; 10082 let bw_refill_time = 100; // ms 10083 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10084 10085 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10086 let guest = Guest::new(Box::new(focal)); 10087 let api_socket = temp_api_path(&guest.tmp_dir); 10088 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10089 10090 let rate_limit_group_arg = if bandwidth { 10091 format!("id=group0,bw_size={bw_size},bw_refill_time={bw_refill_time}") 10092 } else { 10093 format!("id=group0,ops_size={bw_size},ops_refill_time={bw_refill_time}") 10094 }; 10095 10096 let mut disk_args = vec![ 10097 "--disk".to_string(), 10098 format!( 10099 "path={}", 10100 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10101 ), 10102 format!( 10103 "path={}", 10104 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10105 ), 10106 ]; 10107 10108 for i in 0..num_disks { 10109 let test_img_path = String::from( 10110 test_img_dir 10111 .as_path() 10112 .join(format!("blk{}.img", i)) 10113 .to_str() 10114 .unwrap(), 10115 ); 10116 10117 assert!(exec_host_command_output(&format!( 10118 "dd if=/dev/zero of={test_img_path} bs=1M count=1024" 10119 )) 10120 .status 10121 .success()); 10122 10123 disk_args.push(format!( 10124 "path={test_img_path},num_queues={num_queues},rate_limit_group=group0" 10125 )); 10126 } 10127 10128 let mut child = GuestCommand::new(&guest) 10129 .args(["--cpus", &format!("boot={}", num_queues * num_disks)]) 10130 .args(["--memory", "size=4G"]) 10131 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10132 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10133 .args(["--rate-limit-group", &rate_limit_group_arg]) 10134 .args(disk_args) 10135 .default_net() 10136 .args(["--api-socket", &api_socket]) 10137 .capture_output() 10138 .spawn() 10139 .unwrap(); 10140 10141 let r = std::panic::catch_unwind(|| { 10142 guest.wait_vm_boot(None).unwrap(); 10143 10144 let mut fio_command = format!( 10145 "sudo fio --name=global --output-format=json \ 10146 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10147 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10148 ); 10149 10150 // Generate additional argument for each disk: 10151 // --name=job0 --filename=/dev/vdc \ 10152 // --name=job1 --filename=/dev/vdd \ 10153 // --name=job2 --filename=/dev/vde \ 10154 // ... 10155 for i in 0..num_disks { 10156 let c: char = 'c'; 10157 let arg = format!( 10158 " --name=job{i} --filename=/dev/vd{}", 10159 char::from_u32((c as u32) + i).unwrap() 10160 ); 10161 fio_command += &arg; 10162 } 10163 let output = guest.ssh_command(&fio_command).unwrap(); 10164 10165 // Parse fio output 10166 let measured_rate = if bandwidth { 10167 parse_fio_output(&output, &fio_ops, num_queues * num_disks).unwrap() 10168 } else { 10169 parse_fio_output_iops(&output, &fio_ops, num_queues * num_disks).unwrap() 10170 }; 10171 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10172 }); 10173 10174 let _ = child.kill(); 10175 let output = child.wait_with_output().unwrap(); 10176 handle_child_output(r, &output); 10177 } 10178 10179 #[test] 10180 fn test_rate_limiter_block_bandwidth() { 10181 _test_rate_limiter_block(true, 1); 10182 _test_rate_limiter_block(true, 2) 10183 } 10184 10185 #[test] 10186 fn test_rate_limiter_group_block_bandwidth() { 10187 _test_rate_limiter_group_block(true, 1, 1); 10188 _test_rate_limiter_group_block(true, 2, 1); 10189 _test_rate_limiter_group_block(true, 1, 2); 10190 _test_rate_limiter_group_block(true, 2, 2); 10191 } 10192 10193 #[test] 10194 fn test_rate_limiter_block_iops() { 10195 _test_rate_limiter_block(false, 1); 10196 _test_rate_limiter_block(false, 2); 10197 } 10198 10199 #[test] 10200 fn test_rate_limiter_group_block_iops() { 10201 _test_rate_limiter_group_block(false, 1, 1); 10202 _test_rate_limiter_group_block(false, 2, 1); 10203 _test_rate_limiter_group_block(false, 1, 2); 10204 _test_rate_limiter_group_block(false, 2, 2); 10205 } 10206 } 10207