1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use net_util::MacAddr; 14 use std::collections::HashMap; 15 use std::fs; 16 use std::io; 17 use std::io::BufRead; 18 use std::io::Read; 19 use std::io::Seek; 20 use std::io::Write; 21 use std::os::unix::io::AsRawFd; 22 use std::path::PathBuf; 23 use std::process::{Child, Command, Stdio}; 24 use std::string::String; 25 use std::sync::mpsc; 26 use std::sync::mpsc::Receiver; 27 use std::sync::Mutex; 28 use std::thread; 29 use test_infra::*; 30 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 31 use wait_timeout::ChildExt; 32 33 // Constant taken from the VMM crate. 34 const MAX_NUM_PCI_SEGMENTS: u16 = 96; 35 36 #[cfg(target_arch = "x86_64")] 37 mod x86_64 { 38 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 39 pub const JAMMY_NVIDIA_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-nvidia.raw"; 40 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 41 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 42 "focal-server-cloudimg-amd64-custom-20210609-0-backing.qcow2"; 43 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 44 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 45 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20230119-0.raw"; 46 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2022-amd64-2.raw"; 47 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 48 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 49 } 50 51 #[cfg(target_arch = "x86_64")] 52 use x86_64::*; 53 54 #[cfg(target_arch = "aarch64")] 55 mod aarch64 { 56 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 57 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 58 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 59 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 60 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 61 "focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2"; 62 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 63 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 64 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 65 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 66 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 67 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 68 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 69 } 70 71 #[cfg(target_arch = "aarch64")] 72 use aarch64::*; 73 74 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 75 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 76 77 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 78 79 // This enum exists to make it more convenient to 80 // implement test for both D-Bus and REST APIs. 81 enum TargetApi { 82 // API socket 83 HttpApi(String), 84 // well known service name, object path 85 DBusApi(String, String), 86 } 87 88 impl TargetApi { 89 fn new_http_api(tmp_dir: &TempDir) -> Self { 90 Self::HttpApi(temp_api_path(tmp_dir)) 91 } 92 93 fn new_dbus_api(tmp_dir: &TempDir) -> Self { 94 // `tmp_dir` is in the form of "/tmp/chXXXXXX" 95 // and we take the `chXXXXXX` part as a unique identifier for the guest 96 let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); 97 98 Self::DBusApi( 99 format!("org.cloudhypervisor.{id}"), 100 format!("/org/cloudhypervisor/{id}"), 101 ) 102 } 103 104 fn guest_args(&self) -> Vec<String> { 105 match self { 106 TargetApi::HttpApi(api_socket) => { 107 vec![format!("--api-socket={}", api_socket.as_str())] 108 } 109 TargetApi::DBusApi(service_name, object_path) => { 110 vec![ 111 format!("--dbus-service-name={}", service_name.as_str()), 112 format!("--dbus-object-path={}", object_path.as_str()), 113 ] 114 } 115 } 116 } 117 118 fn remote_args(&self) -> Vec<String> { 119 // `guest_args` and `remote_args` are consistent with each other 120 self.guest_args() 121 } 122 123 fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { 124 let mut cmd = Command::new(clh_command("ch-remote")); 125 cmd.args(self.remote_args()); 126 cmd.arg(command); 127 128 if let Some(arg) = arg { 129 cmd.arg(arg); 130 } 131 132 let output = cmd.output().unwrap(); 133 if output.status.success() { 134 true 135 } else { 136 eprintln!("Error running ch-remote command: {:?}", &cmd); 137 let stderr = String::from_utf8_lossy(&output.stderr); 138 eprintln!("stderr: {stderr}"); 139 false 140 } 141 } 142 } 143 144 // Start cloud-hypervisor with no VM parameters, only the API server running. 145 // From the API: Create a VM, boot it and check that it looks as expected. 146 fn _test_api_create_boot(target_api: TargetApi, guest: Guest) { 147 let mut child = GuestCommand::new(&guest) 148 .args(target_api.guest_args()) 149 .capture_output() 150 .spawn() 151 .unwrap(); 152 153 thread::sleep(std::time::Duration::new(1, 0)); 154 155 // Verify API server is running 156 assert!(target_api.remote_command("ping", None)); 157 158 // Create the VM first 159 let cpu_count: u8 = 4; 160 let request_body = guest.api_create_body( 161 cpu_count, 162 direct_kernel_boot_path().to_str().unwrap(), 163 DIRECT_KERNEL_BOOT_CMDLINE, 164 ); 165 166 let temp_config_path = guest.tmp_dir.as_path().join("config"); 167 std::fs::write(&temp_config_path, request_body).unwrap(); 168 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 169 170 assert!(target_api.remote_command("create", Some(create_config),)); 171 172 // Then boot it 173 assert!(target_api.remote_command("boot", None)); 174 thread::sleep(std::time::Duration::new(20, 0)); 175 176 let r = std::panic::catch_unwind(|| { 177 // Check that the VM booted as expected 178 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 179 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 180 }); 181 182 let _ = child.kill(); 183 let output = child.wait_with_output().unwrap(); 184 185 handle_child_output(r, &output); 186 } 187 188 // Start cloud-hypervisor with no VM parameters, only the API server running. 189 // From the API: Create a VM, boot it and check it can be shutdown and then 190 // booted again 191 fn _test_api_shutdown(target_api: TargetApi, guest: Guest) { 192 let mut child = GuestCommand::new(&guest) 193 .args(target_api.guest_args()) 194 .capture_output() 195 .spawn() 196 .unwrap(); 197 198 thread::sleep(std::time::Duration::new(1, 0)); 199 200 // Verify API server is running 201 assert!(target_api.remote_command("ping", None)); 202 203 // Create the VM first 204 let cpu_count: u8 = 4; 205 let request_body = guest.api_create_body( 206 cpu_count, 207 direct_kernel_boot_path().to_str().unwrap(), 208 DIRECT_KERNEL_BOOT_CMDLINE, 209 ); 210 211 let temp_config_path = guest.tmp_dir.as_path().join("config"); 212 std::fs::write(&temp_config_path, request_body).unwrap(); 213 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 214 215 let r = std::panic::catch_unwind(|| { 216 assert!(target_api.remote_command("create", Some(create_config))); 217 218 // Then boot it 219 assert!(target_api.remote_command("boot", None)); 220 221 guest.wait_vm_boot(None).unwrap(); 222 223 // Check that the VM booted as expected 224 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 225 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 226 227 // Sync and shutdown without powering off to prevent filesystem 228 // corruption. 229 guest.ssh_command("sync").unwrap(); 230 guest.ssh_command("sudo shutdown -H now").unwrap(); 231 232 // Wait for the guest to be fully shutdown 233 thread::sleep(std::time::Duration::new(20, 0)); 234 235 // Then shut it down 236 assert!(target_api.remote_command("shutdown", None)); 237 238 // Then boot it again 239 assert!(target_api.remote_command("boot", None)); 240 241 guest.wait_vm_boot(None).unwrap(); 242 243 // Check that the VM booted as expected 244 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 245 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 246 }); 247 248 let _ = child.kill(); 249 let output = child.wait_with_output().unwrap(); 250 251 handle_child_output(r, &output); 252 } 253 254 // Start cloud-hypervisor with no VM parameters, only the API server running. 255 // From the API: Create a VM, boot it and check it can be deleted and then recreated 256 // booted again. 257 fn _test_api_delete(target_api: TargetApi, guest: Guest) { 258 let mut child = GuestCommand::new(&guest) 259 .args(target_api.guest_args()) 260 .capture_output() 261 .spawn() 262 .unwrap(); 263 264 thread::sleep(std::time::Duration::new(1, 0)); 265 266 // Verify API server is running 267 assert!(target_api.remote_command("ping", None)); 268 269 // Create the VM first 270 let cpu_count: u8 = 4; 271 let request_body = guest.api_create_body( 272 cpu_count, 273 direct_kernel_boot_path().to_str().unwrap(), 274 DIRECT_KERNEL_BOOT_CMDLINE, 275 ); 276 let temp_config_path = guest.tmp_dir.as_path().join("config"); 277 std::fs::write(&temp_config_path, request_body).unwrap(); 278 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 279 280 let r = std::panic::catch_unwind(|| { 281 assert!(target_api.remote_command("create", Some(create_config))); 282 283 // Then boot it 284 assert!(target_api.remote_command("boot", None)); 285 286 guest.wait_vm_boot(None).unwrap(); 287 288 // Check that the VM booted as expected 289 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 290 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 291 292 // Sync and shutdown without powering off to prevent filesystem 293 // corruption. 294 guest.ssh_command("sync").unwrap(); 295 guest.ssh_command("sudo shutdown -H now").unwrap(); 296 297 // Wait for the guest to be fully shutdown 298 thread::sleep(std::time::Duration::new(20, 0)); 299 300 // Then delete it 301 assert!(target_api.remote_command("delete", None)); 302 303 assert!(target_api.remote_command("create", Some(create_config))); 304 305 // Then boot it again 306 assert!(target_api.remote_command("boot", None)); 307 308 guest.wait_vm_boot(None).unwrap(); 309 310 // Check that the VM booted as expected 311 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 312 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 313 }); 314 315 let _ = child.kill(); 316 let output = child.wait_with_output().unwrap(); 317 318 handle_child_output(r, &output); 319 } 320 321 // Start cloud-hypervisor with no VM parameters, only the API server running. 322 // From the API: Create a VM, boot it and check that it looks as expected. 323 // Then we pause the VM, check that it's no longer available. 324 // Finally we resume the VM and check that it's available. 325 fn _test_api_pause_resume(target_api: TargetApi, guest: Guest) { 326 let mut child = GuestCommand::new(&guest) 327 .args(target_api.guest_args()) 328 .capture_output() 329 .spawn() 330 .unwrap(); 331 332 thread::sleep(std::time::Duration::new(1, 0)); 333 334 // Verify API server is running 335 assert!(target_api.remote_command("ping", None)); 336 337 // Create the VM first 338 let cpu_count: u8 = 4; 339 let request_body = guest.api_create_body( 340 cpu_count, 341 direct_kernel_boot_path().to_str().unwrap(), 342 DIRECT_KERNEL_BOOT_CMDLINE, 343 ); 344 345 let temp_config_path = guest.tmp_dir.as_path().join("config"); 346 std::fs::write(&temp_config_path, request_body).unwrap(); 347 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 348 349 assert!(target_api.remote_command("create", Some(create_config))); 350 351 // Then boot it 352 assert!(target_api.remote_command("boot", None)); 353 thread::sleep(std::time::Duration::new(20, 0)); 354 355 let r = std::panic::catch_unwind(|| { 356 // Check that the VM booted as expected 357 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 358 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 359 360 // We now pause the VM 361 assert!(target_api.remote_command("pause", None)); 362 363 // Check pausing again fails 364 assert!(!target_api.remote_command("pause", None)); 365 366 thread::sleep(std::time::Duration::new(2, 0)); 367 368 // SSH into the VM should fail 369 assert!(ssh_command_ip( 370 "grep -c processor /proc/cpuinfo", 371 &guest.network.guest_ip, 372 2, 373 5 374 ) 375 .is_err()); 376 377 // Resume the VM 378 assert!(target_api.remote_command("resume", None)); 379 380 // Check resuming again fails 381 assert!(!target_api.remote_command("resume", None)); 382 383 thread::sleep(std::time::Duration::new(2, 0)); 384 385 // Now we should be able to SSH back in and get the right number of CPUs 386 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 387 }); 388 389 let _ = child.kill(); 390 let output = child.wait_with_output().unwrap(); 391 392 handle_child_output(r, &output); 393 } 394 395 fn _test_pty_interaction(pty_path: PathBuf) { 396 let mut cf = std::fs::OpenOptions::new() 397 .write(true) 398 .read(true) 399 .open(pty_path) 400 .unwrap(); 401 402 // Some dumb sleeps but we don't want to write 403 // before the console is up and we don't want 404 // to try and write the next line before the 405 // login process is ready. 406 thread::sleep(std::time::Duration::new(5, 0)); 407 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 408 thread::sleep(std::time::Duration::new(2, 0)); 409 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 410 thread::sleep(std::time::Duration::new(2, 0)); 411 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 412 thread::sleep(std::time::Duration::new(2, 0)); 413 414 // read pty and ensure they have a login shell 415 // some fairly hacky workarounds to avoid looping 416 // forever in case the channel is blocked getting output 417 let ptyc = pty_read(cf); 418 let mut empty = 0; 419 let mut prev = String::new(); 420 loop { 421 thread::sleep(std::time::Duration::new(2, 0)); 422 match ptyc.try_recv() { 423 Ok(line) => { 424 empty = 0; 425 prev = prev + &line; 426 if prev.contains("test_pty_console") { 427 break; 428 } 429 } 430 Err(mpsc::TryRecvError::Empty) => { 431 empty += 1; 432 assert!(empty <= 5, "No login on pty"); 433 } 434 _ => { 435 panic!("No login on pty") 436 } 437 } 438 } 439 } 440 441 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 442 let mut workload_path = dirs::home_dir().unwrap(); 443 workload_path.push("workloads"); 444 445 let mut virtiofsd_path = workload_path; 446 virtiofsd_path.push("virtiofsd"); 447 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 448 449 let virtiofsd_socket_path = 450 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 451 452 // Start the daemon 453 let child = Command::new(virtiofsd_path.as_str()) 454 .args(["--shared-dir", shared_dir]) 455 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 456 .args(["--cache", "never"]) 457 .spawn() 458 .unwrap(); 459 460 thread::sleep(std::time::Duration::new(10, 0)); 461 462 (child, virtiofsd_socket_path) 463 } 464 465 fn prepare_vubd( 466 tmp_dir: &TempDir, 467 blk_img: &str, 468 num_queues: usize, 469 rdonly: bool, 470 direct: bool, 471 ) -> (std::process::Child, String) { 472 let mut workload_path = dirs::home_dir().unwrap(); 473 workload_path.push("workloads"); 474 475 let mut blk_file_path = workload_path; 476 blk_file_path.push(blk_img); 477 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 478 479 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 480 481 // Start the daemon 482 let child = Command::new(clh_command("vhost_user_block")) 483 .args([ 484 "--block-backend", 485 format!( 486 "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" 487 ) 488 .as_str(), 489 ]) 490 .spawn() 491 .unwrap(); 492 493 thread::sleep(std::time::Duration::new(10, 0)); 494 495 (child, vubd_socket_path) 496 } 497 498 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 499 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 500 } 501 502 fn temp_api_path(tmp_dir: &TempDir) -> String { 503 String::from( 504 tmp_dir 505 .as_path() 506 .join("cloud-hypervisor.sock") 507 .to_str() 508 .unwrap(), 509 ) 510 } 511 512 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 513 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 514 } 515 516 // Creates the directory and returns the path. 517 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 518 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 519 std::fs::create_dir(&snapshot_dir).unwrap(); 520 snapshot_dir 521 } 522 523 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 524 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 525 vmcore_file 526 } 527 528 // Creates the path for direct kernel boot and return the path. 529 // For x86_64, this function returns the vmlinux kernel path. 530 // For AArch64, this function returns the PE kernel path. 531 fn direct_kernel_boot_path() -> PathBuf { 532 let mut workload_path = dirs::home_dir().unwrap(); 533 workload_path.push("workloads"); 534 535 let mut kernel_path = workload_path; 536 #[cfg(target_arch = "x86_64")] 537 kernel_path.push("vmlinux"); 538 #[cfg(target_arch = "aarch64")] 539 kernel_path.push("Image"); 540 541 kernel_path 542 } 543 544 fn edk2_path() -> PathBuf { 545 let mut workload_path = dirs::home_dir().unwrap(); 546 workload_path.push("workloads"); 547 let mut edk2_path = workload_path; 548 edk2_path.push(OVMF_NAME); 549 550 edk2_path 551 } 552 553 fn cloud_hypervisor_release_path() -> String { 554 let mut workload_path = dirs::home_dir().unwrap(); 555 workload_path.push("workloads"); 556 557 let mut ch_release_path = workload_path; 558 #[cfg(target_arch = "x86_64")] 559 ch_release_path.push("cloud-hypervisor-static"); 560 #[cfg(target_arch = "aarch64")] 561 ch_release_path.push("cloud-hypervisor-static-aarch64"); 562 563 ch_release_path.into_os_string().into_string().unwrap() 564 } 565 566 fn prepare_vhost_user_net_daemon( 567 tmp_dir: &TempDir, 568 ip: &str, 569 tap: Option<&str>, 570 mtu: Option<u16>, 571 num_queues: usize, 572 client_mode: bool, 573 ) -> (std::process::Command, String) { 574 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 575 576 // Start the daemon 577 let mut net_params = format!( 578 "ip={ip},mask=255.255.255.0,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" 579 ); 580 581 if let Some(tap) = tap { 582 net_params.push_str(format!(",tap={tap}").as_str()); 583 } 584 585 if let Some(mtu) = mtu { 586 net_params.push_str(format!(",mtu={mtu}").as_str()); 587 } 588 589 let mut command = Command::new(clh_command("vhost_user_net")); 590 command.args(["--net-backend", net_params.as_str()]); 591 592 (command, vunet_socket_path) 593 } 594 595 fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { 596 let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); 597 let swtpm_socket_path = String::from( 598 tmp_dir 599 .as_path() 600 .join("swtpm") 601 .join("swtpm.sock") 602 .to_str() 603 .unwrap(), 604 ); 605 std::fs::create_dir(&swtpm_tpm_dir).unwrap(); 606 607 let mut swtpm_command = Command::new("swtpm"); 608 let swtpm_args = [ 609 "socket", 610 "--tpmstate", 611 &format!("dir={swtpm_tpm_dir}"), 612 "--ctrl", 613 &format!("type=unixio,path={swtpm_socket_path}"), 614 "--flags", 615 "startup-clear", 616 "--tpm2", 617 ]; 618 swtpm_command.args(swtpm_args); 619 620 (swtpm_command, swtpm_socket_path) 621 } 622 623 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 624 let mut cmd = Command::new(clh_command("ch-remote")); 625 cmd.args([&format!("--api-socket={api_socket}"), command]); 626 627 if let Some(arg) = arg { 628 cmd.arg(arg); 629 } 630 let output = cmd.output().unwrap(); 631 if output.status.success() { 632 true 633 } else { 634 eprintln!("Error running ch-remote command: {:?}", &cmd); 635 let stderr = String::from_utf8_lossy(&output.stderr); 636 eprintln!("stderr: {stderr}"); 637 false 638 } 639 } 640 641 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 642 let mut cmd = Command::new(clh_command("ch-remote")); 643 cmd.args([&format!("--api-socket={api_socket}"), command]); 644 645 if let Some(arg) = arg { 646 cmd.arg(arg); 647 } 648 649 let output = cmd.output().expect("Failed to launch ch-remote"); 650 651 (output.status.success(), output.stdout) 652 } 653 654 fn resize_command( 655 api_socket: &str, 656 desired_vcpus: Option<u8>, 657 desired_ram: Option<usize>, 658 desired_balloon: Option<usize>, 659 event_file: Option<&str>, 660 ) -> bool { 661 let mut cmd = Command::new(clh_command("ch-remote")); 662 cmd.args([&format!("--api-socket={api_socket}"), "resize"]); 663 664 if let Some(desired_vcpus) = desired_vcpus { 665 cmd.arg(format!("--cpus={desired_vcpus}")); 666 } 667 668 if let Some(desired_ram) = desired_ram { 669 cmd.arg(format!("--memory={desired_ram}")); 670 } 671 672 if let Some(desired_balloon) = desired_balloon { 673 cmd.arg(format!("--balloon={desired_balloon}")); 674 } 675 676 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 677 678 if let Some(event_path) = event_file { 679 let latest_events = [ 680 &MetaEvent { 681 event: "resizing".to_string(), 682 device_id: None, 683 }, 684 &MetaEvent { 685 event: "resized".to_string(), 686 device_id: None, 687 }, 688 ]; 689 // See: #5938 690 thread::sleep(std::time::Duration::new(1, 0)); 691 assert!(check_latest_events_exact(&latest_events, event_path)); 692 } 693 694 ret 695 } 696 697 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 698 let mut cmd = Command::new(clh_command("ch-remote")); 699 cmd.args([ 700 &format!("--api-socket={api_socket}"), 701 "resize-zone", 702 &format!("--id={id}"), 703 &format!("--size={desired_size}"), 704 ]); 705 706 cmd.status().expect("Failed to launch ch-remote").success() 707 } 708 709 // setup OVS-DPDK bridge and ports 710 fn setup_ovs_dpdk() { 711 // setup OVS-DPDK 712 assert!(exec_host_command_status("service openvswitch-switch start").success()); 713 assert!(exec_host_command_status("ovs-vsctl init").success()); 714 assert!( 715 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 716 .success() 717 ); 718 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 719 720 // Create OVS-DPDK bridge and ports 721 assert!(exec_host_command_status( 722 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 723 ) 724 .success()); 725 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 726 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 727 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 728 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 729 } 730 fn cleanup_ovs_dpdk() { 731 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 732 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 733 } 734 // Setup two guests and ensure they are connected through ovs-dpdk 735 fn setup_ovs_dpdk_guests( 736 guest1: &Guest, 737 guest2: &Guest, 738 api_socket: &str, 739 release_binary: bool, 740 ) -> (Child, Child) { 741 setup_ovs_dpdk(); 742 743 let clh_path = if !release_binary { 744 clh_command("cloud-hypervisor") 745 } else { 746 cloud_hypervisor_release_path() 747 }; 748 749 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 750 .args(["--cpus", "boot=2"]) 751 .args(["--memory", "size=0,shared=on"]) 752 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 753 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 754 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 755 .default_disks() 756 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 757 .capture_output() 758 .spawn() 759 .unwrap(); 760 761 #[cfg(target_arch = "x86_64")] 762 let guest_net_iface = "ens5"; 763 #[cfg(target_arch = "aarch64")] 764 let guest_net_iface = "enp0s5"; 765 766 let r = std::panic::catch_unwind(|| { 767 guest1.wait_vm_boot(None).unwrap(); 768 769 guest1 770 .ssh_command(&format!( 771 "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" 772 )) 773 .unwrap(); 774 guest1 775 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 776 .unwrap(); 777 778 let guest_ip = guest1.network.guest_ip.clone(); 779 thread::spawn(move || { 780 ssh_command_ip( 781 "nc -l 12345", 782 &guest_ip, 783 DEFAULT_SSH_RETRIES, 784 DEFAULT_SSH_TIMEOUT, 785 ) 786 .unwrap(); 787 }); 788 }); 789 if r.is_err() { 790 cleanup_ovs_dpdk(); 791 792 let _ = child1.kill(); 793 let output = child1.wait_with_output().unwrap(); 794 handle_child_output(r, &output); 795 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 796 } 797 798 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 799 .args(["--api-socket", api_socket]) 800 .args(["--cpus", "boot=2"]) 801 .args(["--memory", "size=0,shared=on"]) 802 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 803 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 804 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 805 .default_disks() 806 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 807 .capture_output() 808 .spawn() 809 .unwrap(); 810 811 let r = std::panic::catch_unwind(|| { 812 guest2.wait_vm_boot(None).unwrap(); 813 814 guest2 815 .ssh_command(&format!( 816 "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" 817 )) 818 .unwrap(); 819 guest2 820 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 821 .unwrap(); 822 823 // Check the connection works properly between the two VMs 824 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 825 }); 826 if r.is_err() { 827 cleanup_ovs_dpdk(); 828 829 let _ = child1.kill(); 830 let _ = child2.kill(); 831 let output = child2.wait_with_output().unwrap(); 832 handle_child_output(r, &output); 833 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 834 } 835 836 (child1, child2) 837 } 838 839 enum FwType { 840 Ovmf, 841 RustHypervisorFirmware, 842 } 843 844 fn fw_path(_fw_type: FwType) -> String { 845 let mut workload_path = dirs::home_dir().unwrap(); 846 workload_path.push("workloads"); 847 848 let mut fw_path = workload_path; 849 #[cfg(target_arch = "aarch64")] 850 fw_path.push("CLOUDHV_EFI.fd"); 851 #[cfg(target_arch = "x86_64")] 852 { 853 match _fw_type { 854 FwType::Ovmf => fw_path.push(OVMF_NAME), 855 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 856 } 857 } 858 859 fw_path.to_str().unwrap().to_string() 860 } 861 862 #[derive(Debug)] 863 struct MetaEvent { 864 event: String, 865 device_id: Option<String>, 866 } 867 868 impl MetaEvent { 869 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 870 let mut matched = false; 871 if v["event"].as_str().unwrap() == self.event { 872 if let Some(device_id) = &self.device_id { 873 if v["properties"]["id"].as_str().unwrap() == device_id { 874 matched = true 875 } 876 } else { 877 matched = true; 878 } 879 } 880 matched 881 } 882 } 883 884 // Parse the event_monitor file based on the format that each event 885 // is followed by a double newline 886 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 887 let content = fs::read(event_file).unwrap(); 888 let mut ret = Vec::new(); 889 for entry in String::from_utf8_lossy(&content) 890 .trim() 891 .split("\n\n") 892 .collect::<Vec<&str>>() 893 { 894 ret.push(serde_json::from_str(entry).unwrap()); 895 } 896 897 ret 898 } 899 900 // Return true if all events from the input 'expected_events' are matched sequentially 901 // with events from the 'event_file' 902 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 903 let json_events = parse_event_file(event_file); 904 let len = expected_events.len(); 905 let mut idx = 0; 906 for e in &json_events { 907 if idx == len { 908 break; 909 } 910 if expected_events[idx].match_with_json_event(e) { 911 idx += 1; 912 } 913 } 914 915 let ret = idx == len; 916 917 if !ret { 918 eprintln!( 919 "\n\n==== Start 'check_sequential_events' failed ==== \ 920 \n\nexpected_events={:?}\nactual_events={:?} \ 921 \n\n==== End 'check_sequential_events' failed ====", 922 expected_events, json_events, 923 ); 924 } 925 926 ret 927 } 928 929 // Return true if all events from the input 'expected_events' are matched exactly 930 // with events from the 'event_file' 931 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 932 let json_events = parse_event_file(event_file); 933 assert!(expected_events.len() <= json_events.len()); 934 let json_events = &json_events[..expected_events.len()]; 935 936 for (idx, e) in json_events.iter().enumerate() { 937 if !expected_events[idx].match_with_json_event(e) { 938 eprintln!( 939 "\n\n==== Start 'check_sequential_events_exact' failed ==== \ 940 \n\nexpected_events={:?}\nactual_events={:?} \ 941 \n\n==== End 'check_sequential_events_exact' failed ====", 942 expected_events, json_events, 943 ); 944 945 return false; 946 } 947 } 948 949 true 950 } 951 952 // Return true if events from the input 'latest_events' are matched exactly 953 // with the most recent events from the 'event_file' 954 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 955 let json_events = parse_event_file(event_file); 956 assert!(latest_events.len() <= json_events.len()); 957 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 958 959 for (idx, e) in json_events.iter().enumerate() { 960 if !latest_events[idx].match_with_json_event(e) { 961 eprintln!( 962 "\n\n==== Start 'check_latest_events_exact' failed ==== \ 963 \n\nexpected_events={:?}\nactual_events={:?} \ 964 \n\n==== End 'check_latest_events_exact' failed ====", 965 latest_events, json_events, 966 ); 967 968 return false; 969 } 970 } 971 972 true 973 } 974 975 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 976 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 977 let guest = Guest::new(Box::new(focal)); 978 let total_vcpus = threads_per_core * cores_per_package * packages; 979 let direct_kernel_boot_path = direct_kernel_boot_path(); 980 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 981 let fw_path = fw_path(FwType::RustHypervisorFirmware); 982 if use_fw { 983 kernel_path = fw_path.as_str(); 984 } 985 986 let mut child = GuestCommand::new(&guest) 987 .args([ 988 "--cpus", 989 &format!( 990 "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" 991 ), 992 ]) 993 .args(["--memory", "size=512M"]) 994 .args(["--kernel", kernel_path]) 995 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 996 .default_disks() 997 .default_net() 998 .capture_output() 999 .spawn() 1000 .unwrap(); 1001 1002 let r = std::panic::catch_unwind(|| { 1003 guest.wait_vm_boot(None).unwrap(); 1004 assert_eq!( 1005 guest.get_cpu_count().unwrap_or_default(), 1006 u32::from(total_vcpus) 1007 ); 1008 assert_eq!( 1009 guest 1010 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1011 .unwrap() 1012 .trim() 1013 .parse::<u8>() 1014 .unwrap_or(0), 1015 threads_per_core 1016 ); 1017 1018 assert_eq!( 1019 guest 1020 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1021 .unwrap() 1022 .trim() 1023 .parse::<u8>() 1024 .unwrap_or(0), 1025 cores_per_package 1026 ); 1027 1028 assert_eq!( 1029 guest 1030 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1031 .unwrap() 1032 .trim() 1033 .parse::<u8>() 1034 .unwrap_or(0), 1035 packages 1036 ); 1037 1038 #[cfg(target_arch = "x86_64")] 1039 { 1040 let mut cpu_id = 0; 1041 for package_id in 0..packages { 1042 for core_id in 0..cores_per_package { 1043 for _ in 0..threads_per_core { 1044 assert_eq!( 1045 guest 1046 .ssh_command(&format!("cat /sys/devices/system/cpu/cpu{cpu_id}/topology/physical_package_id")) 1047 .unwrap() 1048 .trim() 1049 .parse::<u8>() 1050 .unwrap_or(0), 1051 package_id 1052 ); 1053 1054 assert_eq!( 1055 guest 1056 .ssh_command(&format!( 1057 "cat /sys/devices/system/cpu/cpu{cpu_id}/topology/core_id" 1058 )) 1059 .unwrap() 1060 .trim() 1061 .parse::<u8>() 1062 .unwrap_or(0), 1063 core_id 1064 ); 1065 1066 cpu_id += 1; 1067 } 1068 } 1069 } 1070 } 1071 }); 1072 1073 let _ = child.kill(); 1074 let output = child.wait_with_output().unwrap(); 1075 1076 handle_child_output(r, &output); 1077 } 1078 1079 #[allow(unused_variables)] 1080 fn _test_guest_numa_nodes(acpi: bool) { 1081 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1082 let guest = Guest::new(Box::new(focal)); 1083 let api_socket = temp_api_path(&guest.tmp_dir); 1084 #[cfg(target_arch = "x86_64")] 1085 let kernel_path = direct_kernel_boot_path(); 1086 #[cfg(target_arch = "aarch64")] 1087 let kernel_path = if acpi { 1088 edk2_path() 1089 } else { 1090 direct_kernel_boot_path() 1091 }; 1092 1093 let mut child = GuestCommand::new(&guest) 1094 .args(["--cpus", "boot=6,max=12"]) 1095 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 1096 .args([ 1097 "--memory-zone", 1098 "id=mem0,size=1G,hotplug_size=3G", 1099 "id=mem1,size=2G,hotplug_size=3G", 1100 "id=mem2,size=3G,hotplug_size=3G", 1101 ]) 1102 .args([ 1103 "--numa", 1104 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 1105 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 1106 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 1107 ]) 1108 .args(["--kernel", kernel_path.to_str().unwrap()]) 1109 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1110 .args(["--api-socket", &api_socket]) 1111 .capture_output() 1112 .default_disks() 1113 .default_net() 1114 .spawn() 1115 .unwrap(); 1116 1117 let r = std::panic::catch_unwind(|| { 1118 guest.wait_vm_boot(None).unwrap(); 1119 1120 guest.check_numa_common( 1121 Some(&[960_000, 1_920_000, 2_880_000]), 1122 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 1123 Some(&["10 15 20", "20 10 25", "25 30 10"]), 1124 ); 1125 1126 // AArch64 currently does not support hotplug, and therefore we only 1127 // test hotplug-related function on x86_64 here. 1128 #[cfg(target_arch = "x86_64")] 1129 { 1130 guest.enable_memory_hotplug(); 1131 1132 // Resize every memory zone and check each associated NUMA node 1133 // has been assigned the right amount of memory. 1134 resize_zone_command(&api_socket, "mem0", "4G"); 1135 resize_zone_command(&api_socket, "mem1", "4G"); 1136 resize_zone_command(&api_socket, "mem2", "4G"); 1137 // Resize to the maximum amount of CPUs and check each NUMA 1138 // node has been assigned the right CPUs set. 1139 resize_command(&api_socket, Some(12), None, None, None); 1140 thread::sleep(std::time::Duration::new(5, 0)); 1141 1142 guest.check_numa_common( 1143 Some(&[3_840_000, 3_840_000, 3_840_000]), 1144 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 1145 None, 1146 ); 1147 } 1148 }); 1149 1150 let _ = child.kill(); 1151 let output = child.wait_with_output().unwrap(); 1152 1153 handle_child_output(r, &output); 1154 } 1155 1156 #[allow(unused_variables)] 1157 fn _test_power_button(acpi: bool) { 1158 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1159 let guest = Guest::new(Box::new(focal)); 1160 let mut cmd = GuestCommand::new(&guest); 1161 let api_socket = temp_api_path(&guest.tmp_dir); 1162 1163 #[cfg(target_arch = "x86_64")] 1164 let kernel_path = direct_kernel_boot_path(); 1165 #[cfg(target_arch = "aarch64")] 1166 let kernel_path = if acpi { 1167 edk2_path() 1168 } else { 1169 direct_kernel_boot_path() 1170 }; 1171 1172 cmd.args(["--cpus", "boot=1"]) 1173 .args(["--memory", "size=512M"]) 1174 .args(["--kernel", kernel_path.to_str().unwrap()]) 1175 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1176 .capture_output() 1177 .default_disks() 1178 .default_net() 1179 .args(["--api-socket", &api_socket]); 1180 1181 let child = cmd.spawn().unwrap(); 1182 1183 let r = std::panic::catch_unwind(|| { 1184 guest.wait_vm_boot(None).unwrap(); 1185 assert!(remote_command(&api_socket, "power-button", None)); 1186 }); 1187 1188 let output = child.wait_with_output().unwrap(); 1189 assert!(output.status.success()); 1190 handle_child_output(r, &output); 1191 } 1192 1193 type PrepareNetDaemon = dyn Fn( 1194 &TempDir, 1195 &str, 1196 Option<&str>, 1197 Option<u16>, 1198 usize, 1199 bool, 1200 ) -> (std::process::Command, String); 1201 1202 fn test_vhost_user_net( 1203 tap: Option<&str>, 1204 num_queues: usize, 1205 prepare_daemon: &PrepareNetDaemon, 1206 generate_host_mac: bool, 1207 client_mode_daemon: bool, 1208 ) { 1209 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1210 let guest = Guest::new(Box::new(focal)); 1211 let api_socket = temp_api_path(&guest.tmp_dir); 1212 1213 let kernel_path = direct_kernel_boot_path(); 1214 1215 let host_mac = if generate_host_mac { 1216 Some(MacAddr::local_random()) 1217 } else { 1218 None 1219 }; 1220 1221 let mtu = Some(3000); 1222 1223 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 1224 &guest.tmp_dir, 1225 &guest.network.host_ip, 1226 tap, 1227 mtu, 1228 num_queues, 1229 client_mode_daemon, 1230 ); 1231 1232 let net_params = format!( 1233 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 1234 guest.network.guest_mac, 1235 vunet_socket_path, 1236 num_queues, 1237 if let Some(host_mac) = host_mac { 1238 format!(",host_mac={host_mac}") 1239 } else { 1240 "".to_owned() 1241 }, 1242 if client_mode_daemon { 1243 "server" 1244 } else { 1245 "client" 1246 }, 1247 ); 1248 1249 let mut ch_command = GuestCommand::new(&guest); 1250 ch_command 1251 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 1252 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1253 .args(["--kernel", kernel_path.to_str().unwrap()]) 1254 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1255 .default_disks() 1256 .args(["--net", net_params.as_str()]) 1257 .args(["--api-socket", &api_socket]) 1258 .capture_output(); 1259 1260 let mut daemon_child: std::process::Child; 1261 let mut child: std::process::Child; 1262 1263 if client_mode_daemon { 1264 child = ch_command.spawn().unwrap(); 1265 // Make sure the VMM is waiting for the backend to connect 1266 thread::sleep(std::time::Duration::new(10, 0)); 1267 daemon_child = daemon_command.spawn().unwrap(); 1268 } else { 1269 daemon_child = daemon_command.spawn().unwrap(); 1270 // Make sure the backend is waiting for the VMM to connect 1271 thread::sleep(std::time::Duration::new(10, 0)); 1272 child = ch_command.spawn().unwrap(); 1273 } 1274 1275 let r = std::panic::catch_unwind(|| { 1276 guest.wait_vm_boot(None).unwrap(); 1277 1278 if let Some(tap_name) = tap { 1279 let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); 1280 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1281 } 1282 1283 if let Some(host_mac) = tap { 1284 let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); 1285 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1286 } 1287 1288 #[cfg(target_arch = "aarch64")] 1289 let iface = "enp0s4"; 1290 #[cfg(target_arch = "x86_64")] 1291 let iface = "ens4"; 1292 1293 assert_eq!( 1294 guest 1295 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 1296 .unwrap() 1297 .trim(), 1298 "3000" 1299 ); 1300 1301 // 1 network interface + default localhost ==> 2 interfaces 1302 // It's important to note that this test is fully exercising the 1303 // vhost-user-net implementation and the associated backend since 1304 // it does not define any --net network interface. That means all 1305 // the ssh communication in that test happens through the network 1306 // interface backed by vhost-user-net. 1307 assert_eq!( 1308 guest 1309 .ssh_command("ip -o link | wc -l") 1310 .unwrap() 1311 .trim() 1312 .parse::<u32>() 1313 .unwrap_or_default(), 1314 2 1315 ); 1316 1317 // The following pci devices will appear on guest with PCI-MSI 1318 // interrupt vectors assigned. 1319 // 1 virtio-console with 3 vectors: config, Rx, Tx 1320 // 1 virtio-blk with 2 vectors: config, Request 1321 // 1 virtio-blk with 2 vectors: config, Request 1322 // 1 virtio-rng with 2 vectors: config, Request 1323 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1324 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1325 // Based on the above, the total vectors should 14. 1326 #[cfg(target_arch = "x86_64")] 1327 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1328 #[cfg(target_arch = "aarch64")] 1329 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1330 assert_eq!( 1331 guest 1332 .ssh_command(grep_cmd) 1333 .unwrap() 1334 .trim() 1335 .parse::<u32>() 1336 .unwrap_or_default(), 1337 10 + (num_queues as u32) 1338 ); 1339 1340 // ACPI feature is needed. 1341 #[cfg(target_arch = "x86_64")] 1342 { 1343 guest.enable_memory_hotplug(); 1344 1345 // Add RAM to the VM 1346 let desired_ram = 1024 << 20; 1347 resize_command(&api_socket, None, Some(desired_ram), None, None); 1348 1349 thread::sleep(std::time::Duration::new(10, 0)); 1350 1351 // Here by simply checking the size (through ssh), we validate 1352 // the connection is still working, which means vhost-user-net 1353 // keeps working after the resize. 1354 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1355 } 1356 }); 1357 1358 let _ = child.kill(); 1359 let output = child.wait_with_output().unwrap(); 1360 1361 thread::sleep(std::time::Duration::new(5, 0)); 1362 let _ = daemon_child.kill(); 1363 let _ = daemon_child.wait(); 1364 1365 handle_child_output(r, &output); 1366 } 1367 1368 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1369 1370 fn test_vhost_user_blk( 1371 num_queues: usize, 1372 readonly: bool, 1373 direct: bool, 1374 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1375 ) { 1376 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1377 let guest = Guest::new(Box::new(focal)); 1378 let api_socket = temp_api_path(&guest.tmp_dir); 1379 1380 let kernel_path = direct_kernel_boot_path(); 1381 1382 let (blk_params, daemon_child) = { 1383 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1384 // Start the daemon 1385 let (daemon_child, vubd_socket_path) = 1386 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1387 1388 ( 1389 format!( 1390 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1391 ), 1392 Some(daemon_child), 1393 ) 1394 }; 1395 1396 let mut child = GuestCommand::new(&guest) 1397 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1398 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1399 .args(["--kernel", kernel_path.to_str().unwrap()]) 1400 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1401 .args([ 1402 "--disk", 1403 format!( 1404 "path={}", 1405 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1406 ) 1407 .as_str(), 1408 format!( 1409 "path={}", 1410 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1411 ) 1412 .as_str(), 1413 blk_params.as_str(), 1414 ]) 1415 .default_net() 1416 .args(["--api-socket", &api_socket]) 1417 .capture_output() 1418 .spawn() 1419 .unwrap(); 1420 1421 let r = std::panic::catch_unwind(|| { 1422 guest.wait_vm_boot(None).unwrap(); 1423 1424 // Check both if /dev/vdc exists and if the block size is 16M. 1425 assert_eq!( 1426 guest 1427 .ssh_command("lsblk | grep vdc | grep -c 16M") 1428 .unwrap() 1429 .trim() 1430 .parse::<u32>() 1431 .unwrap_or_default(), 1432 1 1433 ); 1434 1435 // Check if this block is RO or RW. 1436 assert_eq!( 1437 guest 1438 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1439 .unwrap() 1440 .trim() 1441 .parse::<u32>() 1442 .unwrap_or_default(), 1443 readonly as u32 1444 ); 1445 1446 // Check if the number of queues in /sys/block/vdc/mq matches the 1447 // expected num_queues. 1448 assert_eq!( 1449 guest 1450 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1451 .unwrap() 1452 .trim() 1453 .parse::<u32>() 1454 .unwrap_or_default(), 1455 num_queues as u32 1456 ); 1457 1458 // Mount the device 1459 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1460 guest.ssh_command("mkdir mount_image").unwrap(); 1461 guest 1462 .ssh_command( 1463 format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), 1464 ) 1465 .unwrap(); 1466 1467 // Check the content of the block device. The file "foo" should 1468 // contain "bar". 1469 assert_eq!( 1470 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1471 "bar" 1472 ); 1473 1474 // ACPI feature is needed. 1475 #[cfg(target_arch = "x86_64")] 1476 { 1477 guest.enable_memory_hotplug(); 1478 1479 // Add RAM to the VM 1480 let desired_ram = 1024 << 20; 1481 resize_command(&api_socket, None, Some(desired_ram), None, None); 1482 1483 thread::sleep(std::time::Duration::new(10, 0)); 1484 1485 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1486 1487 // Check again the content of the block device after the resize 1488 // has been performed. 1489 assert_eq!( 1490 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1491 "bar" 1492 ); 1493 } 1494 1495 // Unmount the device 1496 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1497 guest.ssh_command("rm -r mount_image").unwrap(); 1498 }); 1499 1500 let _ = child.kill(); 1501 let output = child.wait_with_output().unwrap(); 1502 1503 if let Some(mut daemon_child) = daemon_child { 1504 thread::sleep(std::time::Duration::new(5, 0)); 1505 let _ = daemon_child.kill(); 1506 let _ = daemon_child.wait(); 1507 } 1508 1509 handle_child_output(r, &output); 1510 } 1511 1512 fn test_boot_from_vhost_user_blk( 1513 num_queues: usize, 1514 readonly: bool, 1515 direct: bool, 1516 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1517 ) { 1518 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1519 let guest = Guest::new(Box::new(focal)); 1520 1521 let kernel_path = direct_kernel_boot_path(); 1522 1523 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1524 1525 let (blk_boot_params, daemon_child) = { 1526 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1527 // Start the daemon 1528 let (daemon_child, vubd_socket_path) = prepare_daemon( 1529 &guest.tmp_dir, 1530 disk_path.as_str(), 1531 num_queues, 1532 readonly, 1533 direct, 1534 ); 1535 1536 ( 1537 format!( 1538 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1539 ), 1540 Some(daemon_child), 1541 ) 1542 }; 1543 1544 let mut child = GuestCommand::new(&guest) 1545 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1546 .args(["--memory", "size=512M,shared=on"]) 1547 .args(["--kernel", kernel_path.to_str().unwrap()]) 1548 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1549 .args([ 1550 "--disk", 1551 blk_boot_params.as_str(), 1552 format!( 1553 "path={}", 1554 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1555 ) 1556 .as_str(), 1557 ]) 1558 .default_net() 1559 .capture_output() 1560 .spawn() 1561 .unwrap(); 1562 1563 let r = std::panic::catch_unwind(|| { 1564 guest.wait_vm_boot(None).unwrap(); 1565 1566 // Just check the VM booted correctly. 1567 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1568 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1569 }); 1570 let _ = child.kill(); 1571 let output = child.wait_with_output().unwrap(); 1572 1573 if let Some(mut daemon_child) = daemon_child { 1574 thread::sleep(std::time::Duration::new(5, 0)); 1575 let _ = daemon_child.kill(); 1576 let _ = daemon_child.wait(); 1577 } 1578 1579 handle_child_output(r, &output); 1580 } 1581 1582 fn _test_virtio_fs( 1583 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1584 hotplug: bool, 1585 pci_segment: Option<u16>, 1586 ) { 1587 #[cfg(target_arch = "aarch64")] 1588 let focal_image = if hotplug { 1589 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1590 } else { 1591 FOCAL_IMAGE_NAME.to_string() 1592 }; 1593 #[cfg(target_arch = "x86_64")] 1594 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1595 let focal = UbuntuDiskConfig::new(focal_image); 1596 let guest = Guest::new(Box::new(focal)); 1597 let api_socket = temp_api_path(&guest.tmp_dir); 1598 1599 let mut workload_path = dirs::home_dir().unwrap(); 1600 workload_path.push("workloads"); 1601 1602 let mut shared_dir = workload_path; 1603 shared_dir.push("shared_dir"); 1604 1605 #[cfg(target_arch = "x86_64")] 1606 let kernel_path = direct_kernel_boot_path(); 1607 #[cfg(target_arch = "aarch64")] 1608 let kernel_path = if hotplug { 1609 edk2_path() 1610 } else { 1611 direct_kernel_boot_path() 1612 }; 1613 1614 let (mut daemon_child, virtiofsd_socket_path) = 1615 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1616 1617 let mut guest_command = GuestCommand::new(&guest); 1618 guest_command 1619 .args(["--cpus", "boot=1"]) 1620 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1621 .args(["--kernel", kernel_path.to_str().unwrap()]) 1622 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1623 .default_disks() 1624 .default_net() 1625 .args(["--api-socket", &api_socket]); 1626 if pci_segment.is_some() { 1627 guest_command.args([ 1628 "--platform", 1629 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 1630 ]); 1631 } 1632 1633 let fs_params = format!( 1634 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1635 virtiofsd_socket_path, 1636 if let Some(pci_segment) = pci_segment { 1637 format!(",pci_segment={pci_segment}") 1638 } else { 1639 "".to_owned() 1640 } 1641 ); 1642 1643 if !hotplug { 1644 guest_command.args(["--fs", fs_params.as_str()]); 1645 } 1646 1647 let mut child = guest_command.capture_output().spawn().unwrap(); 1648 1649 let r = std::panic::catch_unwind(|| { 1650 guest.wait_vm_boot(None).unwrap(); 1651 1652 if hotplug { 1653 // Add fs to the VM 1654 let (cmd_success, cmd_output) = 1655 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1656 assert!(cmd_success); 1657 1658 if let Some(pci_segment) = pci_segment { 1659 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1660 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1661 ))); 1662 } else { 1663 assert!(String::from_utf8_lossy(&cmd_output) 1664 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1665 } 1666 1667 thread::sleep(std::time::Duration::new(10, 0)); 1668 } 1669 1670 // Mount shared directory through virtio_fs filesystem 1671 guest 1672 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1673 .unwrap(); 1674 1675 // Check file1 exists and its content is "foo" 1676 assert_eq!( 1677 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1678 "foo" 1679 ); 1680 // Check file2 does not exist 1681 guest 1682 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1683 .unwrap(); 1684 1685 // Check file3 exists and its content is "bar" 1686 assert_eq!( 1687 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1688 "bar" 1689 ); 1690 1691 // ACPI feature is needed. 1692 #[cfg(target_arch = "x86_64")] 1693 { 1694 guest.enable_memory_hotplug(); 1695 1696 // Add RAM to the VM 1697 let desired_ram = 1024 << 20; 1698 resize_command(&api_socket, None, Some(desired_ram), None, None); 1699 1700 thread::sleep(std::time::Duration::new(30, 0)); 1701 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1702 1703 // After the resize, check again that file1 exists and its 1704 // content is "foo". 1705 assert_eq!( 1706 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1707 "foo" 1708 ); 1709 } 1710 1711 if hotplug { 1712 // Remove from VM 1713 guest.ssh_command("sudo umount mount_dir").unwrap(); 1714 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1715 } 1716 }); 1717 1718 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1719 thread::sleep(std::time::Duration::new(10, 0)); 1720 let (daemon_child, virtiofsd_socket_path) = 1721 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1722 1723 let r = std::panic::catch_unwind(|| { 1724 thread::sleep(std::time::Duration::new(10, 0)); 1725 let fs_params = format!( 1726 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1727 virtiofsd_socket_path, 1728 if let Some(pci_segment) = pci_segment { 1729 format!(",pci_segment={pci_segment}") 1730 } else { 1731 "".to_owned() 1732 } 1733 ); 1734 1735 // Add back and check it works 1736 let (cmd_success, cmd_output) = 1737 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1738 assert!(cmd_success); 1739 if let Some(pci_segment) = pci_segment { 1740 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1741 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1742 ))); 1743 } else { 1744 assert!(String::from_utf8_lossy(&cmd_output) 1745 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1746 } 1747 1748 thread::sleep(std::time::Duration::new(10, 0)); 1749 // Mount shared directory through virtio_fs filesystem 1750 guest 1751 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1752 .unwrap(); 1753 1754 // Check file1 exists and its content is "foo" 1755 assert_eq!( 1756 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1757 "foo" 1758 ); 1759 }); 1760 1761 (r, Some(daemon_child)) 1762 } else { 1763 (r, None) 1764 }; 1765 1766 let _ = child.kill(); 1767 let output = child.wait_with_output().unwrap(); 1768 1769 let _ = daemon_child.kill(); 1770 let _ = daemon_child.wait(); 1771 1772 if let Some(mut daemon_child) = hotplug_daemon_child { 1773 let _ = daemon_child.kill(); 1774 let _ = daemon_child.wait(); 1775 } 1776 1777 handle_child_output(r, &output); 1778 } 1779 1780 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1781 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1782 let guest = Guest::new(Box::new(focal)); 1783 1784 let kernel_path = direct_kernel_boot_path(); 1785 1786 let pmem_temp_file = TempFile::new().unwrap(); 1787 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1788 1789 std::process::Command::new("mkfs.ext4") 1790 .arg(pmem_temp_file.as_path()) 1791 .output() 1792 .expect("Expect creating disk image to succeed"); 1793 1794 let mut child = GuestCommand::new(&guest) 1795 .args(["--cpus", "boot=1"]) 1796 .args(["--memory", "size=512M"]) 1797 .args(["--kernel", kernel_path.to_str().unwrap()]) 1798 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1799 .default_disks() 1800 .default_net() 1801 .args([ 1802 "--pmem", 1803 format!( 1804 "file={}{}{}", 1805 pmem_temp_file.as_path().to_str().unwrap(), 1806 if specify_size { ",size=128M" } else { "" }, 1807 if discard_writes { 1808 ",discard_writes=on" 1809 } else { 1810 "" 1811 } 1812 ) 1813 .as_str(), 1814 ]) 1815 .capture_output() 1816 .spawn() 1817 .unwrap(); 1818 1819 let r = std::panic::catch_unwind(|| { 1820 guest.wait_vm_boot(None).unwrap(); 1821 1822 // Check for the presence of /dev/pmem0 1823 assert_eq!( 1824 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1825 "/dev/pmem0" 1826 ); 1827 1828 // Check changes persist after reboot 1829 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1830 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1831 guest 1832 .ssh_command("echo test123 | sudo tee /mnt/test") 1833 .unwrap(); 1834 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1835 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1836 1837 guest.reboot_linux(0, None); 1838 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1839 assert_eq!( 1840 guest 1841 .ssh_command("sudo cat /mnt/test || true") 1842 .unwrap() 1843 .trim(), 1844 if discard_writes { "" } else { "test123" } 1845 ); 1846 }); 1847 1848 let _ = child.kill(); 1849 let output = child.wait_with_output().unwrap(); 1850 1851 handle_child_output(r, &output); 1852 } 1853 1854 fn get_fd_count(pid: u32) -> usize { 1855 fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() 1856 } 1857 1858 fn _test_virtio_vsock(hotplug: bool) { 1859 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1860 let guest = Guest::new(Box::new(focal)); 1861 1862 #[cfg(target_arch = "x86_64")] 1863 let kernel_path = direct_kernel_boot_path(); 1864 #[cfg(target_arch = "aarch64")] 1865 let kernel_path = if hotplug { 1866 edk2_path() 1867 } else { 1868 direct_kernel_boot_path() 1869 }; 1870 1871 let socket = temp_vsock_path(&guest.tmp_dir); 1872 let api_socket = temp_api_path(&guest.tmp_dir); 1873 1874 let mut cmd = GuestCommand::new(&guest); 1875 cmd.args(["--api-socket", &api_socket]); 1876 cmd.args(["--cpus", "boot=1"]); 1877 cmd.args(["--memory", "size=512M"]); 1878 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1879 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1880 cmd.default_disks(); 1881 cmd.default_net(); 1882 1883 if !hotplug { 1884 cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); 1885 } 1886 1887 let mut child = cmd.capture_output().spawn().unwrap(); 1888 1889 let r = std::panic::catch_unwind(|| { 1890 guest.wait_vm_boot(None).unwrap(); 1891 1892 if hotplug { 1893 let (cmd_success, cmd_output) = remote_command_w_output( 1894 &api_socket, 1895 "add-vsock", 1896 Some(format!("cid=3,socket={socket},id=test0").as_str()), 1897 ); 1898 assert!(cmd_success); 1899 assert!(String::from_utf8_lossy(&cmd_output) 1900 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1901 thread::sleep(std::time::Duration::new(10, 0)); 1902 // Check adding a second one fails 1903 assert!(!remote_command( 1904 &api_socket, 1905 "add-vsock", 1906 Some("cid=1234,socket=/tmp/fail") 1907 )); 1908 } 1909 1910 // Validate vsock works as expected. 1911 guest.check_vsock(socket.as_str()); 1912 guest.reboot_linux(0, None); 1913 // Validate vsock still works after a reboot. 1914 guest.check_vsock(socket.as_str()); 1915 1916 if hotplug { 1917 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1918 } 1919 }); 1920 1921 let _ = child.kill(); 1922 let output = child.wait_with_output().unwrap(); 1923 1924 handle_child_output(r, &output); 1925 } 1926 1927 fn get_ksm_pages_shared() -> u32 { 1928 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1929 .unwrap() 1930 .trim() 1931 .parse::<u32>() 1932 .unwrap() 1933 } 1934 1935 fn test_memory_mergeable(mergeable: bool) { 1936 let memory_param = if mergeable { 1937 "mergeable=on" 1938 } else { 1939 "mergeable=off" 1940 }; 1941 1942 // We are assuming the rest of the system in our CI is not using mergeable memory 1943 let ksm_ps_init = get_ksm_pages_shared(); 1944 assert!(ksm_ps_init == 0); 1945 1946 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1947 let guest1 = Guest::new(Box::new(focal1)); 1948 let mut child1 = GuestCommand::new(&guest1) 1949 .args(["--cpus", "boot=1"]) 1950 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1951 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1952 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1953 .default_disks() 1954 .args(["--net", guest1.default_net_string().as_str()]) 1955 .args(["--serial", "tty", "--console", "off"]) 1956 .capture_output() 1957 .spawn() 1958 .unwrap(); 1959 1960 let r = std::panic::catch_unwind(|| { 1961 guest1.wait_vm_boot(None).unwrap(); 1962 }); 1963 if r.is_err() { 1964 let _ = child1.kill(); 1965 let output = child1.wait_with_output().unwrap(); 1966 handle_child_output(r, &output); 1967 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1968 } 1969 1970 let ksm_ps_guest1 = get_ksm_pages_shared(); 1971 1972 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1973 let guest2 = Guest::new(Box::new(focal2)); 1974 let mut child2 = GuestCommand::new(&guest2) 1975 .args(["--cpus", "boot=1"]) 1976 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1977 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1978 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1979 .default_disks() 1980 .args(["--net", guest2.default_net_string().as_str()]) 1981 .args(["--serial", "tty", "--console", "off"]) 1982 .capture_output() 1983 .spawn() 1984 .unwrap(); 1985 1986 let r = std::panic::catch_unwind(|| { 1987 guest2.wait_vm_boot(None).unwrap(); 1988 let ksm_ps_guest2 = get_ksm_pages_shared(); 1989 1990 if mergeable { 1991 println!( 1992 "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" 1993 ); 1994 // We are expecting the number of shared pages to increase as the number of VM increases 1995 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1996 } else { 1997 assert!(ksm_ps_guest1 == 0); 1998 assert!(ksm_ps_guest2 == 0); 1999 } 2000 }); 2001 2002 let _ = child1.kill(); 2003 let _ = child2.kill(); 2004 2005 let output = child1.wait_with_output().unwrap(); 2006 child2.wait().unwrap(); 2007 2008 handle_child_output(r, &output); 2009 } 2010 2011 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 2012 let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); 2013 let reader = io::BufReader::new(smaps); 2014 2015 let mut skip_map: bool = false; 2016 let mut region_name: String = "".to_string(); 2017 let mut region_maps = HashMap::new(); 2018 for line in reader.lines() { 2019 let l = line.unwrap(); 2020 2021 if l.contains('-') { 2022 let values: Vec<&str> = l.split_whitespace().collect(); 2023 region_name = values.last().unwrap().trim().to_string(); 2024 if region_name == "0" { 2025 region_name = "anonymous".to_string() 2026 } 2027 } 2028 2029 // Each section begins with something that looks like: 2030 // Size: 2184 kB 2031 if l.starts_with("Size:") { 2032 let values: Vec<&str> = l.split_whitespace().collect(); 2033 let map_size = values[1].parse::<u32>().unwrap(); 2034 // We skip the assigned guest RAM map, its RSS is only 2035 // dependent on the guest actual memory usage. 2036 // Everything else can be added to the VMM overhead. 2037 skip_map = map_size >= guest_memory_size; 2038 continue; 2039 } 2040 2041 // If this is a map we're taking into account, then we only 2042 // count the RSS. The sum of all counted RSS is the VMM overhead. 2043 if !skip_map && l.starts_with("Rss:") { 2044 let values: Vec<&str> = l.split_whitespace().collect(); 2045 let value = values[1].trim().parse::<u32>().unwrap(); 2046 *region_maps.entry(region_name.clone()).or_insert(0) += value; 2047 } 2048 } 2049 2050 region_maps 2051 } 2052 2053 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 2054 let mut total = 0; 2055 2056 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 2057 eprintln!("{region_name}: {value}"); 2058 total += value; 2059 } 2060 2061 total 2062 } 2063 2064 fn process_rss_kib(pid: u32) -> usize { 2065 let command = format!("ps -q {pid} -o rss="); 2066 let rss = exec_host_command_output(&command); 2067 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 2068 } 2069 2070 // 10MB is our maximum accepted overhead. 2071 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 2072 2073 #[derive(PartialEq, Eq, PartialOrd)] 2074 struct Counters { 2075 rx_bytes: u64, 2076 rx_frames: u64, 2077 tx_bytes: u64, 2078 tx_frames: u64, 2079 read_bytes: u64, 2080 write_bytes: u64, 2081 read_ops: u64, 2082 write_ops: u64, 2083 } 2084 2085 fn get_counters(api_socket: &str) -> Counters { 2086 // Get counters 2087 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 2088 assert!(cmd_success); 2089 2090 let counters: HashMap<&str, HashMap<&str, u64>> = 2091 serde_json::from_slice(&cmd_output).unwrap_or_default(); 2092 2093 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 2094 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 2095 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 2096 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 2097 2098 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 2099 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 2100 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 2101 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 2102 2103 Counters { 2104 rx_bytes, 2105 rx_frames, 2106 tx_bytes, 2107 tx_frames, 2108 read_bytes, 2109 write_bytes, 2110 read_ops, 2111 write_ops, 2112 } 2113 } 2114 2115 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 2116 let (tx, rx) = mpsc::channel::<String>(); 2117 thread::spawn(move || loop { 2118 thread::sleep(std::time::Duration::new(1, 0)); 2119 let mut buf = [0; 512]; 2120 match pty.read(&mut buf) { 2121 Ok(_bytes) => { 2122 let output = std::str::from_utf8(&buf).unwrap().to_string(); 2123 match tx.send(output) { 2124 Ok(_) => (), 2125 Err(_) => break, 2126 } 2127 } 2128 Err(_) => break, 2129 } 2130 }); 2131 rx 2132 } 2133 2134 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 2135 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2136 assert!(cmd_success); 2137 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2138 assert_eq!("Pty", info["config"][pty_type]["mode"]); 2139 PathBuf::from( 2140 info["config"][pty_type]["file"] 2141 .as_str() 2142 .expect("Missing pty path"), 2143 ) 2144 } 2145 2146 // VFIO test network setup. 2147 // We reserve a different IP class for it: 172.18.0.0/24. 2148 #[cfg(target_arch = "x86_64")] 2149 fn setup_vfio_network_interfaces() { 2150 // 'vfio-br0' 2151 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 2152 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 2153 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 2154 // 'vfio-tap0' 2155 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 2156 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 2157 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 2158 // 'vfio-tap1' 2159 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 2160 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 2161 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 2162 // 'vfio-tap2' 2163 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 2164 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 2165 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 2166 // 'vfio-tap3' 2167 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 2168 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 2169 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 2170 } 2171 2172 // Tear VFIO test network down 2173 #[cfg(target_arch = "x86_64")] 2174 fn cleanup_vfio_network_interfaces() { 2175 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 2176 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 2177 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 2178 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 2179 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 2180 } 2181 2182 fn balloon_size(api_socket: &str) -> u64 { 2183 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2184 assert!(cmd_success); 2185 2186 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2187 let total_mem = &info["config"]["memory"]["size"] 2188 .to_string() 2189 .parse::<u64>() 2190 .unwrap(); 2191 let actual_mem = &info["memory_actual_size"] 2192 .to_string() 2193 .parse::<u64>() 2194 .unwrap(); 2195 total_mem - actual_mem 2196 } 2197 2198 fn vm_state(api_socket: &str) -> String { 2199 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2200 assert!(cmd_success); 2201 2202 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2203 let state = &info["state"].as_str().unwrap(); 2204 2205 state.to_string() 2206 } 2207 2208 // This test validates that it can find the virtio-iommu device at first. 2209 // It also verifies that both disks and the network card are attached to 2210 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 2211 // The last interesting part of this test is that it exercises the network 2212 // interface attached to the virtual IOMMU since this is the one used to 2213 // send all commands through SSH. 2214 fn _test_virtio_iommu(acpi: bool) { 2215 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 2216 // Focal image is still old. 2217 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 2218 // the kernel binary has been updated. 2219 #[cfg(target_arch = "aarch64")] 2220 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 2221 #[cfg(target_arch = "x86_64")] 2222 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2223 let focal = UbuntuDiskConfig::new(focal_image); 2224 let guest = Guest::new(Box::new(focal)); 2225 2226 #[cfg(target_arch = "x86_64")] 2227 let kernel_path = direct_kernel_boot_path(); 2228 #[cfg(target_arch = "aarch64")] 2229 let kernel_path = if acpi { 2230 edk2_path() 2231 } else { 2232 direct_kernel_boot_path() 2233 }; 2234 2235 let mut child = GuestCommand::new(&guest) 2236 .args(["--cpus", "boot=1"]) 2237 .args(["--memory", "size=512M"]) 2238 .args(["--kernel", kernel_path.to_str().unwrap()]) 2239 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2240 .args([ 2241 "--disk", 2242 format!( 2243 "path={},iommu=on", 2244 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2245 ) 2246 .as_str(), 2247 format!( 2248 "path={},iommu=on", 2249 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2250 ) 2251 .as_str(), 2252 ]) 2253 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 2254 .capture_output() 2255 .spawn() 2256 .unwrap(); 2257 2258 let r = std::panic::catch_unwind(|| { 2259 guest.wait_vm_boot(None).unwrap(); 2260 2261 // Verify the virtio-iommu device is present. 2262 assert!(guest 2263 .does_device_vendor_pair_match("0x1057", "0x1af4") 2264 .unwrap_or_default()); 2265 2266 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 2267 // different with ACPI. 2268 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 2269 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 2270 // and appear under folder '/sys/kernel/iommu_groups/'. 2271 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 2272 // which is the console. The first disk "0000:00:02.0" is in group '1'. 2273 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 2274 // contains "0000:00:02.0" which is the first disk. 2275 // 2276 // Verify the iommu group of the first disk. 2277 let iommu_group = !acpi as i32; 2278 assert_eq!( 2279 guest 2280 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2281 .unwrap() 2282 .trim(), 2283 "0000:00:02.0" 2284 ); 2285 2286 // Verify the iommu group of the second disk. 2287 let iommu_group = if acpi { 1 } else { 2 }; 2288 assert_eq!( 2289 guest 2290 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2291 .unwrap() 2292 .trim(), 2293 "0000:00:03.0" 2294 ); 2295 2296 // Verify the iommu group of the network card. 2297 let iommu_group = if acpi { 2 } else { 3 }; 2298 assert_eq!( 2299 guest 2300 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2301 .unwrap() 2302 .trim(), 2303 "0000:00:04.0" 2304 ); 2305 }); 2306 2307 let _ = child.kill(); 2308 let output = child.wait_with_output().unwrap(); 2309 2310 handle_child_output(r, &output); 2311 } 2312 2313 fn get_reboot_count(guest: &Guest) -> u32 { 2314 guest 2315 .ssh_command("sudo last | grep -c reboot") 2316 .unwrap() 2317 .trim() 2318 .parse::<u32>() 2319 .unwrap_or_default() 2320 } 2321 2322 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 2323 // Check for PCI device 2324 assert!(guest 2325 .does_device_vendor_pair_match("0x1063", "0x1af4") 2326 .unwrap_or_default()); 2327 2328 // Enable systemd watchdog 2329 guest 2330 .ssh_command(&format!( 2331 "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" 2332 )) 2333 .unwrap(); 2334 } 2335 2336 fn make_guest_panic(guest: &Guest) { 2337 // Check for pvpanic device 2338 assert!(guest 2339 .does_device_vendor_pair_match("0x0011", "0x1b36") 2340 .unwrap_or_default()); 2341 2342 // Trigger guest a panic 2343 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 2344 } 2345 2346 mod common_parallel { 2347 use std::{ 2348 fs::{remove_dir_all, OpenOptions}, 2349 io::SeekFrom, 2350 }; 2351 2352 use crate::*; 2353 2354 #[test] 2355 #[cfg(target_arch = "x86_64")] 2356 fn test_focal_hypervisor_fw() { 2357 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 2358 } 2359 2360 #[test] 2361 #[cfg(target_arch = "x86_64")] 2362 fn test_focal_ovmf() { 2363 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 2364 } 2365 2366 #[cfg(target_arch = "x86_64")] 2367 fn test_simple_launch(fw_path: String, disk_path: &str) { 2368 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 2369 let guest = Guest::new(disk_config); 2370 let event_path = temp_event_monitor_path(&guest.tmp_dir); 2371 2372 let mut child = GuestCommand::new(&guest) 2373 .args(["--cpus", "boot=1"]) 2374 .args(["--memory", "size=512M"]) 2375 .args(["--kernel", fw_path.as_str()]) 2376 .default_disks() 2377 .default_net() 2378 .args(["--serial", "tty", "--console", "off"]) 2379 .args(["--event-monitor", format!("path={event_path}").as_str()]) 2380 .capture_output() 2381 .spawn() 2382 .unwrap(); 2383 2384 let r = std::panic::catch_unwind(|| { 2385 guest.wait_vm_boot(Some(120)).unwrap(); 2386 2387 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2388 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2389 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2390 2391 let expected_sequential_events = [ 2392 &MetaEvent { 2393 event: "starting".to_string(), 2394 device_id: None, 2395 }, 2396 &MetaEvent { 2397 event: "booting".to_string(), 2398 device_id: None, 2399 }, 2400 &MetaEvent { 2401 event: "booted".to_string(), 2402 device_id: None, 2403 }, 2404 &MetaEvent { 2405 event: "activated".to_string(), 2406 device_id: Some("_disk0".to_string()), 2407 }, 2408 &MetaEvent { 2409 event: "reset".to_string(), 2410 device_id: Some("_disk0".to_string()), 2411 }, 2412 ]; 2413 assert!(check_sequential_events( 2414 &expected_sequential_events, 2415 &event_path 2416 )); 2417 2418 // It's been observed on the Bionic image that udev and snapd 2419 // services can cause some delay in the VM's shutdown. Disabling 2420 // them improves the reliability of this test. 2421 let _ = guest.ssh_command("sudo systemctl disable udev"); 2422 let _ = guest.ssh_command("sudo systemctl stop udev"); 2423 let _ = guest.ssh_command("sudo systemctl disable snapd"); 2424 let _ = guest.ssh_command("sudo systemctl stop snapd"); 2425 2426 guest.ssh_command("sudo poweroff").unwrap(); 2427 thread::sleep(std::time::Duration::new(20, 0)); 2428 let latest_events = [ 2429 &MetaEvent { 2430 event: "shutdown".to_string(), 2431 device_id: None, 2432 }, 2433 &MetaEvent { 2434 event: "deleted".to_string(), 2435 device_id: None, 2436 }, 2437 &MetaEvent { 2438 event: "shutdown".to_string(), 2439 device_id: None, 2440 }, 2441 ]; 2442 assert!(check_latest_events_exact(&latest_events, &event_path)); 2443 }); 2444 2445 let _ = child.kill(); 2446 let output = child.wait_with_output().unwrap(); 2447 2448 handle_child_output(r, &output); 2449 } 2450 2451 #[test] 2452 fn test_multi_cpu() { 2453 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2454 let jammy = UbuntuDiskConfig::new(jammy_image); 2455 let guest = Guest::new(Box::new(jammy)); 2456 2457 let mut cmd = GuestCommand::new(&guest); 2458 cmd.args(["--cpus", "boot=2,max=4"]) 2459 .args(["--memory", "size=512M"]) 2460 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2461 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2462 .capture_output() 2463 .default_disks() 2464 .default_net(); 2465 2466 let mut child = cmd.spawn().unwrap(); 2467 2468 let r = std::panic::catch_unwind(|| { 2469 guest.wait_vm_boot(Some(120)).unwrap(); 2470 2471 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2472 2473 assert_eq!( 2474 guest 2475 .ssh_command( 2476 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2477 ) 2478 .unwrap() 2479 .trim(), 2480 "smp: Brought up 1 node, 2 CPUs" 2481 ); 2482 }); 2483 2484 let _ = child.kill(); 2485 let output = child.wait_with_output().unwrap(); 2486 2487 handle_child_output(r, &output); 2488 } 2489 2490 #[test] 2491 fn test_cpu_topology_421() { 2492 test_cpu_topology(4, 2, 1, false); 2493 } 2494 2495 #[test] 2496 fn test_cpu_topology_142() { 2497 test_cpu_topology(1, 4, 2, false); 2498 } 2499 2500 #[test] 2501 fn test_cpu_topology_262() { 2502 test_cpu_topology(2, 6, 2, false); 2503 } 2504 2505 #[test] 2506 #[cfg(target_arch = "x86_64")] 2507 #[cfg(not(feature = "mshv"))] 2508 fn test_cpu_physical_bits() { 2509 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2510 let guest = Guest::new(Box::new(focal)); 2511 let max_phys_bits: u8 = 36; 2512 let mut child = GuestCommand::new(&guest) 2513 .args(["--cpus", &format!("max_phys_bits={max_phys_bits}")]) 2514 .args(["--memory", "size=512M"]) 2515 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2516 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2517 .default_disks() 2518 .default_net() 2519 .capture_output() 2520 .spawn() 2521 .unwrap(); 2522 2523 let r = std::panic::catch_unwind(|| { 2524 guest.wait_vm_boot(None).unwrap(); 2525 2526 assert!( 2527 guest 2528 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2529 .unwrap() 2530 .trim() 2531 .parse::<u8>() 2532 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2533 ); 2534 }); 2535 2536 let _ = child.kill(); 2537 let output = child.wait_with_output().unwrap(); 2538 2539 handle_child_output(r, &output); 2540 } 2541 2542 #[test] 2543 fn test_cpu_affinity() { 2544 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2545 let guest = Guest::new(Box::new(focal)); 2546 2547 // We need the host to have at least 4 CPUs if we want to be able 2548 // to run this test. 2549 let host_cpus_count = exec_host_command_output("nproc"); 2550 assert!( 2551 String::from_utf8_lossy(&host_cpus_count.stdout) 2552 .trim() 2553 .parse::<u16>() 2554 .unwrap_or(0) 2555 >= 4 2556 ); 2557 2558 let mut child = GuestCommand::new(&guest) 2559 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2560 .args(["--memory", "size=512M"]) 2561 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2562 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2563 .default_disks() 2564 .default_net() 2565 .capture_output() 2566 .spawn() 2567 .unwrap(); 2568 2569 let r = std::panic::catch_unwind(|| { 2570 guest.wait_vm_boot(None).unwrap(); 2571 let pid = child.id(); 2572 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2573 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2574 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2575 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2576 }); 2577 2578 let _ = child.kill(); 2579 let output = child.wait_with_output().unwrap(); 2580 handle_child_output(r, &output); 2581 } 2582 2583 #[test] 2584 fn test_virtio_queue_affinity() { 2585 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2586 let guest = Guest::new(Box::new(focal)); 2587 2588 // We need the host to have at least 4 CPUs if we want to be able 2589 // to run this test. 2590 let host_cpus_count = exec_host_command_output("nproc"); 2591 assert!( 2592 String::from_utf8_lossy(&host_cpus_count.stdout) 2593 .trim() 2594 .parse::<u16>() 2595 .unwrap_or(0) 2596 >= 4 2597 ); 2598 2599 let mut child = GuestCommand::new(&guest) 2600 .args(["--cpus", "boot=4"]) 2601 .args(["--memory", "size=512M"]) 2602 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2603 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2604 .args([ 2605 "--disk", 2606 format!( 2607 "path={}", 2608 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2609 ) 2610 .as_str(), 2611 format!( 2612 "path={},num_queues=4,queue_affinity=[0@[0,2],1@[1,3],2@[1],3@[3]]", 2613 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2614 ) 2615 .as_str(), 2616 ]) 2617 .default_net() 2618 .capture_output() 2619 .spawn() 2620 .unwrap(); 2621 2622 let r = std::panic::catch_unwind(|| { 2623 guest.wait_vm_boot(None).unwrap(); 2624 let pid = child.id(); 2625 let taskset_q0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2626 assert_eq!(String::from_utf8_lossy(&taskset_q0.stdout).trim(), "0,2"); 2627 let taskset_q1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2628 assert_eq!(String::from_utf8_lossy(&taskset_q1.stdout).trim(), "1,3"); 2629 let taskset_q2 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q2 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2630 assert_eq!(String::from_utf8_lossy(&taskset_q2.stdout).trim(), "1"); 2631 let taskset_q3 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q3 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2632 assert_eq!(String::from_utf8_lossy(&taskset_q3.stdout).trim(), "3"); 2633 }); 2634 2635 let _ = child.kill(); 2636 let output = child.wait_with_output().unwrap(); 2637 handle_child_output(r, &output); 2638 } 2639 2640 #[test] 2641 #[cfg(not(feature = "mshv"))] 2642 fn test_large_vm() { 2643 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2644 let guest = Guest::new(Box::new(focal)); 2645 let mut cmd = GuestCommand::new(&guest); 2646 cmd.args(["--cpus", "boot=48"]) 2647 .args(["--memory", "size=5120M"]) 2648 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2649 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2650 .args(["--serial", "tty"]) 2651 .args(["--console", "off"]) 2652 .capture_output() 2653 .default_disks() 2654 .default_net(); 2655 2656 let mut child = cmd.spawn().unwrap(); 2657 2658 guest.wait_vm_boot(None).unwrap(); 2659 2660 let r = std::panic::catch_unwind(|| { 2661 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2662 assert_eq!( 2663 guest 2664 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2665 .unwrap() 2666 .trim(), 2667 "0-47" 2668 ); 2669 2670 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2671 }); 2672 2673 let _ = child.kill(); 2674 let output = child.wait_with_output().unwrap(); 2675 2676 handle_child_output(r, &output); 2677 } 2678 2679 #[test] 2680 #[cfg(not(feature = "mshv"))] 2681 fn test_huge_memory() { 2682 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2683 let guest = Guest::new(Box::new(focal)); 2684 let mut cmd = GuestCommand::new(&guest); 2685 cmd.args(["--cpus", "boot=1"]) 2686 .args(["--memory", "size=128G"]) 2687 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2688 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2689 .capture_output() 2690 .default_disks() 2691 .default_net(); 2692 2693 let mut child = cmd.spawn().unwrap(); 2694 2695 guest.wait_vm_boot(Some(120)).unwrap(); 2696 2697 let r = std::panic::catch_unwind(|| { 2698 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2699 }); 2700 2701 let _ = child.kill(); 2702 let output = child.wait_with_output().unwrap(); 2703 2704 handle_child_output(r, &output); 2705 } 2706 2707 #[test] 2708 fn test_power_button() { 2709 _test_power_button(false); 2710 } 2711 2712 #[test] 2713 #[cfg(not(feature = "mshv"))] 2714 fn test_user_defined_memory_regions() { 2715 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2716 let guest = Guest::new(Box::new(focal)); 2717 let api_socket = temp_api_path(&guest.tmp_dir); 2718 2719 let kernel_path = direct_kernel_boot_path(); 2720 2721 let mut child = GuestCommand::new(&guest) 2722 .args(["--cpus", "boot=1"]) 2723 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2724 .args([ 2725 "--memory-zone", 2726 "id=mem0,size=1G,hotplug_size=2G", 2727 "id=mem1,size=1G,shared=on", 2728 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2729 ]) 2730 .args(["--kernel", kernel_path.to_str().unwrap()]) 2731 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2732 .args(["--api-socket", &api_socket]) 2733 .capture_output() 2734 .default_disks() 2735 .default_net() 2736 .spawn() 2737 .unwrap(); 2738 2739 let r = std::panic::catch_unwind(|| { 2740 guest.wait_vm_boot(None).unwrap(); 2741 2742 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2743 2744 guest.enable_memory_hotplug(); 2745 2746 resize_zone_command(&api_socket, "mem0", "3G"); 2747 thread::sleep(std::time::Duration::new(5, 0)); 2748 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2749 resize_zone_command(&api_socket, "mem2", "3G"); 2750 thread::sleep(std::time::Duration::new(5, 0)); 2751 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2752 resize_zone_command(&api_socket, "mem0", "2G"); 2753 thread::sleep(std::time::Duration::new(5, 0)); 2754 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2755 resize_zone_command(&api_socket, "mem2", "2G"); 2756 thread::sleep(std::time::Duration::new(5, 0)); 2757 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2758 2759 guest.reboot_linux(0, None); 2760 2761 // Check the amount of RAM after reboot 2762 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2763 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2764 2765 // Check if we can still resize down to the initial 'boot'size 2766 resize_zone_command(&api_socket, "mem0", "1G"); 2767 thread::sleep(std::time::Duration::new(5, 0)); 2768 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2769 resize_zone_command(&api_socket, "mem2", "1G"); 2770 thread::sleep(std::time::Duration::new(5, 0)); 2771 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2772 }); 2773 2774 let _ = child.kill(); 2775 let output = child.wait_with_output().unwrap(); 2776 2777 handle_child_output(r, &output); 2778 } 2779 2780 #[test] 2781 #[cfg(not(feature = "mshv"))] 2782 fn test_guest_numa_nodes() { 2783 _test_guest_numa_nodes(false); 2784 } 2785 2786 #[test] 2787 #[cfg(target_arch = "x86_64")] 2788 fn test_iommu_segments() { 2789 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2790 let focal = UbuntuDiskConfig::new(focal_image); 2791 let guest = Guest::new(Box::new(focal)); 2792 2793 // Prepare another disk file for the virtio-disk device 2794 let test_disk_path = String::from( 2795 guest 2796 .tmp_dir 2797 .as_path() 2798 .join("test-disk.raw") 2799 .to_str() 2800 .unwrap(), 2801 ); 2802 assert!( 2803 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2804 ); 2805 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2806 2807 let api_socket = temp_api_path(&guest.tmp_dir); 2808 let mut cmd = GuestCommand::new(&guest); 2809 2810 cmd.args(["--cpus", "boot=1"]) 2811 .args(["--api-socket", &api_socket]) 2812 .args(["--memory", "size=512M"]) 2813 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2814 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2815 .args([ 2816 "--platform", 2817 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS},iommu_segments=[1]"), 2818 ]) 2819 .default_disks() 2820 .capture_output() 2821 .default_net(); 2822 2823 let mut child = cmd.spawn().unwrap(); 2824 2825 guest.wait_vm_boot(None).unwrap(); 2826 2827 let r = std::panic::catch_unwind(|| { 2828 let (cmd_success, cmd_output) = remote_command_w_output( 2829 &api_socket, 2830 "add-disk", 2831 Some( 2832 format!( 2833 "path={},id=test0,pci_segment=1,iommu=on", 2834 test_disk_path.as_str() 2835 ) 2836 .as_str(), 2837 ), 2838 ); 2839 assert!(cmd_success); 2840 assert!(String::from_utf8_lossy(&cmd_output) 2841 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2842 2843 // Check IOMMU setup 2844 assert!(guest 2845 .does_device_vendor_pair_match("0x1057", "0x1af4") 2846 .unwrap_or_default()); 2847 assert_eq!( 2848 guest 2849 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2850 .unwrap() 2851 .trim(), 2852 "0001:00:01.0" 2853 ); 2854 }); 2855 2856 let _ = child.kill(); 2857 let output = child.wait_with_output().unwrap(); 2858 2859 handle_child_output(r, &output); 2860 } 2861 2862 #[test] 2863 fn test_pci_msi() { 2864 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2865 let guest = Guest::new(Box::new(focal)); 2866 let mut cmd = GuestCommand::new(&guest); 2867 cmd.args(["--cpus", "boot=1"]) 2868 .args(["--memory", "size=512M"]) 2869 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2870 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2871 .capture_output() 2872 .default_disks() 2873 .default_net(); 2874 2875 let mut child = cmd.spawn().unwrap(); 2876 2877 guest.wait_vm_boot(None).unwrap(); 2878 2879 #[cfg(target_arch = "x86_64")] 2880 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2881 #[cfg(target_arch = "aarch64")] 2882 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2883 2884 let r = std::panic::catch_unwind(|| { 2885 assert_eq!( 2886 guest 2887 .ssh_command(grep_cmd) 2888 .unwrap() 2889 .trim() 2890 .parse::<u32>() 2891 .unwrap_or_default(), 2892 12 2893 ); 2894 }); 2895 2896 let _ = child.kill(); 2897 let output = child.wait_with_output().unwrap(); 2898 2899 handle_child_output(r, &output); 2900 } 2901 2902 #[test] 2903 fn test_virtio_net_ctrl_queue() { 2904 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2905 let guest = Guest::new(Box::new(focal)); 2906 let mut cmd = GuestCommand::new(&guest); 2907 cmd.args(["--cpus", "boot=1"]) 2908 .args(["--memory", "size=512M"]) 2909 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2910 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2911 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2912 .capture_output() 2913 .default_disks(); 2914 2915 let mut child = cmd.spawn().unwrap(); 2916 2917 guest.wait_vm_boot(None).unwrap(); 2918 2919 #[cfg(target_arch = "aarch64")] 2920 let iface = "enp0s4"; 2921 #[cfg(target_arch = "x86_64")] 2922 let iface = "ens4"; 2923 2924 let r = std::panic::catch_unwind(|| { 2925 assert_eq!( 2926 guest 2927 .ssh_command( 2928 format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() 2929 ) 2930 .unwrap() 2931 .trim(), 2932 "success" 2933 ); 2934 assert_eq!( 2935 guest 2936 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 2937 .unwrap() 2938 .trim(), 2939 "3000" 2940 ); 2941 }); 2942 2943 let _ = child.kill(); 2944 let output = child.wait_with_output().unwrap(); 2945 2946 handle_child_output(r, &output); 2947 } 2948 2949 #[test] 2950 fn test_pci_multiple_segments() { 2951 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2952 let guest = Guest::new(Box::new(focal)); 2953 2954 // Prepare another disk file for the virtio-disk device 2955 let test_disk_path = String::from( 2956 guest 2957 .tmp_dir 2958 .as_path() 2959 .join("test-disk.raw") 2960 .to_str() 2961 .unwrap(), 2962 ); 2963 assert!( 2964 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2965 ); 2966 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2967 2968 let mut cmd = GuestCommand::new(&guest); 2969 cmd.args(["--cpus", "boot=1"]) 2970 .args(["--memory", "size=512M"]) 2971 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2972 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2973 .args([ 2974 "--platform", 2975 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 2976 ]) 2977 .args([ 2978 "--disk", 2979 format!( 2980 "path={}", 2981 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2982 ) 2983 .as_str(), 2984 format!( 2985 "path={}", 2986 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2987 ) 2988 .as_str(), 2989 format!("path={test_disk_path},pci_segment=15").as_str(), 2990 ]) 2991 .capture_output() 2992 .default_net(); 2993 2994 let mut child = cmd.spawn().unwrap(); 2995 2996 guest.wait_vm_boot(None).unwrap(); 2997 2998 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2999 3000 let r = std::panic::catch_unwind(|| { 3001 // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. 3002 assert_eq!( 3003 guest 3004 .ssh_command(grep_cmd) 3005 .unwrap() 3006 .trim() 3007 .parse::<u16>() 3008 .unwrap_or_default(), 3009 MAX_NUM_PCI_SEGMENTS 3010 ); 3011 3012 // Check both if /dev/vdc exists and if the block size is 4M. 3013 assert_eq!( 3014 guest 3015 .ssh_command("lsblk | grep vdc | grep -c 4M") 3016 .unwrap() 3017 .trim() 3018 .parse::<u32>() 3019 .unwrap_or_default(), 3020 1 3021 ); 3022 3023 // Mount the device. 3024 guest.ssh_command("mkdir mount_image").unwrap(); 3025 guest 3026 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 3027 .unwrap(); 3028 // Grant all users with write permission. 3029 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 3030 3031 // Write something to the device. 3032 guest 3033 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 3034 .unwrap(); 3035 3036 // Check the content of the block device. The file "foo" should 3037 // contain "bar". 3038 assert_eq!( 3039 guest 3040 .ssh_command("sudo cat mount_image/foo") 3041 .unwrap() 3042 .trim(), 3043 "bar" 3044 ); 3045 }); 3046 3047 let _ = child.kill(); 3048 let output = child.wait_with_output().unwrap(); 3049 3050 handle_child_output(r, &output); 3051 } 3052 3053 #[test] 3054 fn test_pci_multiple_segments_numa_node() { 3055 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3056 let guest = Guest::new(Box::new(focal)); 3057 let api_socket = temp_api_path(&guest.tmp_dir); 3058 #[cfg(target_arch = "x86_64")] 3059 let kernel_path = direct_kernel_boot_path(); 3060 #[cfg(target_arch = "aarch64")] 3061 let kernel_path = edk2_path(); 3062 3063 // Prepare another disk file for the virtio-disk device 3064 let test_disk_path = String::from( 3065 guest 3066 .tmp_dir 3067 .as_path() 3068 .join("test-disk.raw") 3069 .to_str() 3070 .unwrap(), 3071 ); 3072 assert!( 3073 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 3074 ); 3075 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 3076 const TEST_DISK_NODE: u16 = 1; 3077 3078 let mut child = GuestCommand::new(&guest) 3079 .args(["--platform", "num_pci_segments=2"]) 3080 .args(["--cpus", "boot=2"]) 3081 .args(["--memory", "size=0"]) 3082 .args(["--memory-zone", "id=mem0,size=256M", "id=mem1,size=256M"]) 3083 .args([ 3084 "--numa", 3085 "guest_numa_id=0,cpus=[0],distances=[1@20],memory_zones=mem0,pci_segments=[0]", 3086 "guest_numa_id=1,cpus=[1],distances=[0@20],memory_zones=mem1,pci_segments=[1]", 3087 ]) 3088 .args(["--kernel", kernel_path.to_str().unwrap()]) 3089 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3090 .args(["--api-socket", &api_socket]) 3091 .capture_output() 3092 .args([ 3093 "--disk", 3094 format!( 3095 "path={}", 3096 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3097 ) 3098 .as_str(), 3099 format!( 3100 "path={}", 3101 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3102 ) 3103 .as_str(), 3104 format!("path={test_disk_path},pci_segment={TEST_DISK_NODE}").as_str(), 3105 ]) 3106 .default_net() 3107 .spawn() 3108 .unwrap(); 3109 3110 let cmd = "cat /sys/block/vdc/device/../numa_node"; 3111 3112 let r = std::panic::catch_unwind(|| { 3113 guest.wait_vm_boot(None).unwrap(); 3114 3115 assert_eq!( 3116 guest 3117 .ssh_command(cmd) 3118 .unwrap() 3119 .trim() 3120 .parse::<u16>() 3121 .unwrap_or_default(), 3122 TEST_DISK_NODE 3123 ); 3124 }); 3125 3126 let _ = child.kill(); 3127 let output = child.wait_with_output().unwrap(); 3128 3129 handle_child_output(r, &output); 3130 } 3131 3132 #[test] 3133 fn test_direct_kernel_boot() { 3134 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3135 let guest = Guest::new(Box::new(focal)); 3136 3137 let kernel_path = direct_kernel_boot_path(); 3138 3139 let mut child = GuestCommand::new(&guest) 3140 .args(["--cpus", "boot=1"]) 3141 .args(["--memory", "size=512M"]) 3142 .args(["--kernel", kernel_path.to_str().unwrap()]) 3143 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3144 .default_disks() 3145 .default_net() 3146 .capture_output() 3147 .spawn() 3148 .unwrap(); 3149 3150 let r = std::panic::catch_unwind(|| { 3151 guest.wait_vm_boot(None).unwrap(); 3152 3153 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3154 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3155 3156 let grep_cmd = if cfg!(target_arch = "x86_64") { 3157 "grep -c PCI-MSI /proc/interrupts" 3158 } else { 3159 "grep -c ITS-MSI /proc/interrupts" 3160 }; 3161 assert_eq!( 3162 guest 3163 .ssh_command(grep_cmd) 3164 .unwrap() 3165 .trim() 3166 .parse::<u32>() 3167 .unwrap_or_default(), 3168 12 3169 ); 3170 }); 3171 3172 let _ = child.kill(); 3173 let output = child.wait_with_output().unwrap(); 3174 3175 handle_child_output(r, &output); 3176 } 3177 3178 #[test] 3179 #[cfg(target_arch = "x86_64")] 3180 fn test_direct_kernel_boot_bzimage() { 3181 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3182 let guest = Guest::new(Box::new(focal)); 3183 3184 let mut kernel_path = direct_kernel_boot_path(); 3185 // Replace the default kernel with the bzImage. 3186 kernel_path.pop(); 3187 kernel_path.push("bzImage"); 3188 3189 let mut child = GuestCommand::new(&guest) 3190 .args(["--cpus", "boot=1"]) 3191 .args(["--memory", "size=512M"]) 3192 .args(["--kernel", kernel_path.to_str().unwrap()]) 3193 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3194 .default_disks() 3195 .default_net() 3196 .capture_output() 3197 .spawn() 3198 .unwrap(); 3199 3200 let r = std::panic::catch_unwind(|| { 3201 guest.wait_vm_boot(None).unwrap(); 3202 3203 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3204 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3205 3206 let grep_cmd = if cfg!(target_arch = "x86_64") { 3207 "grep -c PCI-MSI /proc/interrupts" 3208 } else { 3209 "grep -c ITS-MSI /proc/interrupts" 3210 }; 3211 assert_eq!( 3212 guest 3213 .ssh_command(grep_cmd) 3214 .unwrap() 3215 .trim() 3216 .parse::<u32>() 3217 .unwrap_or_default(), 3218 12 3219 ); 3220 }); 3221 3222 let _ = child.kill(); 3223 let output = child.wait_with_output().unwrap(); 3224 3225 handle_child_output(r, &output); 3226 } 3227 3228 fn _test_virtio_block(image_name: &str, disable_io_uring: bool, disable_aio: bool) { 3229 let focal = UbuntuDiskConfig::new(image_name.to_string()); 3230 let guest = Guest::new(Box::new(focal)); 3231 3232 let mut workload_path = dirs::home_dir().unwrap(); 3233 workload_path.push("workloads"); 3234 3235 let mut blk_file_path = workload_path; 3236 blk_file_path.push("blk.img"); 3237 3238 let kernel_path = direct_kernel_boot_path(); 3239 3240 let mut cloud_child = GuestCommand::new(&guest) 3241 .args(["--cpus", "boot=4"]) 3242 .args(["--memory", "size=512M,shared=on"]) 3243 .args(["--kernel", kernel_path.to_str().unwrap()]) 3244 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3245 .args([ 3246 "--disk", 3247 format!( 3248 "path={}", 3249 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3250 ) 3251 .as_str(), 3252 format!( 3253 "path={}", 3254 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3255 ) 3256 .as_str(), 3257 format!( 3258 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", 3259 blk_file_path.to_str().unwrap(), 3260 disable_io_uring, 3261 disable_aio, 3262 ) 3263 .as_str(), 3264 ]) 3265 .default_net() 3266 .capture_output() 3267 .spawn() 3268 .unwrap(); 3269 3270 let r = std::panic::catch_unwind(|| { 3271 guest.wait_vm_boot(None).unwrap(); 3272 3273 // Check both if /dev/vdc exists and if the block size is 16M. 3274 assert_eq!( 3275 guest 3276 .ssh_command("lsblk | grep vdc | grep -c 16M") 3277 .unwrap() 3278 .trim() 3279 .parse::<u32>() 3280 .unwrap_or_default(), 3281 1 3282 ); 3283 3284 // Check both if /dev/vdc exists and if this block is RO. 3285 assert_eq!( 3286 guest 3287 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 3288 .unwrap() 3289 .trim() 3290 .parse::<u32>() 3291 .unwrap_or_default(), 3292 1 3293 ); 3294 3295 // Check if the number of queues is 4. 3296 assert_eq!( 3297 guest 3298 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 3299 .unwrap() 3300 .trim() 3301 .parse::<u32>() 3302 .unwrap_or_default(), 3303 4 3304 ); 3305 }); 3306 3307 let _ = cloud_child.kill(); 3308 let output = cloud_child.wait_with_output().unwrap(); 3309 3310 handle_child_output(r, &output); 3311 } 3312 3313 #[test] 3314 fn test_virtio_block_io_uring() { 3315 _test_virtio_block(FOCAL_IMAGE_NAME, false, true) 3316 } 3317 3318 #[test] 3319 fn test_virtio_block_aio() { 3320 _test_virtio_block(FOCAL_IMAGE_NAME, true, false) 3321 } 3322 3323 #[test] 3324 fn test_virtio_block_sync() { 3325 _test_virtio_block(FOCAL_IMAGE_NAME, true, true) 3326 } 3327 3328 #[test] 3329 fn test_virtio_block_qcow2() { 3330 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false, false) 3331 } 3332 3333 #[test] 3334 fn test_virtio_block_qcow2_backing_file() { 3335 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE, false, false) 3336 } 3337 3338 #[test] 3339 fn test_virtio_block_vhd() { 3340 let mut workload_path = dirs::home_dir().unwrap(); 3341 workload_path.push("workloads"); 3342 3343 let mut raw_file_path = workload_path.clone(); 3344 let mut vhd_file_path = workload_path; 3345 raw_file_path.push(FOCAL_IMAGE_NAME); 3346 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 3347 3348 // Generate VHD file from RAW file 3349 std::process::Command::new("qemu-img") 3350 .arg("convert") 3351 .arg("-p") 3352 .args(["-f", "raw"]) 3353 .args(["-O", "vpc"]) 3354 .args(["-o", "subformat=fixed"]) 3355 .arg(raw_file_path.to_str().unwrap()) 3356 .arg(vhd_file_path.to_str().unwrap()) 3357 .output() 3358 .expect("Expect generating VHD image from RAW image"); 3359 3360 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false, false) 3361 } 3362 3363 #[test] 3364 fn test_virtio_block_vhdx() { 3365 let mut workload_path = dirs::home_dir().unwrap(); 3366 workload_path.push("workloads"); 3367 3368 let mut raw_file_path = workload_path.clone(); 3369 let mut vhdx_file_path = workload_path; 3370 raw_file_path.push(FOCAL_IMAGE_NAME); 3371 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 3372 3373 // Generate dynamic VHDX file from RAW file 3374 std::process::Command::new("qemu-img") 3375 .arg("convert") 3376 .arg("-p") 3377 .args(["-f", "raw"]) 3378 .args(["-O", "vhdx"]) 3379 .arg(raw_file_path.to_str().unwrap()) 3380 .arg(vhdx_file_path.to_str().unwrap()) 3381 .output() 3382 .expect("Expect generating dynamic VHDx image from RAW image"); 3383 3384 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false, false) 3385 } 3386 3387 #[test] 3388 fn test_virtio_block_dynamic_vhdx_expand() { 3389 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 3390 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 3391 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 3392 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 3393 3394 let mut workload_path = dirs::home_dir().unwrap(); 3395 workload_path.push("workloads"); 3396 3397 let mut vhdx_file_path = workload_path; 3398 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 3399 let vhdx_path = vhdx_file_path.to_str().unwrap(); 3400 3401 // Generate a 100 MiB dynamic VHDX file 3402 std::process::Command::new("qemu-img") 3403 .arg("create") 3404 .args(["-f", "vhdx"]) 3405 .arg(vhdx_path) 3406 .arg(VIRTUAL_DISK_SIZE.to_string()) 3407 .output() 3408 .expect("Expect generating dynamic VHDx image from RAW image"); 3409 3410 // Check if the size matches with empty VHDx file size 3411 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 3412 3413 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3414 let guest = Guest::new(Box::new(focal)); 3415 let kernel_path = direct_kernel_boot_path(); 3416 3417 let mut cloud_child = GuestCommand::new(&guest) 3418 .args(["--cpus", "boot=1"]) 3419 .args(["--memory", "size=512M"]) 3420 .args(["--kernel", kernel_path.to_str().unwrap()]) 3421 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3422 .args([ 3423 "--disk", 3424 format!( 3425 "path={}", 3426 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3427 ) 3428 .as_str(), 3429 format!( 3430 "path={}", 3431 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3432 ) 3433 .as_str(), 3434 format!("path={vhdx_path}").as_str(), 3435 ]) 3436 .default_net() 3437 .capture_output() 3438 .spawn() 3439 .unwrap(); 3440 3441 let r = std::panic::catch_unwind(|| { 3442 guest.wait_vm_boot(None).unwrap(); 3443 3444 // Check both if /dev/vdc exists and if the block size is 100 MiB. 3445 assert_eq!( 3446 guest 3447 .ssh_command("lsblk | grep vdc | grep -c 100M") 3448 .unwrap() 3449 .trim() 3450 .parse::<u32>() 3451 .unwrap_or_default(), 3452 1 3453 ); 3454 3455 // Write 100 MB of data to the VHDx disk 3456 guest 3457 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 3458 .unwrap(); 3459 }); 3460 3461 // Check if the size matches with expected expanded VHDx file size 3462 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 3463 3464 let _ = cloud_child.kill(); 3465 let output = cloud_child.wait_with_output().unwrap(); 3466 3467 handle_child_output(r, &output); 3468 } 3469 3470 fn vhdx_image_size(disk_name: &str) -> u64 { 3471 std::fs::File::open(disk_name) 3472 .unwrap() 3473 .seek(SeekFrom::End(0)) 3474 .unwrap() 3475 } 3476 3477 #[test] 3478 fn test_virtio_block_direct_and_firmware() { 3479 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3480 let guest = Guest::new(Box::new(focal)); 3481 3482 // The OS disk must be copied to a location that is not backed by 3483 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 3484 // with EINVAL because tmpfs doesn't support this flag. 3485 let mut workloads_path = dirs::home_dir().unwrap(); 3486 workloads_path.push("workloads"); 3487 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 3488 let mut os_path = os_dir.as_path().to_path_buf(); 3489 os_path.push("osdisk.img"); 3490 rate_limited_copy( 3491 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3492 os_path.as_path(), 3493 ) 3494 .expect("copying of OS disk failed"); 3495 3496 let mut child = GuestCommand::new(&guest) 3497 .args(["--cpus", "boot=1"]) 3498 .args(["--memory", "size=512M"]) 3499 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 3500 .args([ 3501 "--disk", 3502 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 3503 format!( 3504 "path={}", 3505 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3506 ) 3507 .as_str(), 3508 ]) 3509 .default_net() 3510 .capture_output() 3511 .spawn() 3512 .unwrap(); 3513 3514 let r = std::panic::catch_unwind(|| { 3515 guest.wait_vm_boot(Some(120)).unwrap(); 3516 }); 3517 3518 let _ = child.kill(); 3519 let output = child.wait_with_output().unwrap(); 3520 3521 handle_child_output(r, &output); 3522 } 3523 3524 #[test] 3525 fn test_vhost_user_net_default() { 3526 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 3527 } 3528 3529 #[test] 3530 fn test_vhost_user_net_named_tap() { 3531 test_vhost_user_net( 3532 Some("mytap0"), 3533 2, 3534 &prepare_vhost_user_net_daemon, 3535 false, 3536 false, 3537 ) 3538 } 3539 3540 #[test] 3541 fn test_vhost_user_net_existing_tap() { 3542 test_vhost_user_net( 3543 Some("vunet-tap0"), 3544 2, 3545 &prepare_vhost_user_net_daemon, 3546 false, 3547 false, 3548 ) 3549 } 3550 3551 #[test] 3552 fn test_vhost_user_net_multiple_queues() { 3553 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 3554 } 3555 3556 #[test] 3557 fn test_vhost_user_net_tap_multiple_queues() { 3558 test_vhost_user_net( 3559 Some("vunet-tap1"), 3560 4, 3561 &prepare_vhost_user_net_daemon, 3562 false, 3563 false, 3564 ) 3565 } 3566 3567 #[test] 3568 fn test_vhost_user_net_host_mac() { 3569 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 3570 } 3571 3572 #[test] 3573 fn test_vhost_user_net_client_mode() { 3574 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 3575 } 3576 3577 #[test] 3578 #[cfg(not(target_arch = "aarch64"))] 3579 fn test_vhost_user_blk_default() { 3580 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 3581 } 3582 3583 #[test] 3584 #[cfg(not(target_arch = "aarch64"))] 3585 fn test_vhost_user_blk_readonly() { 3586 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3587 } 3588 3589 #[test] 3590 #[cfg(not(target_arch = "aarch64"))] 3591 fn test_vhost_user_blk_direct() { 3592 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3593 } 3594 3595 #[test] 3596 fn test_boot_from_vhost_user_blk_default() { 3597 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3598 } 3599 3600 #[test] 3601 #[cfg(target_arch = "x86_64")] 3602 fn test_split_irqchip() { 3603 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3604 let guest = Guest::new(Box::new(focal)); 3605 3606 let mut child = GuestCommand::new(&guest) 3607 .args(["--cpus", "boot=1"]) 3608 .args(["--memory", "size=512M"]) 3609 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3610 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3611 .default_disks() 3612 .default_net() 3613 .capture_output() 3614 .spawn() 3615 .unwrap(); 3616 3617 let r = std::panic::catch_unwind(|| { 3618 guest.wait_vm_boot(None).unwrap(); 3619 3620 assert_eq!( 3621 guest 3622 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3623 .unwrap() 3624 .trim() 3625 .parse::<u32>() 3626 .unwrap_or(1), 3627 0 3628 ); 3629 assert_eq!( 3630 guest 3631 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3632 .unwrap() 3633 .trim() 3634 .parse::<u32>() 3635 .unwrap_or(1), 3636 0 3637 ); 3638 }); 3639 3640 let _ = child.kill(); 3641 let output = child.wait_with_output().unwrap(); 3642 3643 handle_child_output(r, &output); 3644 } 3645 3646 #[test] 3647 #[cfg(target_arch = "x86_64")] 3648 fn test_dmi_serial_number() { 3649 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3650 let guest = Guest::new(Box::new(focal)); 3651 3652 let mut child = GuestCommand::new(&guest) 3653 .args(["--cpus", "boot=1"]) 3654 .args(["--memory", "size=512M"]) 3655 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3656 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3657 .args(["--platform", "serial_number=a=b;c=d"]) 3658 .default_disks() 3659 .default_net() 3660 .capture_output() 3661 .spawn() 3662 .unwrap(); 3663 3664 let r = std::panic::catch_unwind(|| { 3665 guest.wait_vm_boot(None).unwrap(); 3666 3667 assert_eq!( 3668 guest 3669 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3670 .unwrap() 3671 .trim(), 3672 "a=b;c=d" 3673 ); 3674 }); 3675 3676 let _ = child.kill(); 3677 let output = child.wait_with_output().unwrap(); 3678 3679 handle_child_output(r, &output); 3680 } 3681 3682 #[test] 3683 #[cfg(target_arch = "x86_64")] 3684 fn test_dmi_uuid() { 3685 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3686 let guest = Guest::new(Box::new(focal)); 3687 3688 let mut child = GuestCommand::new(&guest) 3689 .args(["--cpus", "boot=1"]) 3690 .args(["--memory", "size=512M"]) 3691 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3692 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3693 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3694 .default_disks() 3695 .default_net() 3696 .capture_output() 3697 .spawn() 3698 .unwrap(); 3699 3700 let r = std::panic::catch_unwind(|| { 3701 guest.wait_vm_boot(None).unwrap(); 3702 3703 assert_eq!( 3704 guest 3705 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3706 .unwrap() 3707 .trim(), 3708 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3709 ); 3710 }); 3711 3712 let _ = child.kill(); 3713 let output = child.wait_with_output().unwrap(); 3714 3715 handle_child_output(r, &output); 3716 } 3717 3718 #[test] 3719 #[cfg(target_arch = "x86_64")] 3720 fn test_dmi_oem_strings() { 3721 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3722 let guest = Guest::new(Box::new(focal)); 3723 3724 let s1 = "io.systemd.credential:xx=yy"; 3725 let s2 = "This is a test string"; 3726 3727 let oem_strings = format!("oem_strings=[{s1},{s2}]"); 3728 3729 let mut child = GuestCommand::new(&guest) 3730 .args(["--cpus", "boot=1"]) 3731 .args(["--memory", "size=512M"]) 3732 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3733 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3734 .args(["--platform", &oem_strings]) 3735 .default_disks() 3736 .default_net() 3737 .capture_output() 3738 .spawn() 3739 .unwrap(); 3740 3741 let r = std::panic::catch_unwind(|| { 3742 guest.wait_vm_boot(None).unwrap(); 3743 3744 assert_eq!( 3745 guest 3746 .ssh_command("sudo dmidecode --oem-string count") 3747 .unwrap() 3748 .trim(), 3749 "2" 3750 ); 3751 3752 assert_eq!( 3753 guest 3754 .ssh_command("sudo dmidecode --oem-string 1") 3755 .unwrap() 3756 .trim(), 3757 s1 3758 ); 3759 3760 assert_eq!( 3761 guest 3762 .ssh_command("sudo dmidecode --oem-string 2") 3763 .unwrap() 3764 .trim(), 3765 s2 3766 ); 3767 }); 3768 3769 let _ = child.kill(); 3770 let output = child.wait_with_output().unwrap(); 3771 3772 handle_child_output(r, &output); 3773 } 3774 3775 #[test] 3776 fn test_virtio_fs() { 3777 _test_virtio_fs(&prepare_virtiofsd, false, None) 3778 } 3779 3780 #[test] 3781 fn test_virtio_fs_hotplug() { 3782 _test_virtio_fs(&prepare_virtiofsd, true, None) 3783 } 3784 3785 #[test] 3786 #[cfg(not(feature = "mshv"))] 3787 fn test_virtio_fs_multi_segment_hotplug() { 3788 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3789 } 3790 3791 #[test] 3792 #[cfg(not(feature = "mshv"))] 3793 fn test_virtio_fs_multi_segment() { 3794 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3795 } 3796 3797 #[test] 3798 fn test_virtio_pmem_persist_writes() { 3799 test_virtio_pmem(false, false) 3800 } 3801 3802 #[test] 3803 fn test_virtio_pmem_discard_writes() { 3804 test_virtio_pmem(true, false) 3805 } 3806 3807 #[test] 3808 fn test_virtio_pmem_with_size() { 3809 test_virtio_pmem(true, true) 3810 } 3811 3812 #[test] 3813 fn test_boot_from_virtio_pmem() { 3814 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3815 let guest = Guest::new(Box::new(focal)); 3816 3817 let kernel_path = direct_kernel_boot_path(); 3818 3819 let mut child = GuestCommand::new(&guest) 3820 .args(["--cpus", "boot=1"]) 3821 .args(["--memory", "size=512M"]) 3822 .args(["--kernel", kernel_path.to_str().unwrap()]) 3823 .args([ 3824 "--disk", 3825 format!( 3826 "path={}", 3827 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3828 ) 3829 .as_str(), 3830 ]) 3831 .default_net() 3832 .args([ 3833 "--pmem", 3834 format!( 3835 "file={},size={}", 3836 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3837 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3838 .unwrap() 3839 .len() 3840 ) 3841 .as_str(), 3842 ]) 3843 .args([ 3844 "--cmdline", 3845 DIRECT_KERNEL_BOOT_CMDLINE 3846 .replace("vda1", "pmem0p1") 3847 .as_str(), 3848 ]) 3849 .capture_output() 3850 .spawn() 3851 .unwrap(); 3852 3853 let r = std::panic::catch_unwind(|| { 3854 guest.wait_vm_boot(None).unwrap(); 3855 3856 // Simple checks to validate the VM booted properly 3857 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3858 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3859 }); 3860 3861 let _ = child.kill(); 3862 let output = child.wait_with_output().unwrap(); 3863 3864 handle_child_output(r, &output); 3865 } 3866 3867 #[test] 3868 fn test_multiple_network_interfaces() { 3869 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3870 let guest = Guest::new(Box::new(focal)); 3871 3872 let kernel_path = direct_kernel_boot_path(); 3873 3874 let mut child = GuestCommand::new(&guest) 3875 .args(["--cpus", "boot=1"]) 3876 .args(["--memory", "size=512M"]) 3877 .args(["--kernel", kernel_path.to_str().unwrap()]) 3878 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3879 .default_disks() 3880 .args([ 3881 "--net", 3882 guest.default_net_string().as_str(), 3883 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3884 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3885 ]) 3886 .capture_output() 3887 .spawn() 3888 .unwrap(); 3889 3890 let r = std::panic::catch_unwind(|| { 3891 guest.wait_vm_boot(None).unwrap(); 3892 3893 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3894 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3895 3896 // 3 network interfaces + default localhost ==> 4 interfaces 3897 assert_eq!( 3898 guest 3899 .ssh_command("ip -o link | wc -l") 3900 .unwrap() 3901 .trim() 3902 .parse::<u32>() 3903 .unwrap_or_default(), 3904 4 3905 ); 3906 }); 3907 3908 let _ = child.kill(); 3909 let output = child.wait_with_output().unwrap(); 3910 3911 handle_child_output(r, &output); 3912 } 3913 3914 #[test] 3915 #[cfg(target_arch = "aarch64")] 3916 fn test_pmu_on() { 3917 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3918 let guest = Guest::new(Box::new(focal)); 3919 let mut child = GuestCommand::new(&guest) 3920 .args(["--cpus", "boot=1"]) 3921 .args(["--memory", "size=512M"]) 3922 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3923 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3924 .default_disks() 3925 .default_net() 3926 .capture_output() 3927 .spawn() 3928 .unwrap(); 3929 3930 let r = std::panic::catch_unwind(|| { 3931 guest.wait_vm_boot(None).unwrap(); 3932 3933 // Test that PMU exists. 3934 assert_eq!( 3935 guest 3936 .ssh_command(GREP_PMU_IRQ_CMD) 3937 .unwrap() 3938 .trim() 3939 .parse::<u32>() 3940 .unwrap_or_default(), 3941 1 3942 ); 3943 }); 3944 3945 let _ = child.kill(); 3946 let output = child.wait_with_output().unwrap(); 3947 3948 handle_child_output(r, &output); 3949 } 3950 3951 #[test] 3952 fn test_serial_off() { 3953 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3954 let guest = Guest::new(Box::new(focal)); 3955 let mut child = GuestCommand::new(&guest) 3956 .args(["--cpus", "boot=1"]) 3957 .args(["--memory", "size=512M"]) 3958 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3959 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3960 .default_disks() 3961 .default_net() 3962 .args(["--serial", "off"]) 3963 .capture_output() 3964 .spawn() 3965 .unwrap(); 3966 3967 let r = std::panic::catch_unwind(|| { 3968 guest.wait_vm_boot(None).unwrap(); 3969 3970 // Test that there is no ttyS0 3971 assert_eq!( 3972 guest 3973 .ssh_command(GREP_SERIAL_IRQ_CMD) 3974 .unwrap() 3975 .trim() 3976 .parse::<u32>() 3977 .unwrap_or(1), 3978 0 3979 ); 3980 }); 3981 3982 let _ = child.kill(); 3983 let output = child.wait_with_output().unwrap(); 3984 3985 handle_child_output(r, &output); 3986 } 3987 3988 #[test] 3989 fn test_serial_null() { 3990 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3991 let guest = Guest::new(Box::new(focal)); 3992 let mut cmd = GuestCommand::new(&guest); 3993 #[cfg(target_arch = "x86_64")] 3994 let console_str: &str = "console=ttyS0"; 3995 #[cfg(target_arch = "aarch64")] 3996 let console_str: &str = "console=ttyAMA0"; 3997 3998 cmd.args(["--cpus", "boot=1"]) 3999 .args(["--memory", "size=512M"]) 4000 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4001 .args([ 4002 "--cmdline", 4003 DIRECT_KERNEL_BOOT_CMDLINE 4004 .replace("console=hvc0 ", console_str) 4005 .as_str(), 4006 ]) 4007 .default_disks() 4008 .default_net() 4009 .args(["--serial", "null"]) 4010 .args(["--console", "off"]) 4011 .capture_output(); 4012 4013 let mut child = cmd.spawn().unwrap(); 4014 4015 let r = std::panic::catch_unwind(|| { 4016 guest.wait_vm_boot(None).unwrap(); 4017 4018 // Test that there is a ttyS0 4019 assert_eq!( 4020 guest 4021 .ssh_command(GREP_SERIAL_IRQ_CMD) 4022 .unwrap() 4023 .trim() 4024 .parse::<u32>() 4025 .unwrap_or_default(), 4026 1 4027 ); 4028 }); 4029 4030 let _ = child.kill(); 4031 let output = child.wait_with_output().unwrap(); 4032 handle_child_output(r, &output); 4033 4034 let r = std::panic::catch_unwind(|| { 4035 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4036 }); 4037 4038 handle_child_output(r, &output); 4039 } 4040 4041 #[test] 4042 fn test_serial_tty() { 4043 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4044 let guest = Guest::new(Box::new(focal)); 4045 4046 let kernel_path = direct_kernel_boot_path(); 4047 4048 #[cfg(target_arch = "x86_64")] 4049 let console_str: &str = "console=ttyS0"; 4050 #[cfg(target_arch = "aarch64")] 4051 let console_str: &str = "console=ttyAMA0"; 4052 4053 let mut child = GuestCommand::new(&guest) 4054 .args(["--cpus", "boot=1"]) 4055 .args(["--memory", "size=512M"]) 4056 .args(["--kernel", kernel_path.to_str().unwrap()]) 4057 .args([ 4058 "--cmdline", 4059 DIRECT_KERNEL_BOOT_CMDLINE 4060 .replace("console=hvc0 ", console_str) 4061 .as_str(), 4062 ]) 4063 .default_disks() 4064 .default_net() 4065 .args(["--serial", "tty"]) 4066 .args(["--console", "off"]) 4067 .capture_output() 4068 .spawn() 4069 .unwrap(); 4070 4071 let r = std::panic::catch_unwind(|| { 4072 guest.wait_vm_boot(None).unwrap(); 4073 4074 // Test that there is a ttyS0 4075 assert_eq!( 4076 guest 4077 .ssh_command(GREP_SERIAL_IRQ_CMD) 4078 .unwrap() 4079 .trim() 4080 .parse::<u32>() 4081 .unwrap_or_default(), 4082 1 4083 ); 4084 }); 4085 4086 // This sleep is needed to wait for the login prompt 4087 thread::sleep(std::time::Duration::new(2, 0)); 4088 4089 let _ = child.kill(); 4090 let output = child.wait_with_output().unwrap(); 4091 handle_child_output(r, &output); 4092 4093 let r = std::panic::catch_unwind(|| { 4094 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4095 }); 4096 4097 handle_child_output(r, &output); 4098 } 4099 4100 #[test] 4101 fn test_serial_file() { 4102 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4103 let guest = Guest::new(Box::new(focal)); 4104 4105 let serial_path = guest.tmp_dir.as_path().join("serial-output"); 4106 #[cfg(target_arch = "x86_64")] 4107 let console_str: &str = "console=ttyS0"; 4108 #[cfg(target_arch = "aarch64")] 4109 let console_str: &str = "console=ttyAMA0"; 4110 4111 let mut child = GuestCommand::new(&guest) 4112 .args(["--cpus", "boot=1"]) 4113 .args(["--memory", "size=512M"]) 4114 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4115 .args([ 4116 "--cmdline", 4117 DIRECT_KERNEL_BOOT_CMDLINE 4118 .replace("console=hvc0 ", console_str) 4119 .as_str(), 4120 ]) 4121 .default_disks() 4122 .default_net() 4123 .args([ 4124 "--serial", 4125 format!("file={}", serial_path.to_str().unwrap()).as_str(), 4126 ]) 4127 .capture_output() 4128 .spawn() 4129 .unwrap(); 4130 4131 let r = std::panic::catch_unwind(|| { 4132 guest.wait_vm_boot(None).unwrap(); 4133 4134 // Test that there is a ttyS0 4135 assert_eq!( 4136 guest 4137 .ssh_command(GREP_SERIAL_IRQ_CMD) 4138 .unwrap() 4139 .trim() 4140 .parse::<u32>() 4141 .unwrap_or_default(), 4142 1 4143 ); 4144 4145 guest.ssh_command("sudo shutdown -h now").unwrap(); 4146 }); 4147 4148 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4149 let _ = child.kill(); 4150 let output = child.wait_with_output().unwrap(); 4151 handle_child_output(r, &output); 4152 4153 let r = std::panic::catch_unwind(|| { 4154 // Check that the cloud-hypervisor binary actually terminated 4155 assert!(output.status.success()); 4156 4157 // Do this check after shutdown of the VM as an easy way to ensure 4158 // all writes are flushed to disk 4159 let mut f = std::fs::File::open(serial_path).unwrap(); 4160 let mut buf = String::new(); 4161 f.read_to_string(&mut buf).unwrap(); 4162 assert!(buf.contains(CONSOLE_TEST_STRING)); 4163 }); 4164 4165 handle_child_output(r, &output); 4166 } 4167 4168 #[test] 4169 fn test_pty_interaction() { 4170 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4171 let guest = Guest::new(Box::new(focal)); 4172 let api_socket = temp_api_path(&guest.tmp_dir); 4173 let serial_option = if cfg!(target_arch = "x86_64") { 4174 " console=ttyS0" 4175 } else { 4176 " console=ttyAMA0" 4177 }; 4178 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4179 4180 let mut child = GuestCommand::new(&guest) 4181 .args(["--cpus", "boot=1"]) 4182 .args(["--memory", "size=512M"]) 4183 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4184 .args(["--cmdline", &cmdline]) 4185 .default_disks() 4186 .default_net() 4187 .args(["--serial", "null"]) 4188 .args(["--console", "pty"]) 4189 .args(["--api-socket", &api_socket]) 4190 .spawn() 4191 .unwrap(); 4192 4193 let r = std::panic::catch_unwind(|| { 4194 guest.wait_vm_boot(None).unwrap(); 4195 // Get pty fd for console 4196 let console_path = get_pty_path(&api_socket, "console"); 4197 _test_pty_interaction(console_path); 4198 4199 guest.ssh_command("sudo shutdown -h now").unwrap(); 4200 }); 4201 4202 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4203 let _ = child.kill(); 4204 let output = child.wait_with_output().unwrap(); 4205 handle_child_output(r, &output); 4206 4207 let r = std::panic::catch_unwind(|| { 4208 // Check that the cloud-hypervisor binary actually terminated 4209 assert!(output.status.success()) 4210 }); 4211 handle_child_output(r, &output); 4212 } 4213 4214 #[test] 4215 fn test_serial_socket_interaction() { 4216 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4217 let guest = Guest::new(Box::new(focal)); 4218 let serial_socket = guest.tmp_dir.as_path().join("serial.socket"); 4219 let serial_socket_pty = guest.tmp_dir.as_path().join("serial.pty"); 4220 let serial_option = if cfg!(target_arch = "x86_64") { 4221 " console=ttyS0" 4222 } else { 4223 " console=ttyAMA0" 4224 }; 4225 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4226 4227 let mut child = GuestCommand::new(&guest) 4228 .args(["--cpus", "boot=1"]) 4229 .args(["--memory", "size=512M"]) 4230 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4231 .args(["--cmdline", &cmdline]) 4232 .default_disks() 4233 .default_net() 4234 .args(["--console", "null"]) 4235 .args([ 4236 "--serial", 4237 format!("socket={}", serial_socket.to_str().unwrap()).as_str(), 4238 ]) 4239 .spawn() 4240 .unwrap(); 4241 4242 let _ = std::panic::catch_unwind(|| { 4243 guest.wait_vm_boot(None).unwrap(); 4244 }); 4245 4246 let mut socat_command = Command::new("socat"); 4247 let socat_args = [ 4248 &format!("pty,link={},raw", serial_socket_pty.display()), 4249 &format!("UNIX-CONNECT:{}", serial_socket.display()), 4250 ]; 4251 socat_command.args(socat_args); 4252 4253 let mut socat_child = socat_command.spawn().unwrap(); 4254 thread::sleep(std::time::Duration::new(1, 0)); 4255 4256 let _ = std::panic::catch_unwind(|| { 4257 _test_pty_interaction(serial_socket_pty); 4258 }); 4259 4260 let _ = socat_child.kill(); 4261 4262 let r = std::panic::catch_unwind(|| { 4263 guest.ssh_command("sudo shutdown -h now").unwrap(); 4264 }); 4265 4266 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4267 let _ = child.kill(); 4268 let output = child.wait_with_output().unwrap(); 4269 handle_child_output(r, &output); 4270 4271 let r = std::panic::catch_unwind(|| { 4272 // Check that the cloud-hypervisor binary actually terminated 4273 if !output.status.success() { 4274 panic!( 4275 "Cloud Hypervisor process failed to terminate gracefully: {:?}", 4276 output.status 4277 ); 4278 } 4279 }); 4280 handle_child_output(r, &output); 4281 } 4282 4283 #[test] 4284 fn test_virtio_console() { 4285 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4286 let guest = Guest::new(Box::new(focal)); 4287 4288 let kernel_path = direct_kernel_boot_path(); 4289 4290 let mut child = GuestCommand::new(&guest) 4291 .args(["--cpus", "boot=1"]) 4292 .args(["--memory", "size=512M"]) 4293 .args(["--kernel", kernel_path.to_str().unwrap()]) 4294 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4295 .default_disks() 4296 .default_net() 4297 .args(["--console", "tty"]) 4298 .args(["--serial", "null"]) 4299 .capture_output() 4300 .spawn() 4301 .unwrap(); 4302 4303 let text = String::from("On a branch floating down river a cricket, singing."); 4304 let cmd = format!("echo {text} | sudo tee /dev/hvc0"); 4305 4306 let r = std::panic::catch_unwind(|| { 4307 guest.wait_vm_boot(None).unwrap(); 4308 4309 assert!(guest 4310 .does_device_vendor_pair_match("0x1043", "0x1af4") 4311 .unwrap_or_default()); 4312 4313 guest.ssh_command(&cmd).unwrap(); 4314 }); 4315 4316 let _ = child.kill(); 4317 let output = child.wait_with_output().unwrap(); 4318 handle_child_output(r, &output); 4319 4320 let r = std::panic::catch_unwind(|| { 4321 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 4322 }); 4323 4324 handle_child_output(r, &output); 4325 } 4326 4327 #[test] 4328 fn test_console_file() { 4329 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4330 let guest = Guest::new(Box::new(focal)); 4331 4332 let console_path = guest.tmp_dir.as_path().join("console-output"); 4333 let mut child = GuestCommand::new(&guest) 4334 .args(["--cpus", "boot=1"]) 4335 .args(["--memory", "size=512M"]) 4336 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4337 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4338 .default_disks() 4339 .default_net() 4340 .args([ 4341 "--console", 4342 format!("file={}", console_path.to_str().unwrap()).as_str(), 4343 ]) 4344 .capture_output() 4345 .spawn() 4346 .unwrap(); 4347 4348 guest.wait_vm_boot(None).unwrap(); 4349 4350 guest.ssh_command("sudo shutdown -h now").unwrap(); 4351 4352 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4353 let _ = child.kill(); 4354 let output = child.wait_with_output().unwrap(); 4355 4356 let r = std::panic::catch_unwind(|| { 4357 // Check that the cloud-hypervisor binary actually terminated 4358 assert!(output.status.success()); 4359 4360 // Do this check after shutdown of the VM as an easy way to ensure 4361 // all writes are flushed to disk 4362 let mut f = std::fs::File::open(console_path).unwrap(); 4363 let mut buf = String::new(); 4364 f.read_to_string(&mut buf).unwrap(); 4365 4366 if !buf.contains(CONSOLE_TEST_STRING) { 4367 eprintln!( 4368 "\n\n==== Console file output ====\n\n{buf}\n\n==== End console file output ====" 4369 ); 4370 } 4371 assert!(buf.contains(CONSOLE_TEST_STRING)); 4372 }); 4373 4374 handle_child_output(r, &output); 4375 } 4376 4377 #[test] 4378 #[cfg(target_arch = "x86_64")] 4379 #[cfg(not(feature = "mshv"))] 4380 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 4381 // backed networking interfaces, bound through a simple bridge on the host. 4382 // So if the nested cloud-hypervisor succeeds in getting a directly 4383 // assigned interface from its cloud-hypervisor host, we should be able to 4384 // ssh into it, and verify that it's running with the right kernel command 4385 // line (We tag the command line from cloud-hypervisor for that purpose). 4386 // The third device is added to validate that hotplug works correctly since 4387 // it is being added to the L2 VM through hotplugging mechanism. 4388 // Also, we pass-through a virtio-blk device to the L2 VM to test the 32-bit 4389 // vfio device support 4390 fn test_vfio() { 4391 setup_vfio_network_interfaces(); 4392 4393 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 4394 let guest = Guest::new_from_ip_range(Box::new(jammy), "172.18", 0); 4395 4396 let mut workload_path = dirs::home_dir().unwrap(); 4397 workload_path.push("workloads"); 4398 4399 let kernel_path = direct_kernel_boot_path(); 4400 4401 let mut vfio_path = workload_path.clone(); 4402 vfio_path.push("vfio"); 4403 4404 let mut cloud_init_vfio_base_path = vfio_path.clone(); 4405 cloud_init_vfio_base_path.push("cloudinit.img"); 4406 4407 // We copy our cloudinit into the vfio mount point, for the nested 4408 // cloud-hypervisor guest to use. 4409 rate_limited_copy( 4410 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 4411 &cloud_init_vfio_base_path, 4412 ) 4413 .expect("copying of cloud-init disk failed"); 4414 4415 let mut vfio_disk_path = workload_path.clone(); 4416 vfio_disk_path.push("vfio.img"); 4417 4418 // Create the vfio disk image 4419 let output = Command::new("mkfs.ext4") 4420 .arg("-d") 4421 .arg(vfio_path.to_str().unwrap()) 4422 .arg(vfio_disk_path.to_str().unwrap()) 4423 .arg("2g") 4424 .output() 4425 .unwrap(); 4426 if !output.status.success() { 4427 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 4428 panic!("mkfs.ext4 command generated an error"); 4429 } 4430 4431 let mut blk_file_path = workload_path; 4432 blk_file_path.push("blk.img"); 4433 4434 let vfio_tap0 = "vfio-tap0"; 4435 let vfio_tap1 = "vfio-tap1"; 4436 let vfio_tap2 = "vfio-tap2"; 4437 let vfio_tap3 = "vfio-tap3"; 4438 4439 let mut child = GuestCommand::new(&guest) 4440 .args(["--cpus", "boot=4"]) 4441 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 4442 .args(["--kernel", kernel_path.to_str().unwrap()]) 4443 .args([ 4444 "--disk", 4445 format!( 4446 "path={}", 4447 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4448 ) 4449 .as_str(), 4450 format!( 4451 "path={}", 4452 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4453 ) 4454 .as_str(), 4455 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 4456 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 4457 ]) 4458 .args([ 4459 "--cmdline", 4460 format!( 4461 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 4462 ) 4463 .as_str(), 4464 ]) 4465 .args([ 4466 "--net", 4467 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 4468 format!( 4469 "tap={},mac={},iommu=on", 4470 vfio_tap1, guest.network.l2_guest_mac1 4471 ) 4472 .as_str(), 4473 format!( 4474 "tap={},mac={},iommu=on", 4475 vfio_tap2, guest.network.l2_guest_mac2 4476 ) 4477 .as_str(), 4478 format!( 4479 "tap={},mac={},iommu=on", 4480 vfio_tap3, guest.network.l2_guest_mac3 4481 ) 4482 .as_str(), 4483 ]) 4484 .capture_output() 4485 .spawn() 4486 .unwrap(); 4487 4488 thread::sleep(std::time::Duration::new(30, 0)); 4489 4490 let r = std::panic::catch_unwind(|| { 4491 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 4492 thread::sleep(std::time::Duration::new(120, 0)); 4493 4494 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 4495 // added to its kernel command line. 4496 // Let's ssh into it and verify that it's there. If it is it means 4497 // we're in the right guest (The L2 one) because the QEMU L1 guest 4498 // does not have this command line tag. 4499 assert!(check_matched_lines_count( 4500 guest.ssh_command_l2_1("cat /proc/cmdline").unwrap().trim(), 4501 vec!["VFIOTAG"], 4502 1 4503 )); 4504 4505 // Let's also verify from the second virtio-net device passed to 4506 // the L2 VM. 4507 assert!(check_matched_lines_count( 4508 guest.ssh_command_l2_2("cat /proc/cmdline").unwrap().trim(), 4509 vec!["VFIOTAG"], 4510 1 4511 )); 4512 4513 // Check the amount of PCI devices appearing in L2 VM. 4514 assert!(check_lines_count( 4515 guest 4516 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4517 .unwrap() 4518 .trim(), 4519 8 4520 )); 4521 4522 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 4523 assert!(check_matched_lines_count( 4524 guest.ssh_command_l2_1("lsblk").unwrap().trim(), 4525 vec!["vdc", "16M"], 4526 1 4527 )); 4528 4529 // Hotplug an extra virtio-net device through L2 VM. 4530 guest 4531 .ssh_command_l1( 4532 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 4533 ) 4534 .unwrap(); 4535 guest 4536 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 4537 .unwrap(); 4538 let vfio_hotplug_output = guest 4539 .ssh_command_l1( 4540 "sudo /mnt/ch-remote \ 4541 --api-socket=/tmp/ch_api.sock \ 4542 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 4543 ) 4544 .unwrap(); 4545 assert!(check_matched_lines_count( 4546 vfio_hotplug_output.trim(), 4547 vec!["{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}"], 4548 1 4549 )); 4550 4551 thread::sleep(std::time::Duration::new(10, 0)); 4552 4553 // Let's also verify from the third virtio-net device passed to 4554 // the L2 VM. This third device has been hotplugged through the L2 4555 // VM, so this is our way to validate hotplug works for VFIO PCI. 4556 assert!(check_matched_lines_count( 4557 guest.ssh_command_l2_3("cat /proc/cmdline").unwrap().trim(), 4558 vec!["VFIOTAG"], 4559 1 4560 )); 4561 4562 // Check the amount of PCI devices appearing in L2 VM. 4563 // There should be one more device than before, raising the count 4564 // up to 9 PCI devices. 4565 assert!(check_lines_count( 4566 guest 4567 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4568 .unwrap() 4569 .trim(), 4570 9 4571 )); 4572 4573 // Let's now verify that we can correctly remove the virtio-net 4574 // device through the "remove-device" command responsible for 4575 // unplugging VFIO devices. 4576 guest 4577 .ssh_command_l1( 4578 "sudo /mnt/ch-remote \ 4579 --api-socket=/tmp/ch_api.sock \ 4580 remove-device vfio123", 4581 ) 4582 .unwrap(); 4583 thread::sleep(std::time::Duration::new(10, 0)); 4584 4585 // Check the amount of PCI devices appearing in L2 VM is back down 4586 // to 8 devices. 4587 assert!(check_lines_count( 4588 guest 4589 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4590 .unwrap() 4591 .trim(), 4592 8 4593 )); 4594 4595 // Perform memory hotplug in L2 and validate the memory is showing 4596 // up as expected. In order to check, we will use the virtio-net 4597 // device already passed through L2 as a VFIO device, this will 4598 // verify that VFIO devices are functional with memory hotplug. 4599 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 4600 guest 4601 .ssh_command_l2_1( 4602 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 4603 ) 4604 .unwrap(); 4605 guest 4606 .ssh_command_l1( 4607 "sudo /mnt/ch-remote \ 4608 --api-socket=/tmp/ch_api.sock \ 4609 resize --memory=1073741824", 4610 ) 4611 .unwrap(); 4612 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 4613 }); 4614 4615 let _ = child.kill(); 4616 let output = child.wait_with_output().unwrap(); 4617 4618 cleanup_vfio_network_interfaces(); 4619 4620 handle_child_output(r, &output); 4621 } 4622 4623 #[test] 4624 fn test_direct_kernel_boot_noacpi() { 4625 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4626 let guest = Guest::new(Box::new(focal)); 4627 4628 let kernel_path = direct_kernel_boot_path(); 4629 4630 let mut child = GuestCommand::new(&guest) 4631 .args(["--cpus", "boot=1"]) 4632 .args(["--memory", "size=512M"]) 4633 .args(["--kernel", kernel_path.to_str().unwrap()]) 4634 .args([ 4635 "--cmdline", 4636 format!("{DIRECT_KERNEL_BOOT_CMDLINE} acpi=off").as_str(), 4637 ]) 4638 .default_disks() 4639 .default_net() 4640 .capture_output() 4641 .spawn() 4642 .unwrap(); 4643 4644 let r = std::panic::catch_unwind(|| { 4645 guest.wait_vm_boot(None).unwrap(); 4646 4647 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4648 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4649 }); 4650 4651 let _ = child.kill(); 4652 let output = child.wait_with_output().unwrap(); 4653 4654 handle_child_output(r, &output); 4655 } 4656 4657 #[test] 4658 fn test_virtio_vsock() { 4659 _test_virtio_vsock(false) 4660 } 4661 4662 #[test] 4663 fn test_virtio_vsock_hotplug() { 4664 _test_virtio_vsock(true); 4665 } 4666 4667 #[test] 4668 fn test_api_http_shutdown() { 4669 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4670 let guest = Guest::new(Box::new(focal)); 4671 4672 _test_api_shutdown(TargetApi::new_http_api(&guest.tmp_dir), guest) 4673 } 4674 4675 #[test] 4676 fn test_api_http_delete() { 4677 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4678 let guest = Guest::new(Box::new(focal)); 4679 4680 _test_api_delete(TargetApi::new_http_api(&guest.tmp_dir), guest); 4681 } 4682 4683 #[test] 4684 fn test_api_http_pause_resume() { 4685 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4686 let guest = Guest::new(Box::new(focal)); 4687 4688 _test_api_pause_resume(TargetApi::new_http_api(&guest.tmp_dir), guest) 4689 } 4690 4691 #[test] 4692 fn test_api_http_create_boot() { 4693 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4694 let guest = Guest::new(Box::new(focal)); 4695 4696 _test_api_create_boot(TargetApi::new_http_api(&guest.tmp_dir), guest) 4697 } 4698 4699 #[test] 4700 fn test_virtio_iommu() { 4701 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4702 } 4703 4704 #[test] 4705 // We cannot force the software running in the guest to reprogram the BAR 4706 // with some different addresses, but we have a reliable way of testing it 4707 // with a standard Linux kernel. 4708 // By removing a device from the PCI tree, and then rescanning the tree, 4709 // Linux consistently chooses to reorganize the PCI device BARs to other 4710 // locations in the guest address space. 4711 // This test creates a dedicated PCI network device to be checked as being 4712 // properly probed first, then removing it, and adding it again by doing a 4713 // rescan. 4714 fn test_pci_bar_reprogramming() { 4715 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4716 let guest = Guest::new(Box::new(focal)); 4717 4718 #[cfg(target_arch = "x86_64")] 4719 let kernel_path = direct_kernel_boot_path(); 4720 #[cfg(target_arch = "aarch64")] 4721 let kernel_path = edk2_path(); 4722 4723 let mut child = GuestCommand::new(&guest) 4724 .args(["--cpus", "boot=1"]) 4725 .args(["--memory", "size=512M"]) 4726 .args(["--kernel", kernel_path.to_str().unwrap()]) 4727 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4728 .default_disks() 4729 .args([ 4730 "--net", 4731 guest.default_net_string().as_str(), 4732 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4733 ]) 4734 .capture_output() 4735 .spawn() 4736 .unwrap(); 4737 4738 let r = std::panic::catch_unwind(|| { 4739 guest.wait_vm_boot(None).unwrap(); 4740 4741 // 2 network interfaces + default localhost ==> 3 interfaces 4742 assert_eq!( 4743 guest 4744 .ssh_command("ip -o link | wc -l") 4745 .unwrap() 4746 .trim() 4747 .parse::<u32>() 4748 .unwrap_or_default(), 4749 3 4750 ); 4751 4752 let init_bar_addr = guest 4753 .ssh_command( 4754 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4755 ) 4756 .unwrap(); 4757 4758 // Remove the PCI device 4759 guest 4760 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4761 .unwrap(); 4762 4763 // Only 1 network interface left + default localhost ==> 2 interfaces 4764 assert_eq!( 4765 guest 4766 .ssh_command("ip -o link | wc -l") 4767 .unwrap() 4768 .trim() 4769 .parse::<u32>() 4770 .unwrap_or_default(), 4771 2 4772 ); 4773 4774 // Remove the PCI device 4775 guest 4776 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4777 .unwrap(); 4778 4779 // Back to 2 network interface + default localhost ==> 3 interfaces 4780 assert_eq!( 4781 guest 4782 .ssh_command("ip -o link | wc -l") 4783 .unwrap() 4784 .trim() 4785 .parse::<u32>() 4786 .unwrap_or_default(), 4787 3 4788 ); 4789 4790 let new_bar_addr = guest 4791 .ssh_command( 4792 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4793 ) 4794 .unwrap(); 4795 4796 // Let's compare the BAR addresses for our virtio-net device. 4797 // They should be different as we expect the BAR reprogramming 4798 // to have happened. 4799 assert_ne!(init_bar_addr, new_bar_addr); 4800 }); 4801 4802 let _ = child.kill(); 4803 let output = child.wait_with_output().unwrap(); 4804 4805 handle_child_output(r, &output); 4806 } 4807 4808 #[test] 4809 fn test_memory_mergeable_off() { 4810 test_memory_mergeable(false) 4811 } 4812 4813 #[test] 4814 #[cfg(target_arch = "x86_64")] 4815 fn test_cpu_hotplug() { 4816 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4817 let guest = Guest::new(Box::new(focal)); 4818 let api_socket = temp_api_path(&guest.tmp_dir); 4819 4820 let kernel_path = direct_kernel_boot_path(); 4821 4822 let mut child = GuestCommand::new(&guest) 4823 .args(["--cpus", "boot=2,max=4"]) 4824 .args(["--memory", "size=512M"]) 4825 .args(["--kernel", kernel_path.to_str().unwrap()]) 4826 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4827 .default_disks() 4828 .default_net() 4829 .args(["--api-socket", &api_socket]) 4830 .capture_output() 4831 .spawn() 4832 .unwrap(); 4833 4834 let r = std::panic::catch_unwind(|| { 4835 guest.wait_vm_boot(None).unwrap(); 4836 4837 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4838 4839 // Resize the VM 4840 let desired_vcpus = 4; 4841 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4842 4843 guest 4844 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4845 .unwrap(); 4846 guest 4847 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4848 .unwrap(); 4849 thread::sleep(std::time::Duration::new(10, 0)); 4850 assert_eq!( 4851 guest.get_cpu_count().unwrap_or_default(), 4852 u32::from(desired_vcpus) 4853 ); 4854 4855 guest.reboot_linux(0, None); 4856 4857 assert_eq!( 4858 guest.get_cpu_count().unwrap_or_default(), 4859 u32::from(desired_vcpus) 4860 ); 4861 4862 // Resize the VM 4863 let desired_vcpus = 2; 4864 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4865 4866 thread::sleep(std::time::Duration::new(10, 0)); 4867 assert_eq!( 4868 guest.get_cpu_count().unwrap_or_default(), 4869 u32::from(desired_vcpus) 4870 ); 4871 4872 // Resize the VM back up to 4 4873 let desired_vcpus = 4; 4874 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4875 4876 guest 4877 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4878 .unwrap(); 4879 guest 4880 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4881 .unwrap(); 4882 thread::sleep(std::time::Duration::new(10, 0)); 4883 assert_eq!( 4884 guest.get_cpu_count().unwrap_or_default(), 4885 u32::from(desired_vcpus) 4886 ); 4887 }); 4888 4889 let _ = child.kill(); 4890 let output = child.wait_with_output().unwrap(); 4891 4892 handle_child_output(r, &output); 4893 } 4894 4895 #[test] 4896 fn test_memory_hotplug() { 4897 #[cfg(target_arch = "aarch64")] 4898 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4899 #[cfg(target_arch = "x86_64")] 4900 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4901 let focal = UbuntuDiskConfig::new(focal_image); 4902 let guest = Guest::new(Box::new(focal)); 4903 let api_socket = temp_api_path(&guest.tmp_dir); 4904 4905 #[cfg(target_arch = "aarch64")] 4906 let kernel_path = edk2_path(); 4907 #[cfg(target_arch = "x86_64")] 4908 let kernel_path = direct_kernel_boot_path(); 4909 4910 let mut child = GuestCommand::new(&guest) 4911 .args(["--cpus", "boot=2,max=4"]) 4912 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4913 .args(["--kernel", kernel_path.to_str().unwrap()]) 4914 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4915 .default_disks() 4916 .default_net() 4917 .args(["--balloon", "size=0"]) 4918 .args(["--api-socket", &api_socket]) 4919 .capture_output() 4920 .spawn() 4921 .unwrap(); 4922 4923 let r = std::panic::catch_unwind(|| { 4924 guest.wait_vm_boot(None).unwrap(); 4925 4926 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4927 4928 guest.enable_memory_hotplug(); 4929 4930 // Add RAM to the VM 4931 let desired_ram = 1024 << 20; 4932 resize_command(&api_socket, None, Some(desired_ram), None, None); 4933 4934 thread::sleep(std::time::Duration::new(10, 0)); 4935 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4936 4937 // Use balloon to remove RAM from the VM 4938 let desired_balloon = 512 << 20; 4939 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4940 4941 thread::sleep(std::time::Duration::new(10, 0)); 4942 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4943 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4944 4945 guest.reboot_linux(0, None); 4946 4947 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4948 4949 // Use balloon add RAM to the VM 4950 let desired_balloon = 0; 4951 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4952 4953 thread::sleep(std::time::Duration::new(10, 0)); 4954 4955 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4956 4957 guest.enable_memory_hotplug(); 4958 4959 // Add RAM to the VM 4960 let desired_ram = 2048 << 20; 4961 resize_command(&api_socket, None, Some(desired_ram), None, None); 4962 4963 thread::sleep(std::time::Duration::new(10, 0)); 4964 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4965 4966 // Remove RAM to the VM (only applies after reboot) 4967 let desired_ram = 1024 << 20; 4968 resize_command(&api_socket, None, Some(desired_ram), None, None); 4969 4970 guest.reboot_linux(1, None); 4971 4972 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4973 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4974 }); 4975 4976 let _ = child.kill(); 4977 let output = child.wait_with_output().unwrap(); 4978 4979 handle_child_output(r, &output); 4980 } 4981 4982 #[test] 4983 #[cfg(not(feature = "mshv"))] 4984 fn test_virtio_mem() { 4985 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4986 let guest = Guest::new(Box::new(focal)); 4987 let api_socket = temp_api_path(&guest.tmp_dir); 4988 4989 let kernel_path = direct_kernel_boot_path(); 4990 4991 let mut child = GuestCommand::new(&guest) 4992 .args(["--cpus", "boot=2,max=4"]) 4993 .args([ 4994 "--memory", 4995 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4996 ]) 4997 .args(["--kernel", kernel_path.to_str().unwrap()]) 4998 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4999 .default_disks() 5000 .default_net() 5001 .args(["--api-socket", &api_socket]) 5002 .capture_output() 5003 .spawn() 5004 .unwrap(); 5005 5006 let r = std::panic::catch_unwind(|| { 5007 guest.wait_vm_boot(None).unwrap(); 5008 5009 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5010 5011 guest.enable_memory_hotplug(); 5012 5013 // Add RAM to the VM 5014 let desired_ram = 1024 << 20; 5015 resize_command(&api_socket, None, Some(desired_ram), None, None); 5016 5017 thread::sleep(std::time::Duration::new(10, 0)); 5018 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5019 5020 // Add RAM to the VM 5021 let desired_ram = 2048 << 20; 5022 resize_command(&api_socket, None, Some(desired_ram), None, None); 5023 5024 thread::sleep(std::time::Duration::new(10, 0)); 5025 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 5026 5027 // Remove RAM from the VM 5028 let desired_ram = 1024 << 20; 5029 resize_command(&api_socket, None, Some(desired_ram), None, None); 5030 5031 thread::sleep(std::time::Duration::new(10, 0)); 5032 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5033 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5034 5035 guest.reboot_linux(0, None); 5036 5037 // Check the amount of memory after reboot is 1GiB 5038 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5039 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5040 5041 // Check we can still resize to 512MiB 5042 let desired_ram = 512 << 20; 5043 resize_command(&api_socket, None, Some(desired_ram), None, None); 5044 thread::sleep(std::time::Duration::new(10, 0)); 5045 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5046 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 5047 }); 5048 5049 let _ = child.kill(); 5050 let output = child.wait_with_output().unwrap(); 5051 5052 handle_child_output(r, &output); 5053 } 5054 5055 #[test] 5056 #[cfg(target_arch = "x86_64")] 5057 #[cfg(not(feature = "mshv"))] 5058 // Test both vCPU and memory resizing together 5059 fn test_resize() { 5060 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5061 let guest = Guest::new(Box::new(focal)); 5062 let api_socket = temp_api_path(&guest.tmp_dir); 5063 5064 let kernel_path = direct_kernel_boot_path(); 5065 5066 let mut child = GuestCommand::new(&guest) 5067 .args(["--cpus", "boot=2,max=4"]) 5068 .args(["--memory", "size=512M,hotplug_size=8192M"]) 5069 .args(["--kernel", kernel_path.to_str().unwrap()]) 5070 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5071 .default_disks() 5072 .default_net() 5073 .args(["--api-socket", &api_socket]) 5074 .capture_output() 5075 .spawn() 5076 .unwrap(); 5077 5078 let r = std::panic::catch_unwind(|| { 5079 guest.wait_vm_boot(None).unwrap(); 5080 5081 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 5082 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5083 5084 guest.enable_memory_hotplug(); 5085 5086 // Resize the VM 5087 let desired_vcpus = 4; 5088 let desired_ram = 1024 << 20; 5089 resize_command( 5090 &api_socket, 5091 Some(desired_vcpus), 5092 Some(desired_ram), 5093 None, 5094 None, 5095 ); 5096 5097 guest 5098 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 5099 .unwrap(); 5100 guest 5101 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 5102 .unwrap(); 5103 thread::sleep(std::time::Duration::new(10, 0)); 5104 assert_eq!( 5105 guest.get_cpu_count().unwrap_or_default(), 5106 u32::from(desired_vcpus) 5107 ); 5108 5109 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5110 }); 5111 5112 let _ = child.kill(); 5113 let output = child.wait_with_output().unwrap(); 5114 5115 handle_child_output(r, &output); 5116 } 5117 5118 #[test] 5119 fn test_memory_overhead() { 5120 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5121 let guest = Guest::new(Box::new(focal)); 5122 5123 let kernel_path = direct_kernel_boot_path(); 5124 5125 let guest_memory_size_kb = 512 * 1024; 5126 5127 let mut child = GuestCommand::new(&guest) 5128 .args(["--cpus", "boot=1"]) 5129 .args(["--memory", format!("size={guest_memory_size_kb}K").as_str()]) 5130 .args(["--kernel", kernel_path.to_str().unwrap()]) 5131 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5132 .default_net() 5133 .default_disks() 5134 .capture_output() 5135 .spawn() 5136 .unwrap(); 5137 5138 guest.wait_vm_boot(None).unwrap(); 5139 5140 let r = std::panic::catch_unwind(|| { 5141 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 5142 eprintln!("Guest memory overhead: {overhead} vs {MAXIMUM_VMM_OVERHEAD_KB}"); 5143 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 5144 }); 5145 5146 let _ = child.kill(); 5147 let output = child.wait_with_output().unwrap(); 5148 5149 handle_child_output(r, &output); 5150 } 5151 5152 #[test] 5153 fn test_disk_hotplug() { 5154 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5155 let guest = Guest::new(Box::new(focal)); 5156 5157 #[cfg(target_arch = "x86_64")] 5158 let kernel_path = direct_kernel_boot_path(); 5159 #[cfg(target_arch = "aarch64")] 5160 let kernel_path = edk2_path(); 5161 5162 let api_socket = temp_api_path(&guest.tmp_dir); 5163 5164 let mut child = GuestCommand::new(&guest) 5165 .args(["--api-socket", &api_socket]) 5166 .args(["--cpus", "boot=1"]) 5167 .args(["--memory", "size=512M"]) 5168 .args(["--kernel", kernel_path.to_str().unwrap()]) 5169 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5170 .default_disks() 5171 .default_net() 5172 .capture_output() 5173 .spawn() 5174 .unwrap(); 5175 5176 let r = std::panic::catch_unwind(|| { 5177 guest.wait_vm_boot(None).unwrap(); 5178 5179 // Check /dev/vdc is not there 5180 assert_eq!( 5181 guest 5182 .ssh_command("lsblk | grep -c vdc.*16M || true") 5183 .unwrap() 5184 .trim() 5185 .parse::<u32>() 5186 .unwrap_or(1), 5187 0 5188 ); 5189 5190 // Now let's add the extra disk. 5191 let mut blk_file_path = dirs::home_dir().unwrap(); 5192 blk_file_path.push("workloads"); 5193 blk_file_path.push("blk.img"); 5194 let (cmd_success, cmd_output) = remote_command_w_output( 5195 &api_socket, 5196 "add-disk", 5197 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5198 ); 5199 assert!(cmd_success); 5200 assert!(String::from_utf8_lossy(&cmd_output) 5201 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5202 5203 thread::sleep(std::time::Duration::new(10, 0)); 5204 5205 // Check that /dev/vdc exists and the block size is 16M. 5206 assert_eq!( 5207 guest 5208 .ssh_command("lsblk | grep vdc | grep -c 16M") 5209 .unwrap() 5210 .trim() 5211 .parse::<u32>() 5212 .unwrap_or_default(), 5213 1 5214 ); 5215 // And check the block device can be read. 5216 guest 5217 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5218 .unwrap(); 5219 5220 // Let's remove it the extra disk. 5221 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5222 thread::sleep(std::time::Duration::new(5, 0)); 5223 // And check /dev/vdc is not there 5224 assert_eq!( 5225 guest 5226 .ssh_command("lsblk | grep -c vdc.*16M || true") 5227 .unwrap() 5228 .trim() 5229 .parse::<u32>() 5230 .unwrap_or(1), 5231 0 5232 ); 5233 5234 // And add it back to validate unplug did work correctly. 5235 let (cmd_success, cmd_output) = remote_command_w_output( 5236 &api_socket, 5237 "add-disk", 5238 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5239 ); 5240 assert!(cmd_success); 5241 assert!(String::from_utf8_lossy(&cmd_output) 5242 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5243 5244 thread::sleep(std::time::Duration::new(10, 0)); 5245 5246 // Check that /dev/vdc exists and the block size is 16M. 5247 assert_eq!( 5248 guest 5249 .ssh_command("lsblk | grep vdc | grep -c 16M") 5250 .unwrap() 5251 .trim() 5252 .parse::<u32>() 5253 .unwrap_or_default(), 5254 1 5255 ); 5256 // And check the block device can be read. 5257 guest 5258 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5259 .unwrap(); 5260 5261 // Reboot the VM. 5262 guest.reboot_linux(0, None); 5263 5264 // Check still there after reboot 5265 assert_eq!( 5266 guest 5267 .ssh_command("lsblk | grep vdc | grep -c 16M") 5268 .unwrap() 5269 .trim() 5270 .parse::<u32>() 5271 .unwrap_or_default(), 5272 1 5273 ); 5274 5275 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5276 5277 thread::sleep(std::time::Duration::new(20, 0)); 5278 5279 // Check device has gone away 5280 assert_eq!( 5281 guest 5282 .ssh_command("lsblk | grep -c vdc.*16M || true") 5283 .unwrap() 5284 .trim() 5285 .parse::<u32>() 5286 .unwrap_or(1), 5287 0 5288 ); 5289 5290 guest.reboot_linux(1, None); 5291 5292 // Check device still absent 5293 assert_eq!( 5294 guest 5295 .ssh_command("lsblk | grep -c vdc.*16M || true") 5296 .unwrap() 5297 .trim() 5298 .parse::<u32>() 5299 .unwrap_or(1), 5300 0 5301 ); 5302 }); 5303 5304 let _ = child.kill(); 5305 let output = child.wait_with_output().unwrap(); 5306 5307 handle_child_output(r, &output); 5308 } 5309 5310 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 5311 const LOOP_CONFIGURE: u64 = 0x4c0a; 5312 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 5313 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 5314 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 5315 5316 #[repr(C)] 5317 struct LoopInfo64 { 5318 lo_device: u64, 5319 lo_inode: u64, 5320 lo_rdevice: u64, 5321 lo_offset: u64, 5322 lo_sizelimit: u64, 5323 lo_number: u32, 5324 lo_encrypt_type: u32, 5325 lo_encrypt_key_size: u32, 5326 lo_flags: u32, 5327 lo_file_name: [u8; 64], 5328 lo_crypt_name: [u8; 64], 5329 lo_encrypt_key: [u8; 32], 5330 lo_init: [u64; 2], 5331 } 5332 5333 impl Default for LoopInfo64 { 5334 fn default() -> Self { 5335 LoopInfo64 { 5336 lo_device: 0, 5337 lo_inode: 0, 5338 lo_rdevice: 0, 5339 lo_offset: 0, 5340 lo_sizelimit: 0, 5341 lo_number: 0, 5342 lo_encrypt_type: 0, 5343 lo_encrypt_key_size: 0, 5344 lo_flags: 0, 5345 lo_file_name: [0; 64], 5346 lo_crypt_name: [0; 64], 5347 lo_encrypt_key: [0; 32], 5348 lo_init: [0; 2], 5349 } 5350 } 5351 } 5352 5353 #[derive(Default)] 5354 #[repr(C)] 5355 struct LoopConfig { 5356 fd: u32, 5357 block_size: u32, 5358 info: LoopInfo64, 5359 _reserved: [u64; 8], 5360 } 5361 5362 // Open loop-control device 5363 let loop_ctl_file = OpenOptions::new() 5364 .read(true) 5365 .write(true) 5366 .open(LOOP_CTL_PATH) 5367 .unwrap(); 5368 5369 // Request a free loop device 5370 let loop_device_number = 5371 unsafe { libc::ioctl(loop_ctl_file.as_raw_fd(), LOOP_CTL_GET_FREE as _) }; 5372 5373 if loop_device_number < 0 { 5374 panic!("Couldn't find a free loop device"); 5375 } 5376 5377 // Create loop device path 5378 let loop_device_path = format!("{LOOP_DEVICE_PREFIX}{loop_device_number}"); 5379 5380 // Open loop device 5381 let loop_device_file = OpenOptions::new() 5382 .read(true) 5383 .write(true) 5384 .open(&loop_device_path) 5385 .unwrap(); 5386 5387 // Open backing file 5388 let backing_file = OpenOptions::new() 5389 .read(true) 5390 .write(true) 5391 .open(backing_file_path) 5392 .unwrap(); 5393 5394 let loop_config = LoopConfig { 5395 fd: backing_file.as_raw_fd() as u32, 5396 block_size, 5397 ..Default::default() 5398 }; 5399 5400 for i in 0..num_retries { 5401 let ret = unsafe { 5402 libc::ioctl( 5403 loop_device_file.as_raw_fd(), 5404 LOOP_CONFIGURE as _, 5405 &loop_config, 5406 ) 5407 }; 5408 if ret != 0 { 5409 if i < num_retries - 1 { 5410 println!( 5411 "Iteration {}: Failed to configure the loop device {}: {}", 5412 i, 5413 loop_device_path, 5414 std::io::Error::last_os_error() 5415 ); 5416 } else { 5417 panic!( 5418 "Failed {} times trying to configure the loop device {}: {}", 5419 num_retries, 5420 loop_device_path, 5421 std::io::Error::last_os_error() 5422 ); 5423 } 5424 } else { 5425 break; 5426 } 5427 5428 // Wait for a bit before retrying 5429 thread::sleep(std::time::Duration::new(5, 0)); 5430 } 5431 5432 loop_device_path 5433 } 5434 5435 #[test] 5436 fn test_virtio_block_topology() { 5437 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5438 let guest = Guest::new(Box::new(focal)); 5439 5440 let kernel_path = direct_kernel_boot_path(); 5441 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5442 5443 let output = exec_host_command_output( 5444 format!( 5445 "qemu-img create -f raw {} 16M", 5446 test_disk_path.to_str().unwrap() 5447 ) 5448 .as_str(), 5449 ); 5450 if !output.status.success() { 5451 let stdout = String::from_utf8_lossy(&output.stdout); 5452 let stderr = String::from_utf8_lossy(&output.stderr); 5453 panic!("qemu-img command failed\nstdout\n{stdout}\nstderr\n{stderr}"); 5454 } 5455 5456 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5457 5458 let mut child = GuestCommand::new(&guest) 5459 .args(["--cpus", "boot=1"]) 5460 .args(["--memory", "size=512M"]) 5461 .args(["--kernel", kernel_path.to_str().unwrap()]) 5462 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5463 .args([ 5464 "--disk", 5465 format!( 5466 "path={}", 5467 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5468 ) 5469 .as_str(), 5470 format!( 5471 "path={}", 5472 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5473 ) 5474 .as_str(), 5475 format!("path={}", &loop_dev).as_str(), 5476 ]) 5477 .default_net() 5478 .capture_output() 5479 .spawn() 5480 .unwrap(); 5481 5482 let r = std::panic::catch_unwind(|| { 5483 guest.wait_vm_boot(None).unwrap(); 5484 5485 // MIN-IO column 5486 assert_eq!( 5487 guest 5488 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5489 .unwrap() 5490 .trim() 5491 .parse::<u32>() 5492 .unwrap_or_default(), 5493 4096 5494 ); 5495 // PHY-SEC column 5496 assert_eq!( 5497 guest 5498 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5499 .unwrap() 5500 .trim() 5501 .parse::<u32>() 5502 .unwrap_or_default(), 5503 4096 5504 ); 5505 // LOG-SEC column 5506 assert_eq!( 5507 guest 5508 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5509 .unwrap() 5510 .trim() 5511 .parse::<u32>() 5512 .unwrap_or_default(), 5513 4096 5514 ); 5515 }); 5516 5517 let _ = child.kill(); 5518 let output = child.wait_with_output().unwrap(); 5519 5520 handle_child_output(r, &output); 5521 5522 Command::new("losetup") 5523 .args(["-d", &loop_dev]) 5524 .output() 5525 .expect("loop device not found"); 5526 } 5527 5528 #[test] 5529 fn test_virtio_balloon_deflate_on_oom() { 5530 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5531 let guest = Guest::new(Box::new(focal)); 5532 5533 let kernel_path = direct_kernel_boot_path(); 5534 5535 let api_socket = temp_api_path(&guest.tmp_dir); 5536 5537 //Let's start a 4G guest with balloon occupied 2G memory 5538 let mut child = GuestCommand::new(&guest) 5539 .args(["--api-socket", &api_socket]) 5540 .args(["--cpus", "boot=1"]) 5541 .args(["--memory", "size=4G"]) 5542 .args(["--kernel", kernel_path.to_str().unwrap()]) 5543 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5544 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5545 .default_disks() 5546 .default_net() 5547 .capture_output() 5548 .spawn() 5549 .unwrap(); 5550 5551 let r = std::panic::catch_unwind(|| { 5552 guest.wait_vm_boot(None).unwrap(); 5553 5554 // Wait for balloon memory's initialization and check its size. 5555 // The virtio-balloon driver might take a few seconds to report the 5556 // balloon effective size back to the VMM. 5557 thread::sleep(std::time::Duration::new(20, 0)); 5558 5559 let orig_balloon = balloon_size(&api_socket); 5560 println!("The original balloon memory size is {orig_balloon} bytes"); 5561 assert!(orig_balloon == 2147483648); 5562 5563 // Two steps to verify if the 'deflate_on_oom' parameter works. 5564 // 1st: run a command to trigger an OOM in the guest. 5565 guest 5566 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5567 .unwrap(); 5568 5569 // Give some time for the OOM to happen in the guest and be reported 5570 // back to the host. 5571 thread::sleep(std::time::Duration::new(20, 0)); 5572 5573 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5574 let deflated_balloon = balloon_size(&api_socket); 5575 println!("After deflating, balloon memory size is {deflated_balloon} bytes"); 5576 // Verify the balloon size deflated 5577 assert!(deflated_balloon < 2147483648); 5578 }); 5579 5580 let _ = child.kill(); 5581 let output = child.wait_with_output().unwrap(); 5582 5583 handle_child_output(r, &output); 5584 } 5585 5586 #[test] 5587 #[cfg(not(feature = "mshv"))] 5588 fn test_virtio_balloon_free_page_reporting() { 5589 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5590 let guest = Guest::new(Box::new(focal)); 5591 5592 //Let's start a 4G guest with balloon occupied 2G memory 5593 let mut child = GuestCommand::new(&guest) 5594 .args(["--cpus", "boot=1"]) 5595 .args(["--memory", "size=4G"]) 5596 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5597 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5598 .args(["--balloon", "size=0,free_page_reporting=on"]) 5599 .default_disks() 5600 .default_net() 5601 .capture_output() 5602 .spawn() 5603 .unwrap(); 5604 5605 let pid = child.id(); 5606 let r = std::panic::catch_unwind(|| { 5607 guest.wait_vm_boot(None).unwrap(); 5608 5609 // Check the initial RSS is less than 1GiB 5610 let rss = process_rss_kib(pid); 5611 println!("RSS {rss} < 1048576"); 5612 assert!(rss < 1048576); 5613 5614 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5615 // seconds 5616 let guest_ip = guest.network.guest_ip.clone(); 5617 thread::spawn(move || { 5618 ssh_command_ip( 5619 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5620 &guest_ip, 5621 DEFAULT_SSH_RETRIES, 5622 DEFAULT_SSH_TIMEOUT, 5623 ) 5624 .unwrap(); 5625 }); 5626 5627 // Wait for 50 seconds to make sure the stress command is consuming 5628 // the expected amount of memory. 5629 thread::sleep(std::time::Duration::new(50, 0)); 5630 let rss = process_rss_kib(pid); 5631 println!("RSS {rss} >= 2097152"); 5632 assert!(rss >= 2097152); 5633 5634 // Wait for an extra minute to make sure the stress command has 5635 // completed and that the guest reported the free pages to the VMM 5636 // through the virtio-balloon device. We expect the RSS to be under 5637 // 2GiB. 5638 thread::sleep(std::time::Duration::new(60, 0)); 5639 let rss = process_rss_kib(pid); 5640 println!("RSS {rss} < 2097152"); 5641 assert!(rss < 2097152); 5642 }); 5643 5644 let _ = child.kill(); 5645 let output = child.wait_with_output().unwrap(); 5646 5647 handle_child_output(r, &output); 5648 } 5649 5650 #[test] 5651 fn test_pmem_hotplug() { 5652 _test_pmem_hotplug(None) 5653 } 5654 5655 #[test] 5656 fn test_pmem_multi_segment_hotplug() { 5657 _test_pmem_hotplug(Some(15)) 5658 } 5659 5660 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5661 #[cfg(target_arch = "aarch64")] 5662 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5663 #[cfg(target_arch = "x86_64")] 5664 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5665 let focal = UbuntuDiskConfig::new(focal_image); 5666 let guest = Guest::new(Box::new(focal)); 5667 5668 #[cfg(target_arch = "x86_64")] 5669 let kernel_path = direct_kernel_boot_path(); 5670 #[cfg(target_arch = "aarch64")] 5671 let kernel_path = edk2_path(); 5672 5673 let api_socket = temp_api_path(&guest.tmp_dir); 5674 5675 let mut cmd = GuestCommand::new(&guest); 5676 5677 cmd.args(["--api-socket", &api_socket]) 5678 .args(["--cpus", "boot=1"]) 5679 .args(["--memory", "size=512M"]) 5680 .args(["--kernel", kernel_path.to_str().unwrap()]) 5681 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5682 .default_disks() 5683 .default_net() 5684 .capture_output(); 5685 5686 if pci_segment.is_some() { 5687 cmd.args([ 5688 "--platform", 5689 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5690 ]); 5691 } 5692 5693 let mut child = cmd.spawn().unwrap(); 5694 5695 let r = std::panic::catch_unwind(|| { 5696 guest.wait_vm_boot(None).unwrap(); 5697 5698 // Check /dev/pmem0 is not there 5699 assert_eq!( 5700 guest 5701 .ssh_command("lsblk | grep -c pmem0 || true") 5702 .unwrap() 5703 .trim() 5704 .parse::<u32>() 5705 .unwrap_or(1), 5706 0 5707 ); 5708 5709 let pmem_temp_file = TempFile::new().unwrap(); 5710 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5711 let (cmd_success, cmd_output) = remote_command_w_output( 5712 &api_socket, 5713 "add-pmem", 5714 Some(&format!( 5715 "file={},id=test0{}", 5716 pmem_temp_file.as_path().to_str().unwrap(), 5717 if let Some(pci_segment) = pci_segment { 5718 format!(",pci_segment={pci_segment}") 5719 } else { 5720 "".to_owned() 5721 } 5722 )), 5723 ); 5724 assert!(cmd_success); 5725 if let Some(pci_segment) = pci_segment { 5726 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5727 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5728 ))); 5729 } else { 5730 assert!(String::from_utf8_lossy(&cmd_output) 5731 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5732 } 5733 5734 // Check that /dev/pmem0 exists and the block size is 128M 5735 assert_eq!( 5736 guest 5737 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5738 .unwrap() 5739 .trim() 5740 .parse::<u32>() 5741 .unwrap_or_default(), 5742 1 5743 ); 5744 5745 guest.reboot_linux(0, None); 5746 5747 // Check still there after reboot 5748 assert_eq!( 5749 guest 5750 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5751 .unwrap() 5752 .trim() 5753 .parse::<u32>() 5754 .unwrap_or_default(), 5755 1 5756 ); 5757 5758 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5759 5760 thread::sleep(std::time::Duration::new(20, 0)); 5761 5762 // Check device has gone away 5763 assert_eq!( 5764 guest 5765 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5766 .unwrap() 5767 .trim() 5768 .parse::<u32>() 5769 .unwrap_or(1), 5770 0 5771 ); 5772 5773 guest.reboot_linux(1, None); 5774 5775 // Check still absent after reboot 5776 assert_eq!( 5777 guest 5778 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5779 .unwrap() 5780 .trim() 5781 .parse::<u32>() 5782 .unwrap_or(1), 5783 0 5784 ); 5785 }); 5786 5787 let _ = child.kill(); 5788 let output = child.wait_with_output().unwrap(); 5789 5790 handle_child_output(r, &output); 5791 } 5792 5793 #[test] 5794 fn test_net_hotplug() { 5795 _test_net_hotplug(None) 5796 } 5797 5798 #[test] 5799 fn test_net_multi_segment_hotplug() { 5800 _test_net_hotplug(Some(15)) 5801 } 5802 5803 fn _test_net_hotplug(pci_segment: Option<u16>) { 5804 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5805 let guest = Guest::new(Box::new(focal)); 5806 5807 #[cfg(target_arch = "x86_64")] 5808 let kernel_path = direct_kernel_boot_path(); 5809 #[cfg(target_arch = "aarch64")] 5810 let kernel_path = edk2_path(); 5811 5812 let api_socket = temp_api_path(&guest.tmp_dir); 5813 5814 // Boot without network 5815 let mut cmd = GuestCommand::new(&guest); 5816 5817 cmd.args(["--api-socket", &api_socket]) 5818 .args(["--cpus", "boot=1"]) 5819 .args(["--memory", "size=512M"]) 5820 .args(["--kernel", kernel_path.to_str().unwrap()]) 5821 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5822 .default_disks() 5823 .capture_output(); 5824 5825 if pci_segment.is_some() { 5826 cmd.args([ 5827 "--platform", 5828 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5829 ]); 5830 } 5831 5832 let mut child = cmd.spawn().unwrap(); 5833 5834 thread::sleep(std::time::Duration::new(20, 0)); 5835 5836 let r = std::panic::catch_unwind(|| { 5837 // Add network 5838 let (cmd_success, cmd_output) = remote_command_w_output( 5839 &api_socket, 5840 "add-net", 5841 Some( 5842 format!( 5843 "{}{},id=test0", 5844 guest.default_net_string(), 5845 if let Some(pci_segment) = pci_segment { 5846 format!(",pci_segment={pci_segment}") 5847 } else { 5848 "".to_owned() 5849 } 5850 ) 5851 .as_str(), 5852 ), 5853 ); 5854 assert!(cmd_success); 5855 5856 if let Some(pci_segment) = pci_segment { 5857 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5858 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5859 ))); 5860 } else { 5861 assert!(String::from_utf8_lossy(&cmd_output) 5862 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5863 } 5864 5865 thread::sleep(std::time::Duration::new(5, 0)); 5866 5867 // 1 network interfaces + default localhost ==> 2 interfaces 5868 assert_eq!( 5869 guest 5870 .ssh_command("ip -o link | wc -l") 5871 .unwrap() 5872 .trim() 5873 .parse::<u32>() 5874 .unwrap_or_default(), 5875 2 5876 ); 5877 5878 // Remove network 5879 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5880 thread::sleep(std::time::Duration::new(5, 0)); 5881 5882 let (cmd_success, cmd_output) = remote_command_w_output( 5883 &api_socket, 5884 "add-net", 5885 Some( 5886 format!( 5887 "{}{},id=test1", 5888 guest.default_net_string(), 5889 if let Some(pci_segment) = pci_segment { 5890 format!(",pci_segment={pci_segment}") 5891 } else { 5892 "".to_owned() 5893 } 5894 ) 5895 .as_str(), 5896 ), 5897 ); 5898 assert!(cmd_success); 5899 5900 if let Some(pci_segment) = pci_segment { 5901 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5902 "{{\"id\":\"test1\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5903 ))); 5904 } else { 5905 assert!(String::from_utf8_lossy(&cmd_output) 5906 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5907 } 5908 5909 thread::sleep(std::time::Duration::new(5, 0)); 5910 5911 // 1 network interfaces + default localhost ==> 2 interfaces 5912 assert_eq!( 5913 guest 5914 .ssh_command("ip -o link | wc -l") 5915 .unwrap() 5916 .trim() 5917 .parse::<u32>() 5918 .unwrap_or_default(), 5919 2 5920 ); 5921 5922 guest.reboot_linux(0, None); 5923 5924 // Check still there after reboot 5925 // 1 network interfaces + default localhost ==> 2 interfaces 5926 assert_eq!( 5927 guest 5928 .ssh_command("ip -o link | wc -l") 5929 .unwrap() 5930 .trim() 5931 .parse::<u32>() 5932 .unwrap_or_default(), 5933 2 5934 ); 5935 }); 5936 5937 let _ = child.kill(); 5938 let output = child.wait_with_output().unwrap(); 5939 5940 handle_child_output(r, &output); 5941 } 5942 5943 #[test] 5944 fn test_initramfs() { 5945 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5946 let guest = Guest::new(Box::new(focal)); 5947 let mut workload_path = dirs::home_dir().unwrap(); 5948 workload_path.push("workloads"); 5949 5950 #[cfg(target_arch = "x86_64")] 5951 let mut kernels = vec![direct_kernel_boot_path()]; 5952 #[cfg(target_arch = "aarch64")] 5953 let kernels = [direct_kernel_boot_path()]; 5954 5955 #[cfg(target_arch = "x86_64")] 5956 { 5957 let mut pvh_kernel_path = workload_path.clone(); 5958 pvh_kernel_path.push("vmlinux"); 5959 kernels.push(pvh_kernel_path); 5960 } 5961 5962 let mut initramfs_path = workload_path; 5963 initramfs_path.push("alpine_initramfs.img"); 5964 5965 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 5966 let cmdline = format!("console=hvc0 quiet TEST_STRING={test_string}"); 5967 5968 kernels.iter().for_each(|k_path| { 5969 let mut child = GuestCommand::new(&guest) 5970 .args(["--kernel", k_path.to_str().unwrap()]) 5971 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 5972 .args(["--cmdline", &cmdline]) 5973 .capture_output() 5974 .spawn() 5975 .unwrap(); 5976 5977 thread::sleep(std::time::Duration::new(20, 0)); 5978 5979 let _ = child.kill(); 5980 let output = child.wait_with_output().unwrap(); 5981 5982 let r = std::panic::catch_unwind(|| { 5983 let s = String::from_utf8_lossy(&output.stdout); 5984 5985 assert_ne!(s.lines().position(|line| line == test_string), None); 5986 }); 5987 5988 handle_child_output(r, &output); 5989 }); 5990 } 5991 5992 // One thing to note about this test. The virtio-net device is heavily used 5993 // through each ssh command. There's no need to perform a dedicated test to 5994 // verify the migration went well for virtio-net. 5995 #[test] 5996 #[cfg(not(feature = "mshv"))] 5997 fn test_snapshot_restore_hotplug_virtiomem() { 5998 _test_snapshot_restore(true); 5999 } 6000 6001 #[test] 6002 fn test_snapshot_restore_basic() { 6003 _test_snapshot_restore(false); 6004 } 6005 6006 fn _test_snapshot_restore(use_hotplug: bool) { 6007 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6008 let guest = Guest::new(Box::new(focal)); 6009 let kernel_path = direct_kernel_boot_path(); 6010 6011 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 6012 6013 let net_id = "net123"; 6014 let net_params = format!( 6015 "id={},tap=,mac={},ip={},mask=255.255.255.0", 6016 net_id, guest.network.guest_mac, guest.network.host_ip 6017 ); 6018 let mut mem_params = "size=2G"; 6019 6020 if use_hotplug { 6021 mem_params = "size=2G,hotplug_method=virtio-mem,hotplug_size=32G" 6022 } 6023 6024 let cloudinit_params = format!( 6025 "path={},iommu=on", 6026 guest.disk_config.disk(DiskType::CloudInit).unwrap() 6027 ); 6028 6029 let socket = temp_vsock_path(&guest.tmp_dir); 6030 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6031 6032 let mut child = GuestCommand::new(&guest) 6033 .args(["--api-socket", &api_socket_source]) 6034 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6035 .args(["--cpus", "boot=4"]) 6036 .args(["--memory", mem_params]) 6037 .args(["--balloon", "size=0"]) 6038 .args(["--kernel", kernel_path.to_str().unwrap()]) 6039 .args([ 6040 "--disk", 6041 format!( 6042 "path={}", 6043 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 6044 ) 6045 .as_str(), 6046 cloudinit_params.as_str(), 6047 ]) 6048 .args(["--net", net_params.as_str()]) 6049 .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) 6050 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6051 .capture_output() 6052 .spawn() 6053 .unwrap(); 6054 6055 let console_text = String::from("On a branch floating down river a cricket, singing."); 6056 // Create the snapshot directory 6057 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 6058 6059 let r = std::panic::catch_unwind(|| { 6060 guest.wait_vm_boot(None).unwrap(); 6061 6062 // Check the number of vCPUs 6063 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 6064 // Check the guest RAM 6065 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 6066 if use_hotplug { 6067 // Increase guest RAM with virtio-mem 6068 resize_command( 6069 &api_socket_source, 6070 None, 6071 Some(6 << 30), 6072 None, 6073 Some(&event_path), 6074 ); 6075 thread::sleep(std::time::Duration::new(5, 0)); 6076 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6077 // Use balloon to remove RAM from the VM 6078 resize_command( 6079 &api_socket_source, 6080 None, 6081 None, 6082 Some(1 << 30), 6083 Some(&event_path), 6084 ); 6085 thread::sleep(std::time::Duration::new(5, 0)); 6086 let total_memory = guest.get_total_memory().unwrap_or_default(); 6087 assert!(total_memory > 4_800_000); 6088 assert!(total_memory < 5_760_000); 6089 } 6090 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 6091 guest.check_devices_common(Some(&socket), Some(&console_text), None); 6092 6093 // x86_64: We check that removing and adding back the virtio-net device 6094 // does not break the snapshot/restore support for virtio-pci. 6095 // This is an important thing to test as the hotplug will 6096 // trigger a PCI BAR reprogramming, which is a good way of 6097 // checking if the stored resources are correctly restored. 6098 // Unplug the virtio-net device 6099 // AArch64: Device hotplug is currently not supported, skipping here. 6100 #[cfg(target_arch = "x86_64")] 6101 { 6102 assert!(remote_command( 6103 &api_socket_source, 6104 "remove-device", 6105 Some(net_id), 6106 )); 6107 thread::sleep(std::time::Duration::new(10, 0)); 6108 let latest_events = [&MetaEvent { 6109 event: "device-removed".to_string(), 6110 device_id: Some(net_id.to_string()), 6111 }]; 6112 // See: #5938 6113 thread::sleep(std::time::Duration::new(1, 0)); 6114 assert!(check_latest_events_exact(&latest_events, &event_path)); 6115 6116 // Plug the virtio-net device again 6117 assert!(remote_command( 6118 &api_socket_source, 6119 "add-net", 6120 Some(net_params.as_str()), 6121 )); 6122 thread::sleep(std::time::Duration::new(10, 0)); 6123 } 6124 6125 // Pause the VM 6126 assert!(remote_command(&api_socket_source, "pause", None)); 6127 let latest_events = [ 6128 &MetaEvent { 6129 event: "pausing".to_string(), 6130 device_id: None, 6131 }, 6132 &MetaEvent { 6133 event: "paused".to_string(), 6134 device_id: None, 6135 }, 6136 ]; 6137 // See: #5938 6138 thread::sleep(std::time::Duration::new(1, 0)); 6139 assert!(check_latest_events_exact(&latest_events, &event_path)); 6140 6141 // Take a snapshot from the VM 6142 assert!(remote_command( 6143 &api_socket_source, 6144 "snapshot", 6145 Some(format!("file://{snapshot_dir}").as_str()), 6146 )); 6147 6148 // Wait to make sure the snapshot is completed 6149 thread::sleep(std::time::Duration::new(10, 0)); 6150 6151 let latest_events = [ 6152 &MetaEvent { 6153 event: "snapshotting".to_string(), 6154 device_id: None, 6155 }, 6156 &MetaEvent { 6157 event: "snapshotted".to_string(), 6158 device_id: None, 6159 }, 6160 ]; 6161 // See: #5938 6162 thread::sleep(std::time::Duration::new(1, 0)); 6163 assert!(check_latest_events_exact(&latest_events, &event_path)); 6164 }); 6165 6166 // Shutdown the source VM and check console output 6167 let _ = child.kill(); 6168 let output = child.wait_with_output().unwrap(); 6169 handle_child_output(r, &output); 6170 6171 let r = std::panic::catch_unwind(|| { 6172 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 6173 }); 6174 6175 handle_child_output(r, &output); 6176 6177 // Remove the vsock socket file. 6178 Command::new("rm") 6179 .arg("-f") 6180 .arg(socket.as_str()) 6181 .output() 6182 .unwrap(); 6183 6184 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 6185 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 6186 6187 // Restore the VM from the snapshot 6188 let mut child = GuestCommand::new(&guest) 6189 .args(["--api-socket", &api_socket_restored]) 6190 .args([ 6191 "--event-monitor", 6192 format!("path={event_path_restored}").as_str(), 6193 ]) 6194 .args([ 6195 "--restore", 6196 format!("source_url=file://{snapshot_dir}").as_str(), 6197 ]) 6198 .capture_output() 6199 .spawn() 6200 .unwrap(); 6201 6202 // Wait for the VM to be restored 6203 thread::sleep(std::time::Duration::new(20, 0)); 6204 let expected_events = [ 6205 &MetaEvent { 6206 event: "starting".to_string(), 6207 device_id: None, 6208 }, 6209 &MetaEvent { 6210 event: "activated".to_string(), 6211 device_id: Some("__console".to_string()), 6212 }, 6213 &MetaEvent { 6214 event: "activated".to_string(), 6215 device_id: Some("__rng".to_string()), 6216 }, 6217 &MetaEvent { 6218 event: "restoring".to_string(), 6219 device_id: None, 6220 }, 6221 ]; 6222 assert!(check_sequential_events( 6223 &expected_events, 6224 &event_path_restored 6225 )); 6226 let latest_events = [&MetaEvent { 6227 event: "restored".to_string(), 6228 device_id: None, 6229 }]; 6230 assert!(check_latest_events_exact( 6231 &latest_events, 6232 &event_path_restored 6233 )); 6234 6235 // Remove the snapshot dir 6236 let _ = remove_dir_all(snapshot_dir.as_str()); 6237 6238 let r = std::panic::catch_unwind(|| { 6239 // Resume the VM 6240 assert!(remote_command(&api_socket_restored, "resume", None)); 6241 // There is no way that we can ensure the 'write()' to the 6242 // event file is completed when the 'resume' request is 6243 // returned successfully, because the 'write()' was done 6244 // asynchronously from a different thread of Cloud 6245 // Hypervisor (e.g. the event-monitor thread). 6246 thread::sleep(std::time::Duration::new(1, 0)); 6247 let latest_events = [ 6248 &MetaEvent { 6249 event: "resuming".to_string(), 6250 device_id: None, 6251 }, 6252 &MetaEvent { 6253 event: "resumed".to_string(), 6254 device_id: None, 6255 }, 6256 ]; 6257 assert!(check_latest_events_exact( 6258 &latest_events, 6259 &event_path_restored 6260 )); 6261 6262 // Perform same checks to validate VM has been properly restored 6263 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 6264 let total_memory = guest.get_total_memory().unwrap_or_default(); 6265 if !use_hotplug { 6266 assert!(total_memory > 1_920_000); 6267 } else { 6268 assert!(total_memory > 4_800_000); 6269 assert!(total_memory < 5_760_000); 6270 // Deflate balloon to restore entire RAM to the VM 6271 resize_command(&api_socket_restored, None, None, Some(0), None); 6272 thread::sleep(std::time::Duration::new(5, 0)); 6273 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6274 // Decrease guest RAM with virtio-mem 6275 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 6276 thread::sleep(std::time::Duration::new(5, 0)); 6277 let total_memory = guest.get_total_memory().unwrap_or_default(); 6278 assert!(total_memory > 4_800_000); 6279 assert!(total_memory < 5_760_000); 6280 } 6281 6282 guest.check_devices_common(Some(&socket), Some(&console_text), None); 6283 }); 6284 // Shutdown the target VM and check console output 6285 let _ = child.kill(); 6286 let output = child.wait_with_output().unwrap(); 6287 handle_child_output(r, &output); 6288 6289 let r = std::panic::catch_unwind(|| { 6290 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 6291 }); 6292 6293 handle_child_output(r, &output); 6294 } 6295 6296 #[test] 6297 fn test_counters() { 6298 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6299 let guest = Guest::new(Box::new(focal)); 6300 let api_socket = temp_api_path(&guest.tmp_dir); 6301 6302 let mut cmd = GuestCommand::new(&guest); 6303 cmd.args(["--cpus", "boot=1"]) 6304 .args(["--memory", "size=512M"]) 6305 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 6306 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6307 .default_disks() 6308 .args(["--net", guest.default_net_string().as_str()]) 6309 .args(["--api-socket", &api_socket]) 6310 .capture_output(); 6311 6312 let mut child = cmd.spawn().unwrap(); 6313 6314 let r = std::panic::catch_unwind(|| { 6315 guest.wait_vm_boot(None).unwrap(); 6316 6317 let orig_counters = get_counters(&api_socket); 6318 guest 6319 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 6320 .unwrap(); 6321 6322 let new_counters = get_counters(&api_socket); 6323 6324 // Check that all the counters have increased 6325 assert!(new_counters > orig_counters); 6326 }); 6327 6328 let _ = child.kill(); 6329 let output = child.wait_with_output().unwrap(); 6330 6331 handle_child_output(r, &output); 6332 } 6333 6334 #[test] 6335 #[cfg(feature = "guest_debug")] 6336 fn test_coredump() { 6337 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6338 let guest = Guest::new(Box::new(focal)); 6339 let api_socket = temp_api_path(&guest.tmp_dir); 6340 6341 let mut cmd = GuestCommand::new(&guest); 6342 cmd.args(["--cpus", "boot=4"]) 6343 .args(["--memory", "size=4G"]) 6344 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6345 .default_disks() 6346 .args(["--net", guest.default_net_string().as_str()]) 6347 .args(["--api-socket", &api_socket]) 6348 .capture_output(); 6349 6350 let mut child = cmd.spawn().unwrap(); 6351 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6352 6353 let r = std::panic::catch_unwind(|| { 6354 guest.wait_vm_boot(None).unwrap(); 6355 6356 assert!(remote_command(&api_socket, "pause", None)); 6357 6358 assert!(remote_command( 6359 &api_socket, 6360 "coredump", 6361 Some(format!("file://{vmcore_file}").as_str()), 6362 )); 6363 6364 // the num of CORE notes should equals to vcpu 6365 let readelf_core_num_cmd = 6366 format!("readelf --all {vmcore_file} |grep CORE |grep -v Type |wc -l"); 6367 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 6368 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 6369 6370 // the num of QEMU notes should equals to vcpu 6371 let readelf_vmm_num_cmd = format!("readelf --all {vmcore_file} |grep QEMU |wc -l"); 6372 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 6373 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 6374 }); 6375 6376 let _ = child.kill(); 6377 let output = child.wait_with_output().unwrap(); 6378 6379 handle_child_output(r, &output); 6380 } 6381 6382 #[test] 6383 #[cfg(feature = "guest_debug")] 6384 fn test_coredump_no_pause() { 6385 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6386 let guest = Guest::new(Box::new(focal)); 6387 let api_socket = temp_api_path(&guest.tmp_dir); 6388 6389 let mut cmd = GuestCommand::new(&guest); 6390 cmd.args(["--cpus", "boot=4"]) 6391 .args(["--memory", "size=4G"]) 6392 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6393 .default_disks() 6394 .args(["--net", guest.default_net_string().as_str()]) 6395 .args(["--api-socket", &api_socket]) 6396 .capture_output(); 6397 6398 let mut child = cmd.spawn().unwrap(); 6399 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6400 6401 let r = std::panic::catch_unwind(|| { 6402 guest.wait_vm_boot(None).unwrap(); 6403 6404 assert!(remote_command( 6405 &api_socket, 6406 "coredump", 6407 Some(format!("file://{vmcore_file}").as_str()), 6408 )); 6409 6410 assert_eq!(vm_state(&api_socket), "Running"); 6411 }); 6412 6413 let _ = child.kill(); 6414 let output = child.wait_with_output().unwrap(); 6415 6416 handle_child_output(r, &output); 6417 } 6418 6419 #[test] 6420 fn test_watchdog() { 6421 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6422 let guest = Guest::new(Box::new(focal)); 6423 let api_socket = temp_api_path(&guest.tmp_dir); 6424 6425 let kernel_path = direct_kernel_boot_path(); 6426 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6427 6428 let mut cmd = GuestCommand::new(&guest); 6429 cmd.args(["--cpus", "boot=1"]) 6430 .args(["--memory", "size=512M"]) 6431 .args(["--kernel", kernel_path.to_str().unwrap()]) 6432 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6433 .default_disks() 6434 .args(["--net", guest.default_net_string().as_str()]) 6435 .args(["--watchdog"]) 6436 .args(["--api-socket", &api_socket]) 6437 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6438 .capture_output(); 6439 6440 let mut child = cmd.spawn().unwrap(); 6441 6442 let r = std::panic::catch_unwind(|| { 6443 guest.wait_vm_boot(None).unwrap(); 6444 6445 let mut expected_reboot_count = 1; 6446 6447 // Enable the watchdog with a 15s timeout 6448 enable_guest_watchdog(&guest, 15); 6449 6450 // Reboot and check that systemd has activated the watchdog 6451 guest.ssh_command("sudo reboot").unwrap(); 6452 guest.wait_vm_boot(None).unwrap(); 6453 expected_reboot_count += 1; 6454 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6455 assert_eq!( 6456 guest 6457 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6458 .unwrap() 6459 .trim() 6460 .parse::<u32>() 6461 .unwrap_or_default(), 6462 2 6463 ); 6464 6465 // Allow some normal time to elapse to check we don't get spurious reboots 6466 thread::sleep(std::time::Duration::new(40, 0)); 6467 // Check no reboot 6468 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6469 6470 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6471 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6472 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6473 guest.wait_vm_boot(Some(50)).unwrap(); 6474 // Check a reboot is triggered by the watchdog 6475 expected_reboot_count += 1; 6476 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6477 6478 #[cfg(target_arch = "x86_64")] 6479 { 6480 // Now pause the VM and remain offline for 30s 6481 assert!(remote_command(&api_socket, "pause", None)); 6482 let latest_events = [ 6483 &MetaEvent { 6484 event: "pausing".to_string(), 6485 device_id: None, 6486 }, 6487 &MetaEvent { 6488 event: "paused".to_string(), 6489 device_id: None, 6490 }, 6491 ]; 6492 assert!(check_latest_events_exact(&latest_events, &event_path)); 6493 assert!(remote_command(&api_socket, "resume", None)); 6494 6495 // Check no reboot 6496 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6497 } 6498 }); 6499 6500 let _ = child.kill(); 6501 let output = child.wait_with_output().unwrap(); 6502 6503 handle_child_output(r, &output); 6504 } 6505 6506 #[test] 6507 fn test_pvpanic() { 6508 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 6509 let guest = Guest::new(Box::new(jammy)); 6510 let api_socket = temp_api_path(&guest.tmp_dir); 6511 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6512 6513 let kernel_path = direct_kernel_boot_path(); 6514 6515 let mut cmd = GuestCommand::new(&guest); 6516 cmd.args(["--cpus", "boot=1"]) 6517 .args(["--memory", "size=512M"]) 6518 .args(["--kernel", kernel_path.to_str().unwrap()]) 6519 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6520 .default_disks() 6521 .args(["--net", guest.default_net_string().as_str()]) 6522 .args(["--pvpanic"]) 6523 .args(["--api-socket", &api_socket]) 6524 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6525 .capture_output(); 6526 6527 let mut child = cmd.spawn().unwrap(); 6528 6529 let r = std::panic::catch_unwind(|| { 6530 guest.wait_vm_boot(None).unwrap(); 6531 6532 // Trigger guest a panic 6533 make_guest_panic(&guest); 6534 6535 // Wait a while for guest 6536 thread::sleep(std::time::Duration::new(10, 0)); 6537 6538 let expected_sequential_events = [&MetaEvent { 6539 event: "panic".to_string(), 6540 device_id: None, 6541 }]; 6542 assert!(check_latest_events_exact( 6543 &expected_sequential_events, 6544 &event_path 6545 )); 6546 }); 6547 6548 let _ = child.kill(); 6549 let output = child.wait_with_output().unwrap(); 6550 6551 handle_child_output(r, &output); 6552 } 6553 6554 #[test] 6555 fn test_tap_from_fd() { 6556 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6557 let guest = Guest::new(Box::new(focal)); 6558 let kernel_path = direct_kernel_boot_path(); 6559 6560 // Create a TAP interface with multi-queue enabled 6561 let num_queue_pairs: usize = 2; 6562 6563 use std::str::FromStr; 6564 let taps = net_util::open_tap( 6565 Some("chtap0"), 6566 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6567 None, 6568 &mut None, 6569 None, 6570 num_queue_pairs, 6571 Some(libc::O_RDWR | libc::O_NONBLOCK), 6572 ) 6573 .unwrap(); 6574 6575 let mut child = GuestCommand::new(&guest) 6576 .args(["--cpus", &format!("boot={num_queue_pairs}")]) 6577 .args(["--memory", "size=512M"]) 6578 .args(["--kernel", kernel_path.to_str().unwrap()]) 6579 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6580 .default_disks() 6581 .args([ 6582 "--net", 6583 &format!( 6584 "fd=[{},{}],mac={},num_queues={}", 6585 taps[0].as_raw_fd(), 6586 taps[1].as_raw_fd(), 6587 guest.network.guest_mac, 6588 num_queue_pairs * 2 6589 ), 6590 ]) 6591 .capture_output() 6592 .spawn() 6593 .unwrap(); 6594 6595 let r = std::panic::catch_unwind(|| { 6596 guest.wait_vm_boot(None).unwrap(); 6597 6598 assert_eq!( 6599 guest 6600 .ssh_command("ip -o link | wc -l") 6601 .unwrap() 6602 .trim() 6603 .parse::<u32>() 6604 .unwrap_or_default(), 6605 2 6606 ); 6607 6608 guest.reboot_linux(0, None); 6609 6610 assert_eq!( 6611 guest 6612 .ssh_command("ip -o link | wc -l") 6613 .unwrap() 6614 .trim() 6615 .parse::<u32>() 6616 .unwrap_or_default(), 6617 2 6618 ); 6619 }); 6620 6621 let _ = child.kill(); 6622 let output = child.wait_with_output().unwrap(); 6623 6624 handle_child_output(r, &output); 6625 } 6626 6627 // By design, a guest VM won't be able to connect to the host 6628 // machine when using a macvtap network interface (while it can 6629 // communicate externally). As a workaround, this integration 6630 // test creates two macvtap interfaces in 'bridge' mode on the 6631 // same physical net interface, one for the guest and one for 6632 // the host. With additional setup on the IP address and the 6633 // routing table, it enables the communications between the 6634 // guest VM and the host machine. 6635 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6636 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6637 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6638 let guest = Guest::new(Box::new(focal)); 6639 let api_socket = temp_api_path(&guest.tmp_dir); 6640 6641 #[cfg(target_arch = "x86_64")] 6642 let kernel_path = direct_kernel_boot_path(); 6643 #[cfg(target_arch = "aarch64")] 6644 let kernel_path = edk2_path(); 6645 6646 let phy_net = "eth0"; 6647 6648 // Create a macvtap interface for the guest VM to use 6649 assert!(exec_host_command_status(&format!( 6650 "sudo ip link add link {phy_net} name {guest_macvtap_name} type macvtap mod bridge" 6651 )) 6652 .success()); 6653 assert!(exec_host_command_status(&format!( 6654 "sudo ip link set {} address {} up", 6655 guest_macvtap_name, guest.network.guest_mac 6656 )) 6657 .success()); 6658 assert!( 6659 exec_host_command_status(&format!("sudo ip link show {guest_macvtap_name}")).success() 6660 ); 6661 6662 let tap_index = 6663 fs::read_to_string(format!("/sys/class/net/{guest_macvtap_name}/ifindex")).unwrap(); 6664 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6665 6666 assert!(exec_host_command_status(&format!("sudo chown $UID.$UID {tap_device}")).success()); 6667 6668 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6669 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6670 assert!(tap_fd1 > 0); 6671 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6672 assert!(tap_fd2 > 0); 6673 6674 // Create a macvtap on the same physical net interface for 6675 // the host machine to use 6676 assert!(exec_host_command_status(&format!( 6677 "sudo ip link add link {phy_net} name {host_macvtap_name} type macvtap mod bridge" 6678 )) 6679 .success()); 6680 // Use default mask "255.255.255.0" 6681 assert!(exec_host_command_status(&format!( 6682 "sudo ip address add {}/24 dev {}", 6683 guest.network.host_ip, host_macvtap_name 6684 )) 6685 .success()); 6686 assert!( 6687 exec_host_command_status(&format!("sudo ip link set dev {host_macvtap_name} up")) 6688 .success() 6689 ); 6690 6691 let mut guest_command = GuestCommand::new(&guest); 6692 guest_command 6693 .args(["--cpus", "boot=2"]) 6694 .args(["--memory", "size=512M"]) 6695 .args(["--kernel", kernel_path.to_str().unwrap()]) 6696 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6697 .default_disks() 6698 .args(["--api-socket", &api_socket]); 6699 6700 let net_params = format!( 6701 "fd=[{},{}],mac={},num_queues=4", 6702 tap_fd1, tap_fd2, guest.network.guest_mac 6703 ); 6704 6705 if !hotplug { 6706 guest_command.args(["--net", &net_params]); 6707 } 6708 6709 let mut child = guest_command.capture_output().spawn().unwrap(); 6710 6711 if hotplug { 6712 // Give some time to the VMM process to listen to the API 6713 // socket. This is the only requirement to avoid the following 6714 // call to ch-remote from failing. 6715 thread::sleep(std::time::Duration::new(10, 0)); 6716 // Hotplug the virtio-net device 6717 let (cmd_success, cmd_output) = 6718 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6719 assert!(cmd_success); 6720 #[cfg(target_arch = "x86_64")] 6721 assert!(String::from_utf8_lossy(&cmd_output) 6722 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6723 #[cfg(target_arch = "aarch64")] 6724 assert!(String::from_utf8_lossy(&cmd_output) 6725 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6726 } 6727 6728 // The functional connectivity provided by the virtio-net device 6729 // gets tested through wait_vm_boot() as it expects to receive a 6730 // HTTP request, and through the SSH command as well. 6731 let r = std::panic::catch_unwind(|| { 6732 guest.wait_vm_boot(None).unwrap(); 6733 6734 assert_eq!( 6735 guest 6736 .ssh_command("ip -o link | wc -l") 6737 .unwrap() 6738 .trim() 6739 .parse::<u32>() 6740 .unwrap_or_default(), 6741 2 6742 ); 6743 6744 guest.reboot_linux(0, None); 6745 6746 assert_eq!( 6747 guest 6748 .ssh_command("ip -o link | wc -l") 6749 .unwrap() 6750 .trim() 6751 .parse::<u32>() 6752 .unwrap_or_default(), 6753 2 6754 ); 6755 }); 6756 6757 let _ = child.kill(); 6758 6759 exec_host_command_status(&format!("sudo ip link del {guest_macvtap_name}")); 6760 exec_host_command_status(&format!("sudo ip link del {host_macvtap_name}")); 6761 6762 let output = child.wait_with_output().unwrap(); 6763 6764 handle_child_output(r, &output); 6765 } 6766 6767 #[test] 6768 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6769 fn test_macvtap() { 6770 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6771 } 6772 6773 #[test] 6774 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6775 fn test_macvtap_hotplug() { 6776 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6777 } 6778 6779 #[test] 6780 #[cfg(not(feature = "mshv"))] 6781 fn test_ovs_dpdk() { 6782 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6783 let guest1 = Guest::new(Box::new(focal1)); 6784 6785 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6786 let guest2 = Guest::new(Box::new(focal2)); 6787 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6788 6789 let (mut child1, mut child2) = 6790 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6791 6792 // Create the snapshot directory 6793 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6794 6795 let r = std::panic::catch_unwind(|| { 6796 // Remove one of the two ports from the OVS bridge 6797 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6798 6799 // Spawn a new netcat listener in the first VM 6800 let guest_ip = guest1.network.guest_ip.clone(); 6801 thread::spawn(move || { 6802 ssh_command_ip( 6803 "nc -l 12345", 6804 &guest_ip, 6805 DEFAULT_SSH_RETRIES, 6806 DEFAULT_SSH_TIMEOUT, 6807 ) 6808 .unwrap(); 6809 }); 6810 6811 // Wait for the server to be listening 6812 thread::sleep(std::time::Duration::new(5, 0)); 6813 6814 // Check the connection fails this time 6815 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 6816 6817 // Add the OVS port back 6818 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6819 6820 // And finally check the connection is functional again 6821 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6822 6823 // Pause the VM 6824 assert!(remote_command(&api_socket_source, "pause", None)); 6825 6826 // Take a snapshot from the VM 6827 assert!(remote_command( 6828 &api_socket_source, 6829 "snapshot", 6830 Some(format!("file://{snapshot_dir}").as_str()), 6831 )); 6832 6833 // Wait to make sure the snapshot is completed 6834 thread::sleep(std::time::Duration::new(10, 0)); 6835 }); 6836 6837 // Shutdown the source VM 6838 let _ = child2.kill(); 6839 let output = child2.wait_with_output().unwrap(); 6840 handle_child_output(r, &output); 6841 6842 // Remove the vhost-user socket file. 6843 Command::new("rm") 6844 .arg("-f") 6845 .arg("/tmp/dpdkvhostclient2") 6846 .output() 6847 .unwrap(); 6848 6849 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6850 // Restore the VM from the snapshot 6851 let mut child2 = GuestCommand::new(&guest2) 6852 .args(["--api-socket", &api_socket_restored]) 6853 .args([ 6854 "--restore", 6855 format!("source_url=file://{snapshot_dir}").as_str(), 6856 ]) 6857 .capture_output() 6858 .spawn() 6859 .unwrap(); 6860 6861 // Wait for the VM to be restored 6862 thread::sleep(std::time::Duration::new(10, 0)); 6863 6864 let r = std::panic::catch_unwind(|| { 6865 // Resume the VM 6866 assert!(remote_command(&api_socket_restored, "resume", None)); 6867 6868 // Spawn a new netcat listener in the first VM 6869 let guest_ip = guest1.network.guest_ip.clone(); 6870 thread::spawn(move || { 6871 ssh_command_ip( 6872 "nc -l 12345", 6873 &guest_ip, 6874 DEFAULT_SSH_RETRIES, 6875 DEFAULT_SSH_TIMEOUT, 6876 ) 6877 .unwrap(); 6878 }); 6879 6880 // Wait for the server to be listening 6881 thread::sleep(std::time::Duration::new(5, 0)); 6882 6883 // And check the connection is still functional after restore 6884 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6885 }); 6886 6887 let _ = child1.kill(); 6888 let _ = child2.kill(); 6889 6890 let output = child1.wait_with_output().unwrap(); 6891 child2.wait().unwrap(); 6892 6893 cleanup_ovs_dpdk(); 6894 6895 handle_child_output(r, &output); 6896 } 6897 6898 fn setup_spdk_nvme(nvme_dir: &std::path::Path) { 6899 cleanup_spdk_nvme(); 6900 6901 assert!(exec_host_command_status(&format!( 6902 "mkdir -p {}", 6903 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6904 )) 6905 .success()); 6906 assert!(exec_host_command_status(&format!( 6907 "truncate {} -s 128M", 6908 nvme_dir.join("test-disk.raw").to_str().unwrap() 6909 )) 6910 .success()); 6911 assert!(exec_host_command_status(&format!( 6912 "mkfs.ext4 {}", 6913 nvme_dir.join("test-disk.raw").to_str().unwrap() 6914 )) 6915 .success()); 6916 6917 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6918 Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6919 .args(["-i", "0", "-m", "0x1"]) 6920 .spawn() 6921 .unwrap(); 6922 thread::sleep(std::time::Duration::new(2, 0)); 6923 6924 assert!(exec_host_command_with_retries( 6925 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER", 6926 3, 6927 std::time::Duration::new(5, 0), 6928 )); 6929 assert!(exec_host_command_status(&format!( 6930 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6931 nvme_dir.join("test-disk.raw").to_str().unwrap() 6932 )) 6933 .success()); 6934 assert!(exec_host_command_status( 6935 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6936 ) 6937 .success()); 6938 assert!(exec_host_command_status( 6939 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6940 ) 6941 .success()); 6942 assert!(exec_host_command_status(&format!( 6943 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6944 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6945 )) 6946 .success()); 6947 } 6948 6949 fn cleanup_spdk_nvme() { 6950 exec_host_command_status("pkill -f nvmf_tgt"); 6951 } 6952 6953 #[test] 6954 fn test_vfio_user() { 6955 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6956 let jammy = UbuntuDiskConfig::new(jammy_image); 6957 let guest = Guest::new(Box::new(jammy)); 6958 6959 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6960 setup_spdk_nvme(spdk_nvme_dir.as_path()); 6961 6962 let api_socket = temp_api_path(&guest.tmp_dir); 6963 let mut child = GuestCommand::new(&guest) 6964 .args(["--api-socket", &api_socket]) 6965 .args(["--cpus", "boot=1"]) 6966 .args(["--memory", "size=512M,shared=on,hugepages=on"]) 6967 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6968 .args(["--serial", "tty", "--console", "off"]) 6969 .default_disks() 6970 .default_net() 6971 .capture_output() 6972 .spawn() 6973 .unwrap(); 6974 6975 let r = std::panic::catch_unwind(|| { 6976 guest.wait_vm_boot(None).unwrap(); 6977 6978 // Hotplug the SPDK-NVMe device to the VM 6979 let (cmd_success, cmd_output) = remote_command_w_output( 6980 &api_socket, 6981 "add-user-device", 6982 Some(&format!( 6983 "socket={},id=vfio_user0", 6984 spdk_nvme_dir 6985 .as_path() 6986 .join("nvme-vfio-user/cntrl") 6987 .to_str() 6988 .unwrap(), 6989 )), 6990 ); 6991 assert!(cmd_success); 6992 assert!(String::from_utf8_lossy(&cmd_output) 6993 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6994 6995 thread::sleep(std::time::Duration::new(10, 0)); 6996 6997 // Check both if /dev/nvme exists and if the block size is 128M. 6998 assert_eq!( 6999 guest 7000 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 7001 .unwrap() 7002 .trim() 7003 .parse::<u32>() 7004 .unwrap_or_default(), 7005 1 7006 ); 7007 7008 // Check changes persist after reboot 7009 assert_eq!( 7010 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 7011 "" 7012 ); 7013 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 7014 guest 7015 .ssh_command("echo test123 | sudo tee /mnt/test") 7016 .unwrap(); 7017 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 7018 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 7019 7020 guest.reboot_linux(0, None); 7021 assert_eq!( 7022 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 7023 "" 7024 ); 7025 assert_eq!( 7026 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 7027 "test123" 7028 ); 7029 }); 7030 7031 cleanup_spdk_nvme(); 7032 7033 let _ = child.kill(); 7034 let output = child.wait_with_output().unwrap(); 7035 7036 handle_child_output(r, &output); 7037 } 7038 7039 #[test] 7040 #[cfg(target_arch = "x86_64")] 7041 fn test_vdpa_block() { 7042 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 7043 assert!(exec_host_command_status("lsmod | grep vdpa_sim_blk").success()); 7044 7045 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7046 let guest = Guest::new(Box::new(focal)); 7047 let api_socket = temp_api_path(&guest.tmp_dir); 7048 7049 let kernel_path = direct_kernel_boot_path(); 7050 7051 let mut child = GuestCommand::new(&guest) 7052 .args(["--cpus", "boot=2"]) 7053 .args(["--memory", "size=512M,hugepages=on"]) 7054 .args(["--kernel", kernel_path.to_str().unwrap()]) 7055 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7056 .default_disks() 7057 .default_net() 7058 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 7059 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 7060 .args(["--api-socket", &api_socket]) 7061 .capture_output() 7062 .spawn() 7063 .unwrap(); 7064 7065 let r = std::panic::catch_unwind(|| { 7066 guest.wait_vm_boot(None).unwrap(); 7067 7068 // Check both if /dev/vdc exists and if the block size is 128M. 7069 assert_eq!( 7070 guest 7071 .ssh_command("lsblk | grep vdc | grep -c 128M") 7072 .unwrap() 7073 .trim() 7074 .parse::<u32>() 7075 .unwrap_or_default(), 7076 1 7077 ); 7078 7079 // Check the content of the block device after we wrote to it. 7080 // The vpda-sim-blk should let us read what we previously wrote. 7081 guest 7082 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 7083 .unwrap(); 7084 assert_eq!( 7085 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 7086 "foobar" 7087 ); 7088 7089 // Hotplug an extra vDPA block device behind the vIOMMU 7090 // Add a new vDPA device to the VM 7091 let (cmd_success, cmd_output) = remote_command_w_output( 7092 &api_socket, 7093 "add-vdpa", 7094 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 7095 ); 7096 assert!(cmd_success); 7097 assert!(String::from_utf8_lossy(&cmd_output) 7098 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 7099 7100 thread::sleep(std::time::Duration::new(10, 0)); 7101 7102 // Check IOMMU setup 7103 assert!(guest 7104 .does_device_vendor_pair_match("0x1057", "0x1af4") 7105 .unwrap_or_default()); 7106 assert_eq!( 7107 guest 7108 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 7109 .unwrap() 7110 .trim(), 7111 "0001:00:01.0" 7112 ); 7113 7114 // Check both if /dev/vdd exists and if the block size is 128M. 7115 assert_eq!( 7116 guest 7117 .ssh_command("lsblk | grep vdd | grep -c 128M") 7118 .unwrap() 7119 .trim() 7120 .parse::<u32>() 7121 .unwrap_or_default(), 7122 1 7123 ); 7124 7125 // Write some content to the block device we've just plugged. 7126 guest 7127 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 7128 .unwrap(); 7129 7130 // Check we can read the content back. 7131 assert_eq!( 7132 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 7133 "foobar" 7134 ); 7135 7136 // Unplug the device 7137 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 7138 assert!(cmd_success); 7139 thread::sleep(std::time::Duration::new(10, 0)); 7140 7141 // Check /dev/vdd doesn't exist anymore 7142 assert_eq!( 7143 guest 7144 .ssh_command("lsblk | grep -c vdd || true") 7145 .unwrap() 7146 .trim() 7147 .parse::<u32>() 7148 .unwrap_or(1), 7149 0 7150 ); 7151 }); 7152 7153 let _ = child.kill(); 7154 let output = child.wait_with_output().unwrap(); 7155 7156 handle_child_output(r, &output); 7157 } 7158 7159 #[test] 7160 #[cfg(target_arch = "x86_64")] 7161 #[ignore = "See #5756"] 7162 fn test_vdpa_net() { 7163 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 7164 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 7165 return; 7166 } 7167 7168 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7169 let guest = Guest::new(Box::new(focal)); 7170 7171 let kernel_path = direct_kernel_boot_path(); 7172 7173 let mut child = GuestCommand::new(&guest) 7174 .args(["--cpus", "boot=2"]) 7175 .args(["--memory", "size=512M,hugepages=on"]) 7176 .args(["--kernel", kernel_path.to_str().unwrap()]) 7177 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7178 .default_disks() 7179 .default_net() 7180 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 7181 .capture_output() 7182 .spawn() 7183 .unwrap(); 7184 7185 let r = std::panic::catch_unwind(|| { 7186 guest.wait_vm_boot(None).unwrap(); 7187 7188 // Check we can find network interface related to vDPA device 7189 assert_eq!( 7190 guest 7191 .ssh_command("ip -o link | grep -c ens6") 7192 .unwrap() 7193 .trim() 7194 .parse::<u32>() 7195 .unwrap_or(0), 7196 1 7197 ); 7198 7199 guest 7200 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 7201 .unwrap(); 7202 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 7203 7204 // Check there is no packet yet on both TX/RX of the network interface 7205 assert_eq!( 7206 guest 7207 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 7208 .unwrap() 7209 .trim() 7210 .parse::<u32>() 7211 .unwrap_or(0), 7212 2 7213 ); 7214 7215 // Send 6 packets with ping command 7216 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 7217 7218 // Check we can find 6 packets on both TX/RX of the network interface 7219 assert_eq!( 7220 guest 7221 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 7222 .unwrap() 7223 .trim() 7224 .parse::<u32>() 7225 .unwrap_or(0), 7226 2 7227 ); 7228 7229 // No need to check for hotplug as we already tested it through 7230 // test_vdpa_block() 7231 }); 7232 7233 let _ = child.kill(); 7234 let output = child.wait_with_output().unwrap(); 7235 7236 handle_child_output(r, &output); 7237 } 7238 7239 #[test] 7240 #[cfg(target_arch = "x86_64")] 7241 fn test_tpm() { 7242 let focal = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7243 let guest = Guest::new(Box::new(focal)); 7244 7245 let (mut swtpm_command, swtpm_socket_path) = prepare_swtpm_daemon(&guest.tmp_dir); 7246 7247 let mut guest_cmd = GuestCommand::new(&guest); 7248 guest_cmd 7249 .args(["--cpus", "boot=1"]) 7250 .args(["--memory", "size=512M"]) 7251 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7252 .args(["--tpm", &format!("socket={swtpm_socket_path}")]) 7253 .capture_output() 7254 .default_disks() 7255 .default_net(); 7256 7257 // Start swtpm daemon 7258 let mut swtpm_child = swtpm_command.spawn().unwrap(); 7259 thread::sleep(std::time::Duration::new(10, 0)); 7260 let mut child = guest_cmd.spawn().unwrap(); 7261 let r = std::panic::catch_unwind(|| { 7262 guest.wait_vm_boot(None).unwrap(); 7263 assert_eq!( 7264 guest.ssh_command("ls /dev/tpm0").unwrap().trim(), 7265 "/dev/tpm0" 7266 ); 7267 guest.ssh_command("sudo tpm2_selftest -f").unwrap(); 7268 guest 7269 .ssh_command("echo 'hello' > /tmp/checksum_test; ") 7270 .unwrap(); 7271 guest.ssh_command("cmp <(sudo tpm2_pcrevent /tmp/checksum_test | grep sha256 | awk '{print $2}') <(sha256sum /tmp/checksum_test| awk '{print $1}')").unwrap(); 7272 }); 7273 7274 let _ = swtpm_child.kill(); 7275 let _d_out = swtpm_child.wait_with_output().unwrap(); 7276 7277 let _ = child.kill(); 7278 let output = child.wait_with_output().unwrap(); 7279 7280 handle_child_output(r, &output); 7281 } 7282 7283 #[test] 7284 #[cfg(target_arch = "x86_64")] 7285 fn test_double_tty() { 7286 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7287 let guest = Guest::new(Box::new(focal)); 7288 let mut cmd = GuestCommand::new(&guest); 7289 let api_socket = temp_api_path(&guest.tmp_dir); 7290 let tty_str: &str = "console=hvc0 earlyprintk=ttyS0 "; 7291 // linux printk module enable console log. 7292 let con_dis_str: &str = "console [hvc0] enabled"; 7293 // linux printk module disable console log. 7294 let con_enb_str: &str = "bootconsole [earlyser0] disabled"; 7295 7296 let kernel_path = direct_kernel_boot_path(); 7297 7298 cmd.args(["--cpus", "boot=1"]) 7299 .args(["--memory", "size=512M"]) 7300 .args(["--kernel", kernel_path.to_str().unwrap()]) 7301 .args([ 7302 "--cmdline", 7303 DIRECT_KERNEL_BOOT_CMDLINE 7304 .replace("console=hvc0 ", tty_str) 7305 .as_str(), 7306 ]) 7307 .capture_output() 7308 .default_disks() 7309 .default_net() 7310 .args(["--serial", "tty"]) 7311 .args(["--console", "tty"]) 7312 .args(["--api-socket", &api_socket]); 7313 7314 let mut child = cmd.spawn().unwrap(); 7315 7316 let mut r = std::panic::catch_unwind(|| { 7317 guest.wait_vm_boot(None).unwrap(); 7318 }); 7319 7320 let _ = child.kill(); 7321 let output = child.wait_with_output().unwrap(); 7322 7323 if r.is_ok() { 7324 r = std::panic::catch_unwind(|| { 7325 let s = String::from_utf8_lossy(&output.stdout); 7326 assert!(s.contains(tty_str)); 7327 assert!(s.contains(con_dis_str)); 7328 assert!(s.contains(con_enb_str)); 7329 }); 7330 } 7331 7332 handle_child_output(r, &output); 7333 } 7334 7335 #[test] 7336 #[cfg(target_arch = "x86_64")] 7337 fn test_nmi() { 7338 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7339 let guest = Guest::new(Box::new(jammy)); 7340 let api_socket = temp_api_path(&guest.tmp_dir); 7341 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7342 7343 let kernel_path = direct_kernel_boot_path(); 7344 let cmd_line = format!("{} {}", DIRECT_KERNEL_BOOT_CMDLINE, "unknown_nmi_panic=1"); 7345 7346 let mut cmd = GuestCommand::new(&guest); 7347 cmd.args(["--cpus", "boot=4"]) 7348 .args(["--memory", "size=512M"]) 7349 .args(["--kernel", kernel_path.to_str().unwrap()]) 7350 .args(["--cmdline", cmd_line.as_str()]) 7351 .default_disks() 7352 .args(["--net", guest.default_net_string().as_str()]) 7353 .args(["--pvpanic"]) 7354 .args(["--api-socket", &api_socket]) 7355 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7356 .capture_output(); 7357 7358 let mut child = cmd.spawn().unwrap(); 7359 7360 let r = std::panic::catch_unwind(|| { 7361 guest.wait_vm_boot(None).unwrap(); 7362 7363 assert!(remote_command(&api_socket, "nmi", None)); 7364 7365 // Wait a while for guest 7366 thread::sleep(std::time::Duration::new(3, 0)); 7367 7368 let expected_sequential_events = [&MetaEvent { 7369 event: "panic".to_string(), 7370 device_id: None, 7371 }]; 7372 assert!(check_latest_events_exact( 7373 &expected_sequential_events, 7374 &event_path 7375 )); 7376 }); 7377 7378 let _ = child.kill(); 7379 let output = child.wait_with_output().unwrap(); 7380 7381 handle_child_output(r, &output); 7382 } 7383 } 7384 7385 mod dbus_api { 7386 use crate::*; 7387 7388 // Start cloud-hypervisor with no VM parameters, running both the HTTP 7389 // and DBus APIs. Alternate calls to the external APIs (HTTP and DBus) 7390 // to create a VM, boot it, and verify that it can be shut down and then 7391 // booted again. 7392 #[test] 7393 fn test_api_dbus_and_http_interleaved() { 7394 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7395 let guest = Guest::new(Box::new(focal)); 7396 let dbus_api = TargetApi::new_dbus_api(&guest.tmp_dir); 7397 let http_api = TargetApi::new_http_api(&guest.tmp_dir); 7398 7399 let mut child = GuestCommand::new(&guest) 7400 .args(dbus_api.guest_args()) 7401 .args(http_api.guest_args()) 7402 .capture_output() 7403 .spawn() 7404 .unwrap(); 7405 7406 thread::sleep(std::time::Duration::new(1, 0)); 7407 7408 // Verify API servers are running 7409 assert!(dbus_api.remote_command("ping", None)); 7410 assert!(http_api.remote_command("ping", None)); 7411 7412 // Create the VM first 7413 let cpu_count: u8 = 4; 7414 let request_body = guest.api_create_body( 7415 cpu_count, 7416 direct_kernel_boot_path().to_str().unwrap(), 7417 DIRECT_KERNEL_BOOT_CMDLINE, 7418 ); 7419 7420 let temp_config_path = guest.tmp_dir.as_path().join("config"); 7421 std::fs::write(&temp_config_path, request_body).unwrap(); 7422 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 7423 7424 let r = std::panic::catch_unwind(|| { 7425 // Create the VM 7426 assert!(dbus_api.remote_command("create", Some(create_config),)); 7427 7428 // Then boot it 7429 assert!(http_api.remote_command("boot", None)); 7430 guest.wait_vm_boot(None).unwrap(); 7431 7432 // Check that the VM booted as expected 7433 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7434 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7435 7436 // Sync and shutdown without powering off to prevent filesystem 7437 // corruption. 7438 guest.ssh_command("sync").unwrap(); 7439 guest.ssh_command("sudo shutdown -H now").unwrap(); 7440 7441 // Wait for the guest to be fully shutdown 7442 thread::sleep(std::time::Duration::new(20, 0)); 7443 7444 // Then shutdown the VM 7445 assert!(dbus_api.remote_command("shutdown", None)); 7446 7447 // Then boot it again 7448 assert!(http_api.remote_command("boot", None)); 7449 guest.wait_vm_boot(None).unwrap(); 7450 7451 // Check that the VM booted as expected 7452 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7453 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7454 }); 7455 7456 let _ = child.kill(); 7457 let output = child.wait_with_output().unwrap(); 7458 7459 handle_child_output(r, &output); 7460 } 7461 7462 #[test] 7463 fn test_api_dbus_create_boot() { 7464 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7465 let guest = Guest::new(Box::new(focal)); 7466 7467 _test_api_create_boot(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7468 } 7469 7470 #[test] 7471 fn test_api_dbus_shutdown() { 7472 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7473 let guest = Guest::new(Box::new(focal)); 7474 7475 _test_api_shutdown(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7476 } 7477 7478 #[test] 7479 fn test_api_dbus_delete() { 7480 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7481 let guest = Guest::new(Box::new(focal)); 7482 7483 _test_api_delete(TargetApi::new_dbus_api(&guest.tmp_dir), guest); 7484 } 7485 7486 #[test] 7487 fn test_api_dbus_pause_resume() { 7488 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7489 let guest = Guest::new(Box::new(focal)); 7490 7491 _test_api_pause_resume(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7492 } 7493 } 7494 7495 mod common_sequential { 7496 #[cfg(not(feature = "mshv"))] 7497 use crate::*; 7498 7499 #[test] 7500 #[cfg(not(feature = "mshv"))] 7501 fn test_memory_mergeable_on() { 7502 test_memory_mergeable(true) 7503 } 7504 } 7505 7506 mod windows { 7507 use crate::*; 7508 use once_cell::sync::Lazy; 7509 7510 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 7511 7512 struct WindowsGuest { 7513 guest: Guest, 7514 auth: PasswordAuth, 7515 } 7516 7517 trait FsType { 7518 const FS_FAT: u8; 7519 const FS_NTFS: u8; 7520 } 7521 impl FsType for WindowsGuest { 7522 const FS_FAT: u8 = 0; 7523 const FS_NTFS: u8 = 1; 7524 } 7525 7526 impl WindowsGuest { 7527 fn new() -> Self { 7528 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 7529 let guest = Guest::new(Box::new(disk)); 7530 let auth = PasswordAuth { 7531 username: String::from("administrator"), 7532 password: String::from("Admin123"), 7533 }; 7534 7535 WindowsGuest { guest, auth } 7536 } 7537 7538 fn guest(&self) -> &Guest { 7539 &self.guest 7540 } 7541 7542 fn ssh_cmd(&self, cmd: &str) -> String { 7543 ssh_command_ip_with_auth( 7544 cmd, 7545 &self.auth, 7546 &self.guest.network.guest_ip, 7547 DEFAULT_SSH_RETRIES, 7548 DEFAULT_SSH_TIMEOUT, 7549 ) 7550 .unwrap() 7551 } 7552 7553 fn cpu_count(&self) -> u8 { 7554 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 7555 .trim() 7556 .parse::<u8>() 7557 .unwrap_or(0) 7558 } 7559 7560 fn ram_size(&self) -> usize { 7561 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 7562 .trim() 7563 .parse::<usize>() 7564 .unwrap_or(0) 7565 } 7566 7567 fn netdev_count(&self) -> u8 { 7568 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7569 .trim() 7570 .parse::<u8>() 7571 .unwrap_or(0) 7572 } 7573 7574 fn disk_count(&self) -> u8 { 7575 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 7576 .trim() 7577 .parse::<u8>() 7578 .unwrap_or(0) 7579 } 7580 7581 fn reboot(&self) { 7582 let _ = self.ssh_cmd("shutdown /r /t 0"); 7583 } 7584 7585 fn shutdown(&self) { 7586 let _ = self.ssh_cmd("shutdown /s /t 0"); 7587 } 7588 7589 fn run_dnsmasq(&self) -> std::process::Child { 7590 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 7591 let dhcp_host = format!( 7592 "--dhcp-host={},{}", 7593 self.guest.network.guest_mac, self.guest.network.guest_ip 7594 ); 7595 let dhcp_range = format!( 7596 "--dhcp-range=eth,{},{}", 7597 self.guest.network.guest_ip, self.guest.network.guest_ip 7598 ); 7599 7600 Command::new("dnsmasq") 7601 .arg("--no-daemon") 7602 .arg("--log-queries") 7603 .arg(listen_address.as_str()) 7604 .arg("--except-interface=lo") 7605 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 7606 .arg("--conf-file=/dev/null") 7607 .arg(dhcp_host.as_str()) 7608 .arg(dhcp_range.as_str()) 7609 .spawn() 7610 .unwrap() 7611 } 7612 7613 // TODO Cleanup image file explicitly after test, if there's some space issues. 7614 fn disk_new(&self, fs: u8, sz: usize) -> String { 7615 let mut guard = NEXT_DISK_ID.lock().unwrap(); 7616 let id = *guard; 7617 *guard = id + 1; 7618 7619 let img = PathBuf::from(format!("/tmp/test-hotplug-{id}.raw")); 7620 let _ = fs::remove_file(&img); 7621 7622 // Create an image file 7623 let out = Command::new("qemu-img") 7624 .args([ 7625 "create", 7626 "-f", 7627 "raw", 7628 img.to_str().unwrap(), 7629 format!("{sz}m").as_str(), 7630 ]) 7631 .output() 7632 .expect("qemu-img command failed") 7633 .stdout; 7634 println!("{out:?}"); 7635 7636 // Associate image to a loop device 7637 let out = Command::new("losetup") 7638 .args(["--show", "-f", img.to_str().unwrap()]) 7639 .output() 7640 .expect("failed to create loop device") 7641 .stdout; 7642 let _tmp = String::from_utf8_lossy(&out); 7643 let loop_dev = _tmp.trim(); 7644 println!("{out:?}"); 7645 7646 // Create a partition table 7647 // echo 'type=7' | sudo sfdisk "${LOOP}" 7648 let mut child = Command::new("sfdisk") 7649 .args([loop_dev]) 7650 .stdin(Stdio::piped()) 7651 .spawn() 7652 .unwrap(); 7653 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 7654 stdin 7655 .write_all("type=7".as_bytes()) 7656 .expect("failed to write stdin"); 7657 let out = child.wait_with_output().expect("sfdisk failed").stdout; 7658 println!("{out:?}"); 7659 7660 // Disengage the loop device 7661 let out = Command::new("losetup") 7662 .args(["-d", loop_dev]) 7663 .output() 7664 .expect("loop device not found") 7665 .stdout; 7666 println!("{out:?}"); 7667 7668 // Re-associate loop device pointing to the partition only 7669 let out = Command::new("losetup") 7670 .args([ 7671 "--show", 7672 "--offset", 7673 (512 * 2048).to_string().as_str(), 7674 "-f", 7675 img.to_str().unwrap(), 7676 ]) 7677 .output() 7678 .expect("failed to create loop device") 7679 .stdout; 7680 let _tmp = String::from_utf8_lossy(&out); 7681 let loop_dev = _tmp.trim(); 7682 println!("{out:?}"); 7683 7684 // Create filesystem. 7685 let fs_cmd = match fs { 7686 WindowsGuest::FS_FAT => "mkfs.msdos", 7687 WindowsGuest::FS_NTFS => "mkfs.ntfs", 7688 _ => panic!("Unknown filesystem type '{fs}'"), 7689 }; 7690 let out = Command::new(fs_cmd) 7691 .args([&loop_dev]) 7692 .output() 7693 .unwrap_or_else(|_| panic!("{fs_cmd} failed")) 7694 .stdout; 7695 println!("{out:?}"); 7696 7697 // Disengage the loop device 7698 let out = Command::new("losetup") 7699 .args(["-d", loop_dev]) 7700 .output() 7701 .unwrap_or_else(|_| panic!("loop device '{loop_dev}' not found")) 7702 .stdout; 7703 println!("{out:?}"); 7704 7705 img.to_str().unwrap().to_string() 7706 } 7707 7708 fn disks_set_rw(&self) { 7709 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 7710 } 7711 7712 fn disks_online(&self) { 7713 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 7714 } 7715 7716 fn disk_file_put(&self, fname: &str, data: &str) { 7717 let _ = self.ssh_cmd(&format!( 7718 "powershell -Command \"'{data}' | Set-Content -Path {fname}\"" 7719 )); 7720 } 7721 7722 fn disk_file_read(&self, fname: &str) -> String { 7723 self.ssh_cmd(&format!( 7724 "powershell -Command \"Get-Content -Path {fname}\"" 7725 )) 7726 } 7727 7728 fn wait_for_boot(&self) -> bool { 7729 let cmd = "dir /b c:\\ | find \"Windows\""; 7730 let tmo_max = 180; 7731 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 7732 // interval must be small. 7733 let tmo_int = 2; 7734 let out = ssh_command_ip_with_auth( 7735 cmd, 7736 &self.auth, 7737 &self.guest.network.guest_ip, 7738 { 7739 let mut ret = 1; 7740 let mut tmo_acc = 0; 7741 loop { 7742 tmo_acc += tmo_int * ret; 7743 if tmo_acc >= tmo_max { 7744 break; 7745 } 7746 ret += 1; 7747 } 7748 ret 7749 }, 7750 tmo_int, 7751 ) 7752 .unwrap(); 7753 7754 if "Windows" == out.trim() { 7755 return true; 7756 } 7757 7758 false 7759 } 7760 } 7761 7762 fn vcpu_threads_count(pid: u32) -> u8 { 7763 // ps -T -p 12345 | grep vcpu | wc -l 7764 let out = Command::new("ps") 7765 .args(["-T", "-p", format!("{pid}").as_str()]) 7766 .output() 7767 .expect("ps command failed") 7768 .stdout; 7769 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 7770 } 7771 7772 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 7773 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 7774 let out = Command::new("ps") 7775 .args(["-T", "-p", format!("{pid}").as_str()]) 7776 .output() 7777 .expect("ps command failed") 7778 .stdout; 7779 let mut n = 0; 7780 String::from_utf8_lossy(&out) 7781 .split_whitespace() 7782 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 7783 n 7784 } 7785 7786 fn disk_ctrl_threads_count(pid: u32) -> u8 { 7787 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 7788 let out = Command::new("ps") 7789 .args(["-T", "-p", format!("{pid}").as_str()]) 7790 .output() 7791 .expect("ps command failed") 7792 .stdout; 7793 let mut n = 0; 7794 String::from_utf8_lossy(&out) 7795 .split_whitespace() 7796 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 7797 n 7798 } 7799 7800 #[test] 7801 fn test_windows_guest() { 7802 let windows_guest = WindowsGuest::new(); 7803 7804 let mut child = GuestCommand::new(windows_guest.guest()) 7805 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7806 .args(["--memory", "size=4G"]) 7807 .args(["--kernel", edk2_path().to_str().unwrap()]) 7808 .args(["--serial", "tty"]) 7809 .args(["--console", "off"]) 7810 .default_disks() 7811 .default_net() 7812 .capture_output() 7813 .spawn() 7814 .unwrap(); 7815 7816 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7817 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7818 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7819 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7820 7821 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7822 7823 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7824 7825 let r = std::panic::catch_unwind(|| { 7826 // Wait to make sure Windows boots up 7827 assert!(windows_guest.wait_for_boot()); 7828 7829 windows_guest.shutdown(); 7830 }); 7831 7832 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7833 let _ = child.kill(); 7834 let output = child.wait_with_output().unwrap(); 7835 7836 let _ = child_dnsmasq.kill(); 7837 let _ = child_dnsmasq.wait(); 7838 7839 handle_child_output(r, &output); 7840 } 7841 7842 #[test] 7843 fn test_windows_guest_multiple_queues() { 7844 let windows_guest = WindowsGuest::new(); 7845 7846 let mut ovmf_path = dirs::home_dir().unwrap(); 7847 ovmf_path.push("workloads"); 7848 ovmf_path.push(OVMF_NAME); 7849 7850 let mut child = GuestCommand::new(windows_guest.guest()) 7851 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 7852 .args(["--memory", "size=4G"]) 7853 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7854 .args(["--serial", "tty"]) 7855 .args(["--console", "off"]) 7856 .args([ 7857 "--disk", 7858 format!( 7859 "path={},num_queues=4", 7860 windows_guest 7861 .guest() 7862 .disk_config 7863 .disk(DiskType::OperatingSystem) 7864 .unwrap() 7865 ) 7866 .as_str(), 7867 ]) 7868 .args([ 7869 "--net", 7870 format!( 7871 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 7872 windows_guest.guest().network.guest_mac, 7873 windows_guest.guest().network.host_ip 7874 ) 7875 .as_str(), 7876 ]) 7877 .capture_output() 7878 .spawn() 7879 .unwrap(); 7880 7881 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7882 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7883 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7884 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7885 7886 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7887 7888 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7889 7890 let r = std::panic::catch_unwind(|| { 7891 // Wait to make sure Windows boots up 7892 assert!(windows_guest.wait_for_boot()); 7893 7894 windows_guest.shutdown(); 7895 }); 7896 7897 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7898 let _ = child.kill(); 7899 let output = child.wait_with_output().unwrap(); 7900 7901 let _ = child_dnsmasq.kill(); 7902 let _ = child_dnsmasq.wait(); 7903 7904 handle_child_output(r, &output); 7905 } 7906 7907 #[test] 7908 #[cfg(not(feature = "mshv"))] 7909 #[ignore = "See #4327"] 7910 fn test_windows_guest_snapshot_restore() { 7911 let windows_guest = WindowsGuest::new(); 7912 7913 let mut ovmf_path = dirs::home_dir().unwrap(); 7914 ovmf_path.push("workloads"); 7915 ovmf_path.push(OVMF_NAME); 7916 7917 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 7918 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 7919 7920 let mut child = GuestCommand::new(windows_guest.guest()) 7921 .args(["--api-socket", &api_socket_source]) 7922 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 7923 .args(["--memory", "size=4G"]) 7924 .args(["--kernel", ovmf_path.to_str().unwrap()]) 7925 .args(["--serial", "tty"]) 7926 .args(["--console", "off"]) 7927 .default_disks() 7928 .default_net() 7929 .capture_output() 7930 .spawn() 7931 .unwrap(); 7932 7933 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 7934 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7935 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 7936 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 7937 7938 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 7939 7940 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 7941 7942 // Wait to make sure Windows boots up 7943 assert!(windows_guest.wait_for_boot()); 7944 7945 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 7946 7947 // Pause the VM 7948 assert!(remote_command(&api_socket_source, "pause", None)); 7949 7950 // Take a snapshot from the VM 7951 assert!(remote_command( 7952 &api_socket_source, 7953 "snapshot", 7954 Some(format!("file://{snapshot_dir}").as_str()), 7955 )); 7956 7957 // Wait to make sure the snapshot is completed 7958 thread::sleep(std::time::Duration::new(30, 0)); 7959 7960 let _ = child.kill(); 7961 child.wait().unwrap(); 7962 7963 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 7964 7965 // Restore the VM from the snapshot 7966 let mut child = GuestCommand::new(windows_guest.guest()) 7967 .args(["--api-socket", &api_socket_restored]) 7968 .args([ 7969 "--restore", 7970 format!("source_url=file://{snapshot_dir}").as_str(), 7971 ]) 7972 .capture_output() 7973 .spawn() 7974 .unwrap(); 7975 7976 // Wait for the VM to be restored 7977 thread::sleep(std::time::Duration::new(20, 0)); 7978 7979 let r = std::panic::catch_unwind(|| { 7980 // Resume the VM 7981 assert!(remote_command(&api_socket_restored, "resume", None)); 7982 7983 windows_guest.shutdown(); 7984 }); 7985 7986 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 7987 let _ = child.kill(); 7988 let output = child.wait_with_output().unwrap(); 7989 7990 let _ = child_dnsmasq.kill(); 7991 let _ = child_dnsmasq.wait(); 7992 7993 handle_child_output(r, &output); 7994 } 7995 7996 #[test] 7997 #[cfg(not(feature = "mshv"))] 7998 #[cfg(not(target_arch = "aarch64"))] 7999 fn test_windows_guest_cpu_hotplug() { 8000 let windows_guest = WindowsGuest::new(); 8001 8002 let mut ovmf_path = dirs::home_dir().unwrap(); 8003 ovmf_path.push("workloads"); 8004 ovmf_path.push(OVMF_NAME); 8005 8006 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8007 let api_socket = temp_api_path(&tmp_dir); 8008 8009 let mut child = GuestCommand::new(windows_guest.guest()) 8010 .args(["--api-socket", &api_socket]) 8011 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 8012 .args(["--memory", "size=4G"]) 8013 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8014 .args(["--serial", "tty"]) 8015 .args(["--console", "off"]) 8016 .default_disks() 8017 .default_net() 8018 .capture_output() 8019 .spawn() 8020 .unwrap(); 8021 8022 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8023 8024 let r = std::panic::catch_unwind(|| { 8025 // Wait to make sure Windows boots up 8026 assert!(windows_guest.wait_for_boot()); 8027 8028 let vcpu_num = 2; 8029 // Check the initial number of CPUs the guest sees 8030 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8031 // Check the initial number of vcpu threads in the CH process 8032 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8033 8034 let vcpu_num = 6; 8035 // Hotplug some CPUs 8036 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8037 // Wait to make sure CPUs are added 8038 thread::sleep(std::time::Duration::new(10, 0)); 8039 // Check the guest sees the correct number 8040 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8041 // Check the CH process has the correct number of vcpu threads 8042 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8043 8044 let vcpu_num = 4; 8045 // Remove some CPUs. Note that Windows doesn't support hot-remove. 8046 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8047 // Wait to make sure CPUs are removed 8048 thread::sleep(std::time::Duration::new(10, 0)); 8049 // Reboot to let Windows catch up 8050 windows_guest.reboot(); 8051 // Wait to make sure Windows completely rebooted 8052 thread::sleep(std::time::Duration::new(60, 0)); 8053 // Check the guest sees the correct number 8054 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8055 // Check the CH process has the correct number of vcpu threads 8056 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8057 8058 windows_guest.shutdown(); 8059 }); 8060 8061 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8062 let _ = child.kill(); 8063 let output = child.wait_with_output().unwrap(); 8064 8065 let _ = child_dnsmasq.kill(); 8066 let _ = child_dnsmasq.wait(); 8067 8068 handle_child_output(r, &output); 8069 } 8070 8071 #[test] 8072 #[cfg(not(feature = "mshv"))] 8073 #[cfg(not(target_arch = "aarch64"))] 8074 fn test_windows_guest_ram_hotplug() { 8075 let windows_guest = WindowsGuest::new(); 8076 8077 let mut ovmf_path = dirs::home_dir().unwrap(); 8078 ovmf_path.push("workloads"); 8079 ovmf_path.push(OVMF_NAME); 8080 8081 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8082 let api_socket = temp_api_path(&tmp_dir); 8083 8084 let mut child = GuestCommand::new(windows_guest.guest()) 8085 .args(["--api-socket", &api_socket]) 8086 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8087 .args(["--memory", "size=2G,hotplug_size=5G"]) 8088 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8089 .args(["--serial", "tty"]) 8090 .args(["--console", "off"]) 8091 .default_disks() 8092 .default_net() 8093 .capture_output() 8094 .spawn() 8095 .unwrap(); 8096 8097 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8098 8099 let r = std::panic::catch_unwind(|| { 8100 // Wait to make sure Windows boots up 8101 assert!(windows_guest.wait_for_boot()); 8102 8103 let ram_size = 2 * 1024 * 1024 * 1024; 8104 // Check the initial number of RAM the guest sees 8105 let current_ram_size = windows_guest.ram_size(); 8106 // This size seems to be reserved by the system and thus the 8107 // reported amount differs by this constant value. 8108 let reserved_ram_size = ram_size - current_ram_size; 8109 // Verify that there's not more than 4mb constant diff wasted 8110 // by the reserved ram. 8111 assert!(reserved_ram_size < 4 * 1024 * 1024); 8112 8113 let ram_size = 4 * 1024 * 1024 * 1024; 8114 // Hotplug some RAM 8115 resize_command(&api_socket, None, Some(ram_size), None, None); 8116 // Wait to make sure RAM has been added 8117 thread::sleep(std::time::Duration::new(10, 0)); 8118 // Check the guest sees the correct number 8119 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8120 8121 let ram_size = 3 * 1024 * 1024 * 1024; 8122 // Unplug some RAM. Note that hot-remove most likely won't work. 8123 resize_command(&api_socket, None, Some(ram_size), None, None); 8124 // Wait to make sure RAM has been added 8125 thread::sleep(std::time::Duration::new(10, 0)); 8126 // Reboot to let Windows catch up 8127 windows_guest.reboot(); 8128 // Wait to make sure guest completely rebooted 8129 thread::sleep(std::time::Duration::new(60, 0)); 8130 // Check the guest sees the correct number 8131 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8132 8133 windows_guest.shutdown(); 8134 }); 8135 8136 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8137 let _ = child.kill(); 8138 let output = child.wait_with_output().unwrap(); 8139 8140 let _ = child_dnsmasq.kill(); 8141 let _ = child_dnsmasq.wait(); 8142 8143 handle_child_output(r, &output); 8144 } 8145 8146 #[test] 8147 #[cfg(not(feature = "mshv"))] 8148 fn test_windows_guest_netdev_hotplug() { 8149 let windows_guest = WindowsGuest::new(); 8150 8151 let mut ovmf_path = dirs::home_dir().unwrap(); 8152 ovmf_path.push("workloads"); 8153 ovmf_path.push(OVMF_NAME); 8154 8155 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8156 let api_socket = temp_api_path(&tmp_dir); 8157 8158 let mut child = GuestCommand::new(windows_guest.guest()) 8159 .args(["--api-socket", &api_socket]) 8160 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8161 .args(["--memory", "size=4G"]) 8162 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8163 .args(["--serial", "tty"]) 8164 .args(["--console", "off"]) 8165 .default_disks() 8166 .default_net() 8167 .capture_output() 8168 .spawn() 8169 .unwrap(); 8170 8171 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8172 8173 let r = std::panic::catch_unwind(|| { 8174 // Wait to make sure Windows boots up 8175 assert!(windows_guest.wait_for_boot()); 8176 8177 // Initially present network device 8178 let netdev_num = 1; 8179 assert_eq!(windows_guest.netdev_count(), netdev_num); 8180 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8181 8182 // Hotplug network device 8183 let (cmd_success, cmd_output) = remote_command_w_output( 8184 &api_socket, 8185 "add-net", 8186 Some(windows_guest.guest().default_net_string().as_str()), 8187 ); 8188 assert!(cmd_success); 8189 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 8190 thread::sleep(std::time::Duration::new(5, 0)); 8191 // Verify the device is on the system 8192 let netdev_num = 2; 8193 assert_eq!(windows_guest.netdev_count(), netdev_num); 8194 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8195 8196 // Remove network device 8197 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 8198 assert!(cmd_success); 8199 thread::sleep(std::time::Duration::new(5, 0)); 8200 // Verify the device has been removed 8201 let netdev_num = 1; 8202 assert_eq!(windows_guest.netdev_count(), netdev_num); 8203 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8204 8205 windows_guest.shutdown(); 8206 }); 8207 8208 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8209 let _ = child.kill(); 8210 let output = child.wait_with_output().unwrap(); 8211 8212 let _ = child_dnsmasq.kill(); 8213 let _ = child_dnsmasq.wait(); 8214 8215 handle_child_output(r, &output); 8216 } 8217 8218 #[test] 8219 #[ignore = "See #6037"] 8220 #[cfg(not(feature = "mshv"))] 8221 #[cfg(not(target_arch = "aarch64"))] 8222 fn test_windows_guest_disk_hotplug() { 8223 let windows_guest = WindowsGuest::new(); 8224 8225 let mut ovmf_path = dirs::home_dir().unwrap(); 8226 ovmf_path.push("workloads"); 8227 ovmf_path.push(OVMF_NAME); 8228 8229 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8230 let api_socket = temp_api_path(&tmp_dir); 8231 8232 let mut child = GuestCommand::new(windows_guest.guest()) 8233 .args(["--api-socket", &api_socket]) 8234 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8235 .args(["--memory", "size=4G"]) 8236 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8237 .args(["--serial", "tty"]) 8238 .args(["--console", "off"]) 8239 .default_disks() 8240 .default_net() 8241 .capture_output() 8242 .spawn() 8243 .unwrap(); 8244 8245 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8246 8247 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 8248 8249 let r = std::panic::catch_unwind(|| { 8250 // Wait to make sure Windows boots up 8251 assert!(windows_guest.wait_for_boot()); 8252 8253 // Initially present disk device 8254 let disk_num = 1; 8255 assert_eq!(windows_guest.disk_count(), disk_num); 8256 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8257 8258 // Hotplug disk device 8259 let (cmd_success, cmd_output) = remote_command_w_output( 8260 &api_socket, 8261 "add-disk", 8262 Some(format!("path={disk},readonly=off").as_str()), 8263 ); 8264 assert!(cmd_success); 8265 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 8266 thread::sleep(std::time::Duration::new(5, 0)); 8267 // Online disk device 8268 windows_guest.disks_set_rw(); 8269 windows_guest.disks_online(); 8270 // Verify the device is on the system 8271 let disk_num = 2; 8272 assert_eq!(windows_guest.disk_count(), disk_num); 8273 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8274 8275 let data = "hello"; 8276 let fname = "d:\\world"; 8277 windows_guest.disk_file_put(fname, data); 8278 8279 // Unmount disk device 8280 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 8281 assert!(cmd_success); 8282 thread::sleep(std::time::Duration::new(5, 0)); 8283 // Verify the device has been removed 8284 let disk_num = 1; 8285 assert_eq!(windows_guest.disk_count(), disk_num); 8286 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8287 8288 // Remount and check the file exists with the expected contents 8289 let (cmd_success, _cmd_output) = remote_command_w_output( 8290 &api_socket, 8291 "add-disk", 8292 Some(format!("path={disk},readonly=off").as_str()), 8293 ); 8294 assert!(cmd_success); 8295 thread::sleep(std::time::Duration::new(5, 0)); 8296 let out = windows_guest.disk_file_read(fname); 8297 assert_eq!(data, out.trim()); 8298 8299 // Intentionally no unmount, it'll happen at shutdown. 8300 8301 windows_guest.shutdown(); 8302 }); 8303 8304 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8305 let _ = child.kill(); 8306 let output = child.wait_with_output().unwrap(); 8307 8308 let _ = child_dnsmasq.kill(); 8309 let _ = child_dnsmasq.wait(); 8310 8311 handle_child_output(r, &output); 8312 } 8313 8314 #[test] 8315 #[ignore = "See #6037"] 8316 #[cfg(not(feature = "mshv"))] 8317 #[cfg(not(target_arch = "aarch64"))] 8318 fn test_windows_guest_disk_hotplug_multi() { 8319 let windows_guest = WindowsGuest::new(); 8320 8321 let mut ovmf_path = dirs::home_dir().unwrap(); 8322 ovmf_path.push("workloads"); 8323 ovmf_path.push(OVMF_NAME); 8324 8325 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8326 let api_socket = temp_api_path(&tmp_dir); 8327 8328 let mut child = GuestCommand::new(windows_guest.guest()) 8329 .args(["--api-socket", &api_socket]) 8330 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8331 .args(["--memory", "size=2G"]) 8332 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8333 .args(["--serial", "tty"]) 8334 .args(["--console", "off"]) 8335 .default_disks() 8336 .default_net() 8337 .capture_output() 8338 .spawn() 8339 .unwrap(); 8340 8341 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8342 8343 // Predefined data to used at various test stages 8344 let disk_test_data: [[String; 4]; 2] = [ 8345 [ 8346 "_disk2".to_string(), 8347 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 8348 "d:\\world".to_string(), 8349 "hello".to_string(), 8350 ], 8351 [ 8352 "_disk3".to_string(), 8353 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 8354 "e:\\hello".to_string(), 8355 "world".to_string(), 8356 ], 8357 ]; 8358 8359 let r = std::panic::catch_unwind(|| { 8360 // Wait to make sure Windows boots up 8361 assert!(windows_guest.wait_for_boot()); 8362 8363 // Initially present disk device 8364 let disk_num = 1; 8365 assert_eq!(windows_guest.disk_count(), disk_num); 8366 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8367 8368 for it in &disk_test_data { 8369 let disk_id = it[0].as_str(); 8370 let disk = it[1].as_str(); 8371 // Hotplug disk device 8372 let (cmd_success, cmd_output) = remote_command_w_output( 8373 &api_socket, 8374 "add-disk", 8375 Some(format!("path={disk},readonly=off").as_str()), 8376 ); 8377 assert!(cmd_success); 8378 assert!(String::from_utf8_lossy(&cmd_output) 8379 .contains(format!("\"id\":\"{disk_id}\"").as_str())); 8380 thread::sleep(std::time::Duration::new(5, 0)); 8381 // Online disk devices 8382 windows_guest.disks_set_rw(); 8383 windows_guest.disks_online(); 8384 } 8385 // Verify the devices are on the system 8386 let disk_num = (disk_test_data.len() + 1) as u8; 8387 assert_eq!(windows_guest.disk_count(), disk_num); 8388 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8389 8390 // Put test data 8391 for it in &disk_test_data { 8392 let fname = it[2].as_str(); 8393 let data = it[3].as_str(); 8394 windows_guest.disk_file_put(fname, data); 8395 } 8396 8397 // Unmount disk devices 8398 for it in &disk_test_data { 8399 let disk_id = it[0].as_str(); 8400 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 8401 assert!(cmd_success); 8402 thread::sleep(std::time::Duration::new(5, 0)); 8403 } 8404 8405 // Verify the devices have been removed 8406 let disk_num = 1; 8407 assert_eq!(windows_guest.disk_count(), disk_num); 8408 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8409 8410 // Remount 8411 for it in &disk_test_data { 8412 let disk = it[1].as_str(); 8413 let (cmd_success, _cmd_output) = remote_command_w_output( 8414 &api_socket, 8415 "add-disk", 8416 Some(format!("path={disk},readonly=off").as_str()), 8417 ); 8418 assert!(cmd_success); 8419 thread::sleep(std::time::Duration::new(5, 0)); 8420 } 8421 8422 // Check the files exists with the expected contents 8423 for it in &disk_test_data { 8424 let fname = it[2].as_str(); 8425 let data = it[3].as_str(); 8426 let out = windows_guest.disk_file_read(fname); 8427 assert_eq!(data, out.trim()); 8428 } 8429 8430 // Intentionally no unmount, it'll happen at shutdown. 8431 8432 windows_guest.shutdown(); 8433 }); 8434 8435 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8436 let _ = child.kill(); 8437 let output = child.wait_with_output().unwrap(); 8438 8439 let _ = child_dnsmasq.kill(); 8440 let _ = child_dnsmasq.wait(); 8441 8442 handle_child_output(r, &output); 8443 } 8444 8445 #[test] 8446 #[cfg(not(feature = "mshv"))] 8447 #[cfg(not(target_arch = "aarch64"))] 8448 fn test_windows_guest_netdev_multi() { 8449 let windows_guest = WindowsGuest::new(); 8450 8451 let mut ovmf_path = dirs::home_dir().unwrap(); 8452 ovmf_path.push("workloads"); 8453 ovmf_path.push(OVMF_NAME); 8454 8455 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8456 let api_socket = temp_api_path(&tmp_dir); 8457 8458 let mut child = GuestCommand::new(windows_guest.guest()) 8459 .args(["--api-socket", &api_socket]) 8460 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8461 .args(["--memory", "size=4G"]) 8462 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8463 .args(["--serial", "tty"]) 8464 .args(["--console", "off"]) 8465 .default_disks() 8466 // The multi net dev config is borrowed from test_multiple_network_interfaces 8467 .args([ 8468 "--net", 8469 windows_guest.guest().default_net_string().as_str(), 8470 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 8471 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 8472 ]) 8473 .capture_output() 8474 .spawn() 8475 .unwrap(); 8476 8477 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8478 8479 let r = std::panic::catch_unwind(|| { 8480 // Wait to make sure Windows boots up 8481 assert!(windows_guest.wait_for_boot()); 8482 8483 let netdev_num = 3; 8484 assert_eq!(windows_guest.netdev_count(), netdev_num); 8485 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8486 8487 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 8488 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 8489 8490 windows_guest.shutdown(); 8491 }); 8492 8493 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8494 let _ = child.kill(); 8495 let output = child.wait_with_output().unwrap(); 8496 8497 let _ = child_dnsmasq.kill(); 8498 let _ = child_dnsmasq.wait(); 8499 8500 handle_child_output(r, &output); 8501 } 8502 } 8503 8504 #[cfg(target_arch = "x86_64")] 8505 mod sgx { 8506 use crate::*; 8507 8508 #[test] 8509 fn test_sgx() { 8510 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 8511 let jammy = UbuntuDiskConfig::new(jammy_image); 8512 let guest = Guest::new(Box::new(jammy)); 8513 8514 let mut child = GuestCommand::new(&guest) 8515 .args(["--cpus", "boot=1"]) 8516 .args(["--memory", "size=512M"]) 8517 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8518 .default_disks() 8519 .default_net() 8520 .args(["--sgx-epc", "id=epc0,size=64M"]) 8521 .capture_output() 8522 .spawn() 8523 .unwrap(); 8524 8525 let r = std::panic::catch_unwind(|| { 8526 guest.wait_vm_boot(None).unwrap(); 8527 8528 // Check if SGX is correctly detected in the guest. 8529 guest.check_sgx_support().unwrap(); 8530 8531 // Validate the SGX EPC section is 64MiB. 8532 assert_eq!( 8533 guest 8534 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 8535 .unwrap() 8536 .trim(), 8537 "0x0000000004000000" 8538 ); 8539 }); 8540 8541 let _ = child.kill(); 8542 let output = child.wait_with_output().unwrap(); 8543 8544 handle_child_output(r, &output); 8545 } 8546 } 8547 8548 #[cfg(target_arch = "x86_64")] 8549 mod vfio { 8550 use crate::*; 8551 8552 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 8553 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8554 let guest = Guest::new(Box::new(jammy)); 8555 let api_socket = temp_api_path(&guest.tmp_dir); 8556 8557 let mut child = GuestCommand::new(&guest) 8558 .args(["--cpus", "boot=4"]) 8559 .args([ 8560 "--memory", 8561 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(), 8562 ]) 8563 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8564 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8565 .args(["--api-socket", &api_socket]) 8566 .default_disks() 8567 .default_net() 8568 .capture_output() 8569 .spawn() 8570 .unwrap(); 8571 8572 let r = std::panic::catch_unwind(|| { 8573 guest.wait_vm_boot(None).unwrap(); 8574 8575 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8576 8577 guest.enable_memory_hotplug(); 8578 8579 // Add RAM to the VM 8580 let desired_ram = 6 << 30; 8581 resize_command(&api_socket, None, Some(desired_ram), None, None); 8582 thread::sleep(std::time::Duration::new(30, 0)); 8583 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 8584 8585 // Check the VFIO device works when RAM is increased to 6GiB 8586 guest.check_nvidia_gpu(); 8587 }); 8588 8589 let _ = child.kill(); 8590 let output = child.wait_with_output().unwrap(); 8591 8592 handle_child_output(r, &output); 8593 } 8594 8595 #[test] 8596 fn test_nvidia_card_memory_hotplug_acpi() { 8597 test_nvidia_card_memory_hotplug("acpi") 8598 } 8599 8600 #[test] 8601 fn test_nvidia_card_memory_hotplug_virtio_mem() { 8602 test_nvidia_card_memory_hotplug("virtio-mem") 8603 } 8604 8605 #[test] 8606 fn test_nvidia_card_pci_hotplug() { 8607 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8608 let guest = Guest::new(Box::new(jammy)); 8609 let api_socket = temp_api_path(&guest.tmp_dir); 8610 8611 let mut child = GuestCommand::new(&guest) 8612 .args(["--cpus", "boot=4"]) 8613 .args(["--memory", "size=4G"]) 8614 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8615 .args(["--api-socket", &api_socket]) 8616 .default_disks() 8617 .default_net() 8618 .capture_output() 8619 .spawn() 8620 .unwrap(); 8621 8622 let r = std::panic::catch_unwind(|| { 8623 guest.wait_vm_boot(None).unwrap(); 8624 8625 // Hotplug the card to the VM 8626 let (cmd_success, cmd_output) = remote_command_w_output( 8627 &api_socket, 8628 "add-device", 8629 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 8630 ); 8631 assert!(cmd_success); 8632 assert!(String::from_utf8_lossy(&cmd_output) 8633 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 8634 8635 thread::sleep(std::time::Duration::new(10, 0)); 8636 8637 // Check the VFIO device works after hotplug 8638 guest.check_nvidia_gpu(); 8639 }); 8640 8641 let _ = child.kill(); 8642 let output = child.wait_with_output().unwrap(); 8643 8644 handle_child_output(r, &output); 8645 } 8646 8647 #[test] 8648 fn test_nvidia_card_reboot() { 8649 let jammy = UbuntuDiskConfig::new(JAMMY_NVIDIA_IMAGE_NAME.to_string()); 8650 let guest = Guest::new(Box::new(jammy)); 8651 let api_socket = temp_api_path(&guest.tmp_dir); 8652 8653 let mut child = GuestCommand::new(&guest) 8654 .args(["--cpus", "boot=4"]) 8655 .args(["--memory", "size=4G"]) 8656 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8657 .args(["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 8658 .args(["--api-socket", &api_socket]) 8659 .default_disks() 8660 .default_net() 8661 .capture_output() 8662 .spawn() 8663 .unwrap(); 8664 8665 let r = std::panic::catch_unwind(|| { 8666 guest.wait_vm_boot(None).unwrap(); 8667 8668 // Check the VFIO device works after boot 8669 guest.check_nvidia_gpu(); 8670 8671 guest.reboot_linux(0, None); 8672 8673 // Check the VFIO device works after reboot 8674 guest.check_nvidia_gpu(); 8675 }); 8676 8677 let _ = child.kill(); 8678 let output = child.wait_with_output().unwrap(); 8679 8680 handle_child_output(r, &output); 8681 } 8682 } 8683 8684 mod live_migration { 8685 use crate::*; 8686 8687 fn start_live_migration( 8688 migration_socket: &str, 8689 src_api_socket: &str, 8690 dest_api_socket: &str, 8691 local: bool, 8692 ) -> bool { 8693 // Start to receive migration from the destintion VM 8694 let mut receive_migration = Command::new(clh_command("ch-remote")) 8695 .args([ 8696 &format!("--api-socket={dest_api_socket}"), 8697 "receive-migration", 8698 &format! {"unix:{migration_socket}"}, 8699 ]) 8700 .stderr(Stdio::piped()) 8701 .stdout(Stdio::piped()) 8702 .spawn() 8703 .unwrap(); 8704 // Give it '1s' to make sure the 'migration_socket' file is properly created 8705 thread::sleep(std::time::Duration::new(1, 0)); 8706 // Start to send migration from the source VM 8707 8708 let mut args = [ 8709 format!("--api-socket={}", &src_api_socket), 8710 "send-migration".to_string(), 8711 format! {"unix:{migration_socket}"}, 8712 ] 8713 .to_vec(); 8714 8715 if local { 8716 args.insert(2, "--local".to_string()); 8717 } 8718 8719 let mut send_migration = Command::new(clh_command("ch-remote")) 8720 .args(&args) 8721 .stderr(Stdio::piped()) 8722 .stdout(Stdio::piped()) 8723 .spawn() 8724 .unwrap(); 8725 8726 // The 'send-migration' command should be executed successfully within the given timeout 8727 let send_success = if let Some(status) = send_migration 8728 .wait_timeout(std::time::Duration::from_secs(30)) 8729 .unwrap() 8730 { 8731 status.success() 8732 } else { 8733 false 8734 }; 8735 8736 if !send_success { 8737 let _ = send_migration.kill(); 8738 let output = send_migration.wait_with_output().unwrap(); 8739 eprintln!( 8740 "\n\n==== Start 'send_migration' output ==== \ 8741 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8742 \n\n==== End 'send_migration' output ====\n\n", 8743 String::from_utf8_lossy(&output.stdout), 8744 String::from_utf8_lossy(&output.stderr) 8745 ); 8746 } 8747 8748 // The 'receive-migration' command should be executed successfully within the given timeout 8749 let receive_success = if let Some(status) = receive_migration 8750 .wait_timeout(std::time::Duration::from_secs(30)) 8751 .unwrap() 8752 { 8753 status.success() 8754 } else { 8755 false 8756 }; 8757 8758 if !receive_success { 8759 let _ = receive_migration.kill(); 8760 let output = receive_migration.wait_with_output().unwrap(); 8761 eprintln!( 8762 "\n\n==== Start 'receive_migration' output ==== \ 8763 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 8764 \n\n==== End 'receive_migration' output ====\n\n", 8765 String::from_utf8_lossy(&output.stdout), 8766 String::from_utf8_lossy(&output.stderr) 8767 ); 8768 } 8769 8770 send_success && receive_success 8771 } 8772 8773 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 8774 let mut src_vm = src_vm; 8775 let mut dest_vm = dest_vm; 8776 8777 let _ = src_vm.kill(); 8778 let src_output = src_vm.wait_with_output().unwrap(); 8779 eprintln!( 8780 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 8781 String::from_utf8_lossy(&src_output.stdout) 8782 ); 8783 eprintln!( 8784 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 8785 String::from_utf8_lossy(&src_output.stderr) 8786 ); 8787 let _ = dest_vm.kill(); 8788 let dest_output = dest_vm.wait_with_output().unwrap(); 8789 eprintln!( 8790 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 8791 String::from_utf8_lossy(&dest_output.stdout) 8792 ); 8793 eprintln!( 8794 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 8795 String::from_utf8_lossy(&dest_output.stderr) 8796 ); 8797 8798 if let Some(ovs_vm) = ovs_vm { 8799 let mut ovs_vm = ovs_vm; 8800 let _ = ovs_vm.kill(); 8801 let ovs_output = ovs_vm.wait_with_output().unwrap(); 8802 eprintln!( 8803 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 8804 String::from_utf8_lossy(&ovs_output.stdout) 8805 ); 8806 eprintln!( 8807 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 8808 String::from_utf8_lossy(&ovs_output.stderr) 8809 ); 8810 8811 cleanup_ovs_dpdk(); 8812 } 8813 8814 panic!("Test failed: {message}") 8815 } 8816 8817 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 8818 // same host. It ensures the following behaviors: 8819 // 1. The source VM is up and functional (including various virtio-devices are working properly); 8820 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 8821 // 3. The source VM terminated gracefully after live migration; 8822 // 4. The destination VM is functional (including various virtio-devices are working properly) after 8823 // live migration; 8824 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 8825 fn _test_live_migration(upgrade_test: bool, local: bool) { 8826 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8827 let guest = Guest::new(Box::new(focal)); 8828 let kernel_path = direct_kernel_boot_path(); 8829 let console_text = String::from("On a branch floating down river a cricket, singing."); 8830 let net_id = "net123"; 8831 let net_params = format!( 8832 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8833 net_id, guest.network.guest_mac, guest.network.host_ip 8834 ); 8835 8836 let memory_param: &[&str] = if local { 8837 &["--memory", "size=4G,shared=on"] 8838 } else { 8839 &["--memory", "size=4G"] 8840 }; 8841 8842 let boot_vcpus = 2; 8843 let max_vcpus = 4; 8844 8845 let pmem_temp_file = TempFile::new().unwrap(); 8846 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 8847 std::process::Command::new("mkfs.ext4") 8848 .arg(pmem_temp_file.as_path()) 8849 .output() 8850 .expect("Expect creating disk image to succeed"); 8851 let pmem_path = String::from("/dev/pmem0"); 8852 8853 // Start the source VM 8854 let src_vm_path = if !upgrade_test { 8855 clh_command("cloud-hypervisor") 8856 } else { 8857 cloud_hypervisor_release_path() 8858 }; 8859 let src_api_socket = temp_api_path(&guest.tmp_dir); 8860 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 8861 src_vm_cmd 8862 .args([ 8863 "--cpus", 8864 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 8865 ]) 8866 .args(memory_param) 8867 .args(["--kernel", kernel_path.to_str().unwrap()]) 8868 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 8869 .default_disks() 8870 .args(["--net", net_params.as_str()]) 8871 .args(["--api-socket", &src_api_socket]) 8872 .args([ 8873 "--pmem", 8874 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 8875 ]); 8876 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 8877 8878 // Start the destination VM 8879 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 8880 dest_api_socket.push_str(".dest"); 8881 let mut dest_child = GuestCommand::new(&guest) 8882 .args(["--api-socket", &dest_api_socket]) 8883 .capture_output() 8884 .spawn() 8885 .unwrap(); 8886 8887 let r = std::panic::catch_unwind(|| { 8888 guest.wait_vm_boot(None).unwrap(); 8889 8890 // Make sure the source VM is functaionl 8891 // Check the number of vCPUs 8892 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8893 8894 // Check the guest RAM 8895 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8896 8897 // Check the guest virtio-devices, e.g. block, rng, console, and net 8898 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8899 8900 // x86_64: Following what's done in the `test_snapshot_restore`, we need 8901 // to make sure that removing and adding back the virtio-net device does 8902 // not break the live-migration support for virtio-pci. 8903 #[cfg(target_arch = "x86_64")] 8904 { 8905 assert!(remote_command( 8906 &src_api_socket, 8907 "remove-device", 8908 Some(net_id), 8909 )); 8910 thread::sleep(std::time::Duration::new(10, 0)); 8911 8912 // Plug the virtio-net device again 8913 assert!(remote_command( 8914 &src_api_socket, 8915 "add-net", 8916 Some(net_params.as_str()), 8917 )); 8918 thread::sleep(std::time::Duration::new(10, 0)); 8919 } 8920 8921 // Start the live-migration 8922 let migration_socket = String::from( 8923 guest 8924 .tmp_dir 8925 .as_path() 8926 .join("live-migration.sock") 8927 .to_str() 8928 .unwrap(), 8929 ); 8930 8931 assert!( 8932 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 8933 "Unsuccessful command: 'send-migration' or 'receive-migration'." 8934 ); 8935 }); 8936 8937 // Check and report any errors occurred during the live-migration 8938 if r.is_err() { 8939 print_and_panic( 8940 src_child, 8941 dest_child, 8942 None, 8943 "Error occurred during live-migration", 8944 ); 8945 } 8946 8947 // Check the source vm has been terminated successful (give it '3s' to settle) 8948 thread::sleep(std::time::Duration::new(3, 0)); 8949 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 8950 print_and_panic( 8951 src_child, 8952 dest_child, 8953 None, 8954 "source VM was not terminated successfully.", 8955 ); 8956 }; 8957 8958 // Post live-migration check to make sure the destination VM is funcational 8959 let r = std::panic::catch_unwind(|| { 8960 // Perform same checks to validate VM has been properly migrated 8961 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 8962 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 8963 8964 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 8965 }); 8966 8967 // Clean-up the destination VM and make sure it terminated correctly 8968 let _ = dest_child.kill(); 8969 let dest_output = dest_child.wait_with_output().unwrap(); 8970 handle_child_output(r, &dest_output); 8971 8972 // Check the destination VM has the expected 'concole_text' from its output 8973 let r = std::panic::catch_unwind(|| { 8974 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 8975 }); 8976 handle_child_output(r, &dest_output); 8977 } 8978 8979 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 8980 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 8981 let guest = Guest::new(Box::new(focal)); 8982 let kernel_path = direct_kernel_boot_path(); 8983 let console_text = String::from("On a branch floating down river a cricket, singing."); 8984 let net_id = "net123"; 8985 let net_params = format!( 8986 "id={},tap=,mac={},ip={},mask=255.255.255.0", 8987 net_id, guest.network.guest_mac, guest.network.host_ip 8988 ); 8989 8990 let memory_param: &[&str] = if local { 8991 &[ 8992 "--memory", 8993 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 8994 "--balloon", 8995 "size=0", 8996 ] 8997 } else { 8998 &[ 8999 "--memory", 9000 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 9001 "--balloon", 9002 "size=0", 9003 ] 9004 }; 9005 9006 let boot_vcpus = 2; 9007 let max_vcpus = 4; 9008 9009 let pmem_temp_file = TempFile::new().unwrap(); 9010 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9011 std::process::Command::new("mkfs.ext4") 9012 .arg(pmem_temp_file.as_path()) 9013 .output() 9014 .expect("Expect creating disk image to succeed"); 9015 let pmem_path = String::from("/dev/pmem0"); 9016 9017 // Start the source VM 9018 let src_vm_path = if !upgrade_test { 9019 clh_command("cloud-hypervisor") 9020 } else { 9021 cloud_hypervisor_release_path() 9022 }; 9023 let src_api_socket = temp_api_path(&guest.tmp_dir); 9024 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9025 src_vm_cmd 9026 .args([ 9027 "--cpus", 9028 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9029 ]) 9030 .args(memory_param) 9031 .args(["--kernel", kernel_path.to_str().unwrap()]) 9032 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9033 .default_disks() 9034 .args(["--net", net_params.as_str()]) 9035 .args(["--api-socket", &src_api_socket]) 9036 .args([ 9037 "--pmem", 9038 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9039 ]); 9040 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9041 9042 // Start the destination VM 9043 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9044 dest_api_socket.push_str(".dest"); 9045 let mut dest_child = GuestCommand::new(&guest) 9046 .args(["--api-socket", &dest_api_socket]) 9047 .capture_output() 9048 .spawn() 9049 .unwrap(); 9050 9051 let r = std::panic::catch_unwind(|| { 9052 guest.wait_vm_boot(None).unwrap(); 9053 9054 // Make sure the source VM is functaionl 9055 // Check the number of vCPUs 9056 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9057 9058 // Check the guest RAM 9059 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9060 // Increase the guest RAM 9061 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 9062 thread::sleep(std::time::Duration::new(5, 0)); 9063 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9064 // Use balloon to remove RAM from the VM 9065 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 9066 thread::sleep(std::time::Duration::new(5, 0)); 9067 let total_memory = guest.get_total_memory().unwrap_or_default(); 9068 assert!(total_memory > 4_800_000); 9069 assert!(total_memory < 5_760_000); 9070 9071 // Check the guest virtio-devices, e.g. block, rng, console, and net 9072 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9073 9074 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9075 // to make sure that removing and adding back the virtio-net device does 9076 // not break the live-migration support for virtio-pci. 9077 #[cfg(target_arch = "x86_64")] 9078 { 9079 assert!(remote_command( 9080 &src_api_socket, 9081 "remove-device", 9082 Some(net_id), 9083 )); 9084 thread::sleep(std::time::Duration::new(10, 0)); 9085 9086 // Plug the virtio-net device again 9087 assert!(remote_command( 9088 &src_api_socket, 9089 "add-net", 9090 Some(net_params.as_str()), 9091 )); 9092 thread::sleep(std::time::Duration::new(10, 0)); 9093 } 9094 9095 // Start the live-migration 9096 let migration_socket = String::from( 9097 guest 9098 .tmp_dir 9099 .as_path() 9100 .join("live-migration.sock") 9101 .to_str() 9102 .unwrap(), 9103 ); 9104 9105 assert!( 9106 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9107 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9108 ); 9109 }); 9110 9111 // Check and report any errors occurred during the live-migration 9112 if r.is_err() { 9113 print_and_panic( 9114 src_child, 9115 dest_child, 9116 None, 9117 "Error occurred during live-migration", 9118 ); 9119 } 9120 9121 // Check the source vm has been terminated successful (give it '3s' to settle) 9122 thread::sleep(std::time::Duration::new(3, 0)); 9123 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9124 print_and_panic( 9125 src_child, 9126 dest_child, 9127 None, 9128 "source VM was not terminated successfully.", 9129 ); 9130 }; 9131 9132 // Post live-migration check to make sure the destination VM is funcational 9133 let r = std::panic::catch_unwind(|| { 9134 // Perform same checks to validate VM has been properly migrated 9135 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9136 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9137 9138 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9139 9140 // Perform checks on guest RAM using balloon 9141 let total_memory = guest.get_total_memory().unwrap_or_default(); 9142 assert!(total_memory > 4_800_000); 9143 assert!(total_memory < 5_760_000); 9144 // Deflate balloon to restore entire RAM to the VM 9145 resize_command(&dest_api_socket, None, None, Some(0), None); 9146 thread::sleep(std::time::Duration::new(5, 0)); 9147 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9148 // Decrease guest RAM with virtio-mem 9149 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 9150 thread::sleep(std::time::Duration::new(5, 0)); 9151 let total_memory = guest.get_total_memory().unwrap_or_default(); 9152 assert!(total_memory > 4_800_000); 9153 assert!(total_memory < 5_760_000); 9154 }); 9155 9156 // Clean-up the destination VM and make sure it terminated correctly 9157 let _ = dest_child.kill(); 9158 let dest_output = dest_child.wait_with_output().unwrap(); 9159 handle_child_output(r, &dest_output); 9160 9161 // Check the destination VM has the expected 'concole_text' from its output 9162 let r = std::panic::catch_unwind(|| { 9163 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9164 }); 9165 handle_child_output(r, &dest_output); 9166 } 9167 9168 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 9169 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9170 let guest = Guest::new(Box::new(focal)); 9171 let kernel_path = direct_kernel_boot_path(); 9172 let console_text = String::from("On a branch floating down river a cricket, singing."); 9173 let net_id = "net123"; 9174 let net_params = format!( 9175 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9176 net_id, guest.network.guest_mac, guest.network.host_ip 9177 ); 9178 9179 let memory_param: &[&str] = if local { 9180 &[ 9181 "--memory", 9182 "size=0,hotplug_method=virtio-mem,shared=on", 9183 "--memory-zone", 9184 "id=mem0,size=1G,hotplug_size=4G,shared=on", 9185 "id=mem1,size=1G,hotplug_size=4G,shared=on", 9186 "id=mem2,size=2G,hotplug_size=4G,shared=on", 9187 "--numa", 9188 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9189 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9190 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9191 ] 9192 } else { 9193 &[ 9194 "--memory", 9195 "size=0,hotplug_method=virtio-mem", 9196 "--memory-zone", 9197 "id=mem0,size=1G,hotplug_size=4G", 9198 "id=mem1,size=1G,hotplug_size=4G", 9199 "id=mem2,size=2G,hotplug_size=4G", 9200 "--numa", 9201 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9202 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9203 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9204 ] 9205 }; 9206 9207 let boot_vcpus = 6; 9208 let max_vcpus = 12; 9209 9210 let pmem_temp_file = TempFile::new().unwrap(); 9211 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9212 std::process::Command::new("mkfs.ext4") 9213 .arg(pmem_temp_file.as_path()) 9214 .output() 9215 .expect("Expect creating disk image to succeed"); 9216 let pmem_path = String::from("/dev/pmem0"); 9217 9218 // Start the source VM 9219 let src_vm_path = if !upgrade_test { 9220 clh_command("cloud-hypervisor") 9221 } else { 9222 cloud_hypervisor_release_path() 9223 }; 9224 let src_api_socket = temp_api_path(&guest.tmp_dir); 9225 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9226 src_vm_cmd 9227 .args([ 9228 "--cpus", 9229 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9230 ]) 9231 .args(memory_param) 9232 .args(["--kernel", kernel_path.to_str().unwrap()]) 9233 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9234 .default_disks() 9235 .args(["--net", net_params.as_str()]) 9236 .args(["--api-socket", &src_api_socket]) 9237 .args([ 9238 "--pmem", 9239 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9240 ]); 9241 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9242 9243 // Start the destination VM 9244 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9245 dest_api_socket.push_str(".dest"); 9246 let mut dest_child = GuestCommand::new(&guest) 9247 .args(["--api-socket", &dest_api_socket]) 9248 .capture_output() 9249 .spawn() 9250 .unwrap(); 9251 9252 let r = std::panic::catch_unwind(|| { 9253 guest.wait_vm_boot(None).unwrap(); 9254 9255 // Make sure the source VM is functaionl 9256 // Check the number of vCPUs 9257 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9258 9259 // Check the guest RAM 9260 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 9261 9262 // Check the guest virtio-devices, e.g. block, rng, console, and net 9263 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9264 9265 // Check the NUMA parameters are applied correctly and resize 9266 // each zone to test the case where we migrate a VM with the 9267 // virtio-mem regions being used. 9268 { 9269 guest.check_numa_common( 9270 Some(&[960_000, 960_000, 1_920_000]), 9271 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9272 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9273 ); 9274 9275 // AArch64 currently does not support hotplug, and therefore we only 9276 // test hotplug-related function on x86_64 here. 9277 #[cfg(target_arch = "x86_64")] 9278 { 9279 guest.enable_memory_hotplug(); 9280 9281 // Resize every memory zone and check each associated NUMA node 9282 // has been assigned the right amount of memory. 9283 resize_zone_command(&src_api_socket, "mem0", "2G"); 9284 resize_zone_command(&src_api_socket, "mem1", "2G"); 9285 resize_zone_command(&src_api_socket, "mem2", "3G"); 9286 thread::sleep(std::time::Duration::new(5, 0)); 9287 9288 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 9289 } 9290 } 9291 9292 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9293 // to make sure that removing and adding back the virtio-net device does 9294 // not break the live-migration support for virtio-pci. 9295 #[cfg(target_arch = "x86_64")] 9296 { 9297 assert!(remote_command( 9298 &src_api_socket, 9299 "remove-device", 9300 Some(net_id), 9301 )); 9302 thread::sleep(std::time::Duration::new(10, 0)); 9303 9304 // Plug the virtio-net device again 9305 assert!(remote_command( 9306 &src_api_socket, 9307 "add-net", 9308 Some(net_params.as_str()), 9309 )); 9310 thread::sleep(std::time::Duration::new(10, 0)); 9311 } 9312 9313 // Start the live-migration 9314 let migration_socket = String::from( 9315 guest 9316 .tmp_dir 9317 .as_path() 9318 .join("live-migration.sock") 9319 .to_str() 9320 .unwrap(), 9321 ); 9322 9323 assert!( 9324 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9325 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9326 ); 9327 }); 9328 9329 // Check and report any errors occurred during the live-migration 9330 if r.is_err() { 9331 print_and_panic( 9332 src_child, 9333 dest_child, 9334 None, 9335 "Error occurred during live-migration", 9336 ); 9337 } 9338 9339 // Check the source vm has been terminated successful (give it '3s' to settle) 9340 thread::sleep(std::time::Duration::new(3, 0)); 9341 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9342 print_and_panic( 9343 src_child, 9344 dest_child, 9345 None, 9346 "source VM was not terminated successfully.", 9347 ); 9348 }; 9349 9350 // Post live-migration check to make sure the destination VM is funcational 9351 let r = std::panic::catch_unwind(|| { 9352 // Perform same checks to validate VM has been properly migrated 9353 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9354 #[cfg(target_arch = "x86_64")] 9355 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 9356 #[cfg(target_arch = "aarch64")] 9357 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9358 9359 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9360 9361 // Perform NUMA related checks 9362 { 9363 #[cfg(target_arch = "aarch64")] 9364 { 9365 guest.check_numa_common( 9366 Some(&[960_000, 960_000, 1_920_000]), 9367 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9368 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9369 ); 9370 } 9371 9372 // AArch64 currently does not support hotplug, and therefore we only 9373 // test hotplug-related function on x86_64 here. 9374 #[cfg(target_arch = "x86_64")] 9375 { 9376 guest.check_numa_common( 9377 Some(&[1_920_000, 1_920_000, 2_880_000]), 9378 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9379 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9380 ); 9381 9382 guest.enable_memory_hotplug(); 9383 9384 // Resize every memory zone and check each associated NUMA node 9385 // has been assigned the right amount of memory. 9386 resize_zone_command(&dest_api_socket, "mem0", "4G"); 9387 resize_zone_command(&dest_api_socket, "mem1", "4G"); 9388 resize_zone_command(&dest_api_socket, "mem2", "4G"); 9389 // Resize to the maximum amount of CPUs and check each NUMA 9390 // node has been assigned the right CPUs set. 9391 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 9392 thread::sleep(std::time::Duration::new(5, 0)); 9393 9394 guest.check_numa_common( 9395 Some(&[3_840_000, 3_840_000, 3_840_000]), 9396 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 9397 None, 9398 ); 9399 } 9400 } 9401 }); 9402 9403 // Clean-up the destination VM and make sure it terminated correctly 9404 let _ = dest_child.kill(); 9405 let dest_output = dest_child.wait_with_output().unwrap(); 9406 handle_child_output(r, &dest_output); 9407 9408 // Check the destination VM has the expected 'concole_text' from its output 9409 let r = std::panic::catch_unwind(|| { 9410 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9411 }); 9412 handle_child_output(r, &dest_output); 9413 } 9414 9415 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 9416 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9417 let guest = Guest::new(Box::new(focal)); 9418 let kernel_path = direct_kernel_boot_path(); 9419 let console_text = String::from("On a branch floating down river a cricket, singing."); 9420 let net_id = "net123"; 9421 let net_params = format!( 9422 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9423 net_id, guest.network.guest_mac, guest.network.host_ip 9424 ); 9425 9426 let memory_param: &[&str] = if local { 9427 &["--memory", "size=4G,shared=on"] 9428 } else { 9429 &["--memory", "size=4G"] 9430 }; 9431 9432 let boot_vcpus = 2; 9433 let max_vcpus = 4; 9434 9435 let pmem_temp_file = TempFile::new().unwrap(); 9436 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9437 std::process::Command::new("mkfs.ext4") 9438 .arg(pmem_temp_file.as_path()) 9439 .output() 9440 .expect("Expect creating disk image to succeed"); 9441 let pmem_path = String::from("/dev/pmem0"); 9442 9443 // Start the source VM 9444 let src_vm_path = if !upgrade_test { 9445 clh_command("cloud-hypervisor") 9446 } else { 9447 cloud_hypervisor_release_path() 9448 }; 9449 let src_api_socket = temp_api_path(&guest.tmp_dir); 9450 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9451 src_vm_cmd 9452 .args([ 9453 "--cpus", 9454 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9455 ]) 9456 .args(memory_param) 9457 .args(["--kernel", kernel_path.to_str().unwrap()]) 9458 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9459 .default_disks() 9460 .args(["--net", net_params.as_str()]) 9461 .args(["--api-socket", &src_api_socket]) 9462 .args([ 9463 "--pmem", 9464 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9465 ]) 9466 .args(["--watchdog"]); 9467 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9468 9469 // Start the destination VM 9470 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9471 dest_api_socket.push_str(".dest"); 9472 let mut dest_child = GuestCommand::new(&guest) 9473 .args(["--api-socket", &dest_api_socket]) 9474 .capture_output() 9475 .spawn() 9476 .unwrap(); 9477 9478 let r = std::panic::catch_unwind(|| { 9479 guest.wait_vm_boot(None).unwrap(); 9480 9481 // Make sure the source VM is functaionl 9482 // Check the number of vCPUs 9483 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9484 // Check the guest RAM 9485 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9486 // Check the guest virtio-devices, e.g. block, rng, console, and net 9487 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9488 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9489 // to make sure that removing and adding back the virtio-net device does 9490 // not break the live-migration support for virtio-pci. 9491 #[cfg(target_arch = "x86_64")] 9492 { 9493 assert!(remote_command( 9494 &src_api_socket, 9495 "remove-device", 9496 Some(net_id), 9497 )); 9498 thread::sleep(std::time::Duration::new(10, 0)); 9499 9500 // Plug the virtio-net device again 9501 assert!(remote_command( 9502 &src_api_socket, 9503 "add-net", 9504 Some(net_params.as_str()), 9505 )); 9506 thread::sleep(std::time::Duration::new(10, 0)); 9507 } 9508 9509 // Enable watchdog and ensure its functional 9510 let mut expected_reboot_count = 1; 9511 // Enable the watchdog with a 15s timeout 9512 enable_guest_watchdog(&guest, 15); 9513 // Reboot and check that systemd has activated the watchdog 9514 guest.ssh_command("sudo reboot").unwrap(); 9515 guest.wait_vm_boot(None).unwrap(); 9516 expected_reboot_count += 1; 9517 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9518 assert_eq!( 9519 guest 9520 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 9521 .unwrap() 9522 .trim() 9523 .parse::<u32>() 9524 .unwrap_or_default(), 9525 2 9526 ); 9527 // Allow some normal time to elapse to check we don't get spurious reboots 9528 thread::sleep(std::time::Duration::new(40, 0)); 9529 // Check no reboot 9530 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9531 9532 // Start the live-migration 9533 let migration_socket = String::from( 9534 guest 9535 .tmp_dir 9536 .as_path() 9537 .join("live-migration.sock") 9538 .to_str() 9539 .unwrap(), 9540 ); 9541 9542 assert!( 9543 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9544 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9545 ); 9546 }); 9547 9548 // Check and report any errors occurred during the live-migration 9549 if r.is_err() { 9550 print_and_panic( 9551 src_child, 9552 dest_child, 9553 None, 9554 "Error occurred during live-migration", 9555 ); 9556 } 9557 9558 // Check the source vm has been terminated successful (give it '3s' to settle) 9559 thread::sleep(std::time::Duration::new(3, 0)); 9560 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9561 print_and_panic( 9562 src_child, 9563 dest_child, 9564 None, 9565 "source VM was not terminated successfully.", 9566 ); 9567 }; 9568 9569 // Post live-migration check to make sure the destination VM is funcational 9570 let r = std::panic::catch_unwind(|| { 9571 // Perform same checks to validate VM has been properly migrated 9572 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9573 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9574 9575 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9576 9577 // Perform checks on watchdog 9578 let mut expected_reboot_count = 2; 9579 9580 // Allow some normal time to elapse to check we don't get spurious reboots 9581 thread::sleep(std::time::Duration::new(40, 0)); 9582 // Check no reboot 9583 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9584 9585 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 9586 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 9587 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 9588 guest.wait_vm_boot(Some(50)).unwrap(); 9589 // Check a reboot is triggered by the watchdog 9590 expected_reboot_count += 1; 9591 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9592 9593 #[cfg(target_arch = "x86_64")] 9594 { 9595 // Now pause the VM and remain offline for 30s 9596 assert!(remote_command(&dest_api_socket, "pause", None)); 9597 thread::sleep(std::time::Duration::new(30, 0)); 9598 assert!(remote_command(&dest_api_socket, "resume", None)); 9599 9600 // Check no reboot 9601 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 9602 } 9603 }); 9604 9605 // Clean-up the destination VM and make sure it terminated correctly 9606 let _ = dest_child.kill(); 9607 let dest_output = dest_child.wait_with_output().unwrap(); 9608 handle_child_output(r, &dest_output); 9609 9610 // Check the destination VM has the expected 'concole_text' from its output 9611 let r = std::panic::catch_unwind(|| { 9612 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9613 }); 9614 handle_child_output(r, &dest_output); 9615 } 9616 9617 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 9618 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9619 let ovs_guest = Guest::new(Box::new(ovs_focal)); 9620 9621 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9622 let migration_guest = Guest::new(Box::new(migration_focal)); 9623 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 9624 9625 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 9626 let (mut ovs_child, mut src_child) = 9627 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 9628 9629 // Start the destination VM 9630 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 9631 dest_api_socket.push_str(".dest"); 9632 let mut dest_child = GuestCommand::new(&migration_guest) 9633 .args(["--api-socket", &dest_api_socket]) 9634 .capture_output() 9635 .spawn() 9636 .unwrap(); 9637 9638 let r = std::panic::catch_unwind(|| { 9639 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 9640 thread::sleep(std::time::Duration::new(1, 0)); 9641 9642 // Start the live-migration 9643 let migration_socket = String::from( 9644 migration_guest 9645 .tmp_dir 9646 .as_path() 9647 .join("live-migration.sock") 9648 .to_str() 9649 .unwrap(), 9650 ); 9651 9652 assert!( 9653 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9654 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9655 ); 9656 }); 9657 9658 // Check and report any errors occurred during the live-migration 9659 if r.is_err() { 9660 print_and_panic( 9661 src_child, 9662 dest_child, 9663 Some(ovs_child), 9664 "Error occurred during live-migration", 9665 ); 9666 } 9667 9668 // Check the source vm has been terminated successful (give it '3s' to settle) 9669 thread::sleep(std::time::Duration::new(3, 0)); 9670 if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { 9671 print_and_panic( 9672 src_child, 9673 dest_child, 9674 Some(ovs_child), 9675 "source VM was not terminated successfully.", 9676 ); 9677 }; 9678 9679 // Post live-migration check to make sure the destination VM is funcational 9680 let r = std::panic::catch_unwind(|| { 9681 // Perform same checks to validate VM has been properly migrated 9682 // Spawn a new netcat listener in the OVS VM 9683 let guest_ip = ovs_guest.network.guest_ip.clone(); 9684 thread::spawn(move || { 9685 ssh_command_ip( 9686 "nc -l 12345", 9687 &guest_ip, 9688 DEFAULT_SSH_RETRIES, 9689 DEFAULT_SSH_TIMEOUT, 9690 ) 9691 .unwrap(); 9692 }); 9693 9694 // Wait for the server to be listening 9695 thread::sleep(std::time::Duration::new(5, 0)); 9696 9697 // And check the connection is still functional after live-migration 9698 migration_guest 9699 .ssh_command("nc -vz 172.100.0.1 12345") 9700 .unwrap(); 9701 }); 9702 9703 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 9704 let _ = dest_child.kill(); 9705 let _ = ovs_child.kill(); 9706 let dest_output = dest_child.wait_with_output().unwrap(); 9707 let ovs_output = ovs_child.wait_with_output().unwrap(); 9708 9709 cleanup_ovs_dpdk(); 9710 9711 handle_child_output(r, &dest_output); 9712 handle_child_output(Ok(()), &ovs_output); 9713 } 9714 9715 mod live_migration_parallel { 9716 use super::*; 9717 #[test] 9718 fn test_live_migration_basic() { 9719 _test_live_migration(false, false) 9720 } 9721 9722 #[test] 9723 fn test_live_migration_local() { 9724 _test_live_migration(false, true) 9725 } 9726 9727 #[test] 9728 #[cfg(not(feature = "mshv"))] 9729 fn test_live_migration_numa() { 9730 _test_live_migration_numa(false, false) 9731 } 9732 9733 #[test] 9734 #[cfg(not(feature = "mshv"))] 9735 fn test_live_migration_numa_local() { 9736 _test_live_migration_numa(false, true) 9737 } 9738 9739 #[test] 9740 fn test_live_migration_watchdog() { 9741 _test_live_migration_watchdog(false, false) 9742 } 9743 9744 #[test] 9745 fn test_live_migration_watchdog_local() { 9746 _test_live_migration_watchdog(false, true) 9747 } 9748 9749 #[test] 9750 fn test_live_migration_balloon() { 9751 _test_live_migration_balloon(false, false) 9752 } 9753 9754 #[test] 9755 fn test_live_migration_balloon_local() { 9756 _test_live_migration_balloon(false, true) 9757 } 9758 9759 #[test] 9760 #[ignore = "See #6134"] 9761 fn test_live_upgrade_basic() { 9762 _test_live_migration(true, false) 9763 } 9764 9765 #[test] 9766 #[ignore = "See #6134"] 9767 fn test_live_upgrade_local() { 9768 _test_live_migration(true, true) 9769 } 9770 9771 #[test] 9772 #[ignore = "See #6134"] 9773 #[cfg(not(feature = "mshv"))] 9774 fn test_live_upgrade_numa() { 9775 _test_live_migration_numa(true, false) 9776 } 9777 9778 #[test] 9779 #[ignore = "See #6134"] 9780 #[cfg(not(feature = "mshv"))] 9781 fn test_live_upgrade_numa_local() { 9782 _test_live_migration_numa(true, true) 9783 } 9784 9785 #[test] 9786 #[ignore = "See #6134"] 9787 fn test_live_upgrade_watchdog() { 9788 _test_live_migration_watchdog(true, false) 9789 } 9790 9791 #[test] 9792 #[ignore = "See #6134"] 9793 fn test_live_upgrade_watchdog_local() { 9794 _test_live_migration_watchdog(true, true) 9795 } 9796 9797 #[test] 9798 #[ignore = "See #6134"] 9799 fn test_live_upgrade_balloon() { 9800 _test_live_migration_balloon(true, false) 9801 } 9802 9803 #[test] 9804 #[ignore = "See #6134"] 9805 fn test_live_upgrade_balloon_local() { 9806 _test_live_migration_balloon(true, true) 9807 } 9808 } 9809 9810 mod live_migration_sequential { 9811 #[cfg(target_arch = "x86_64")] 9812 #[cfg(not(feature = "mshv"))] 9813 use super::*; 9814 9815 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 9816 #[test] 9817 #[ignore = "See #5532"] 9818 #[cfg(target_arch = "x86_64")] 9819 #[cfg(not(feature = "mshv"))] 9820 fn test_live_migration_ovs_dpdk() { 9821 _test_live_migration_ovs_dpdk(false, false); 9822 } 9823 9824 #[test] 9825 #[cfg(target_arch = "x86_64")] 9826 #[cfg(not(feature = "mshv"))] 9827 fn test_live_migration_ovs_dpdk_local() { 9828 _test_live_migration_ovs_dpdk(false, true); 9829 } 9830 9831 #[test] 9832 #[ignore = "See #5532"] 9833 #[cfg(target_arch = "x86_64")] 9834 #[cfg(not(feature = "mshv"))] 9835 fn test_live_upgrade_ovs_dpdk() { 9836 _test_live_migration_ovs_dpdk(true, false); 9837 } 9838 9839 #[test] 9840 #[ignore = "See #5532"] 9841 #[cfg(target_arch = "x86_64")] 9842 #[cfg(not(feature = "mshv"))] 9843 fn test_live_upgrade_ovs_dpdk_local() { 9844 _test_live_migration_ovs_dpdk(true, true); 9845 } 9846 } 9847 } 9848 9849 #[cfg(target_arch = "aarch64")] 9850 mod aarch64_acpi { 9851 use crate::*; 9852 9853 #[test] 9854 fn test_simple_launch_acpi() { 9855 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9856 9857 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 9858 let guest = Guest::new(disk_config); 9859 9860 let mut child = GuestCommand::new(&guest) 9861 .args(["--cpus", "boot=1"]) 9862 .args(["--memory", "size=512M"]) 9863 .args(["--kernel", edk2_path().to_str().unwrap()]) 9864 .default_disks() 9865 .default_net() 9866 .args(["--serial", "tty", "--console", "off"]) 9867 .capture_output() 9868 .spawn() 9869 .unwrap(); 9870 9871 let r = std::panic::catch_unwind(|| { 9872 guest.wait_vm_boot(Some(120)).unwrap(); 9873 9874 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 9875 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 9876 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 9877 }); 9878 9879 let _ = child.kill(); 9880 let output = child.wait_with_output().unwrap(); 9881 9882 handle_child_output(r, &output); 9883 }); 9884 } 9885 9886 #[test] 9887 fn test_guest_numa_nodes_acpi() { 9888 _test_guest_numa_nodes(true); 9889 } 9890 9891 #[test] 9892 fn test_cpu_topology_421_acpi() { 9893 test_cpu_topology(4, 2, 1, true); 9894 } 9895 9896 #[test] 9897 fn test_cpu_topology_142_acpi() { 9898 test_cpu_topology(1, 4, 2, true); 9899 } 9900 9901 #[test] 9902 fn test_cpu_topology_262_acpi() { 9903 test_cpu_topology(2, 6, 2, true); 9904 } 9905 9906 #[test] 9907 fn test_power_button_acpi() { 9908 _test_power_button(true); 9909 } 9910 9911 #[test] 9912 fn test_virtio_iommu() { 9913 _test_virtio_iommu(true) 9914 } 9915 } 9916 9917 mod rate_limiter { 9918 use super::*; 9919 9920 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 9921 // compared to given 'limit' rate. 9922 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 9923 let upper_limit = limit * (1_f64 + difference); 9924 let lower_limit = limit * (1_f64 - difference); 9925 9926 if measured > lower_limit && measured < upper_limit { 9927 return true; 9928 } 9929 9930 eprintln!( 9931 "\n\n==== Start 'check_rate_limit' failed ==== \ 9932 \n\nmeasured={measured}, , lower_limit={lower_limit}, upper_limit={upper_limit} \ 9933 \n\n==== End 'check_rate_limit' failed ====\n\n" 9934 ); 9935 9936 false 9937 } 9938 9939 fn _test_rate_limiter_net(rx: bool) { 9940 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9941 let guest = Guest::new(Box::new(focal)); 9942 9943 let test_timeout = 10; 9944 let num_queues = 2; 9945 let queue_size = 256; 9946 let bw_size = 10485760_u64; // bytes 9947 let bw_refill_time = 100; // ms 9948 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 9949 9950 let net_params = format!( 9951 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 9952 guest.network.guest_mac, 9953 guest.network.host_ip, 9954 num_queues, 9955 queue_size, 9956 bw_size, 9957 bw_refill_time, 9958 ); 9959 9960 let mut child = GuestCommand::new(&guest) 9961 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 9962 .args(["--memory", "size=4G"]) 9963 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 9964 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9965 .default_disks() 9966 .args(["--net", net_params.as_str()]) 9967 .capture_output() 9968 .spawn() 9969 .unwrap(); 9970 9971 let r = std::panic::catch_unwind(|| { 9972 guest.wait_vm_boot(None).unwrap(); 9973 let measured_bps = 9974 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, true) 9975 .unwrap(); 9976 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 9977 }); 9978 9979 let _ = child.kill(); 9980 let output = child.wait_with_output().unwrap(); 9981 handle_child_output(r, &output); 9982 } 9983 9984 #[test] 9985 fn test_rate_limiter_net_rx() { 9986 _test_rate_limiter_net(true); 9987 } 9988 9989 #[test] 9990 fn test_rate_limiter_net_tx() { 9991 _test_rate_limiter_net(false); 9992 } 9993 9994 fn _test_rate_limiter_block(bandwidth: bool, num_queues: u32) { 9995 let test_timeout = 10; 9996 let fio_ops = FioOps::RandRW; 9997 9998 let bw_size = if bandwidth { 9999 10485760_u64 // bytes 10000 } else { 10001 100_u64 // I/O 10002 }; 10003 let bw_refill_time = 100; // ms 10004 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10005 10006 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10007 let guest = Guest::new(Box::new(focal)); 10008 let api_socket = temp_api_path(&guest.tmp_dir); 10009 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10010 let blk_rate_limiter_test_img = 10011 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 10012 10013 // Create the test block image 10014 assert!(exec_host_command_output(&format!( 10015 "dd if=/dev/zero of={blk_rate_limiter_test_img} bs=1M count=1024" 10016 )) 10017 .status 10018 .success()); 10019 10020 let test_blk_params = if bandwidth { 10021 format!( 10022 "path={blk_rate_limiter_test_img},num_queues={num_queues},bw_size={bw_size},bw_refill_time={bw_refill_time}" 10023 ) 10024 } else { 10025 format!( 10026 "path={blk_rate_limiter_test_img},num_queues={num_queues},ops_size={bw_size},ops_refill_time={bw_refill_time}" 10027 ) 10028 }; 10029 10030 let mut child = GuestCommand::new(&guest) 10031 .args(["--cpus", &format!("boot={num_queues}")]) 10032 .args(["--memory", "size=4G"]) 10033 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10034 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10035 .args([ 10036 "--disk", 10037 format!( 10038 "path={}", 10039 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10040 ) 10041 .as_str(), 10042 format!( 10043 "path={}", 10044 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10045 ) 10046 .as_str(), 10047 test_blk_params.as_str(), 10048 ]) 10049 .default_net() 10050 .args(["--api-socket", &api_socket]) 10051 .capture_output() 10052 .spawn() 10053 .unwrap(); 10054 10055 let r = std::panic::catch_unwind(|| { 10056 guest.wait_vm_boot(None).unwrap(); 10057 10058 let fio_command = format!( 10059 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 10060 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10061 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10062 ); 10063 let output = guest.ssh_command(&fio_command).unwrap(); 10064 10065 // Parse fio output 10066 let measured_rate = if bandwidth { 10067 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 10068 } else { 10069 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 10070 }; 10071 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10072 }); 10073 10074 let _ = child.kill(); 10075 let output = child.wait_with_output().unwrap(); 10076 handle_child_output(r, &output); 10077 } 10078 10079 fn _test_rate_limiter_group_block(bandwidth: bool, num_queues: u32, num_disks: u32) { 10080 let test_timeout = 10; 10081 let fio_ops = FioOps::RandRW; 10082 10083 let bw_size = if bandwidth { 10084 10485760_u64 // bytes 10085 } else { 10086 100_u64 // I/O 10087 }; 10088 let bw_refill_time = 100; // ms 10089 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10090 10091 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10092 let guest = Guest::new(Box::new(focal)); 10093 let api_socket = temp_api_path(&guest.tmp_dir); 10094 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10095 10096 let rate_limit_group_arg = if bandwidth { 10097 format!("id=group0,bw_size={bw_size},bw_refill_time={bw_refill_time}") 10098 } else { 10099 format!("id=group0,ops_size={bw_size},ops_refill_time={bw_refill_time}") 10100 }; 10101 10102 let mut disk_args = vec![ 10103 "--disk".to_string(), 10104 format!( 10105 "path={}", 10106 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10107 ), 10108 format!( 10109 "path={}", 10110 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10111 ), 10112 ]; 10113 10114 for i in 0..num_disks { 10115 let test_img_path = String::from( 10116 test_img_dir 10117 .as_path() 10118 .join(format!("blk{}.img", i)) 10119 .to_str() 10120 .unwrap(), 10121 ); 10122 10123 assert!(exec_host_command_output(&format!( 10124 "dd if=/dev/zero of={test_img_path} bs=1M count=1024" 10125 )) 10126 .status 10127 .success()); 10128 10129 disk_args.push(format!( 10130 "path={test_img_path},num_queues={num_queues},rate_limit_group=group0" 10131 )); 10132 } 10133 10134 let mut child = GuestCommand::new(&guest) 10135 .args(["--cpus", &format!("boot={}", num_queues * num_disks)]) 10136 .args(["--memory", "size=4G"]) 10137 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10138 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10139 .args(["--rate-limit-group", &rate_limit_group_arg]) 10140 .args(disk_args) 10141 .default_net() 10142 .args(["--api-socket", &api_socket]) 10143 .capture_output() 10144 .spawn() 10145 .unwrap(); 10146 10147 let r = std::panic::catch_unwind(|| { 10148 guest.wait_vm_boot(None).unwrap(); 10149 10150 let mut fio_command = format!( 10151 "sudo fio --name=global --output-format=json \ 10152 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10153 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10154 ); 10155 10156 // Generate additional argument for each disk: 10157 // --name=job0 --filename=/dev/vdc \ 10158 // --name=job1 --filename=/dev/vdd \ 10159 // --name=job2 --filename=/dev/vde \ 10160 // ... 10161 for i in 0..num_disks { 10162 let c: char = 'c'; 10163 let arg = format!( 10164 " --name=job{i} --filename=/dev/vd{}", 10165 char::from_u32((c as u32) + i).unwrap() 10166 ); 10167 fio_command += &arg; 10168 } 10169 let output = guest.ssh_command(&fio_command).unwrap(); 10170 10171 // Parse fio output 10172 let measured_rate = if bandwidth { 10173 parse_fio_output(&output, &fio_ops, num_queues * num_disks).unwrap() 10174 } else { 10175 parse_fio_output_iops(&output, &fio_ops, num_queues * num_disks).unwrap() 10176 }; 10177 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10178 }); 10179 10180 let _ = child.kill(); 10181 let output = child.wait_with_output().unwrap(); 10182 handle_child_output(r, &output); 10183 } 10184 10185 #[test] 10186 fn test_rate_limiter_block_bandwidth() { 10187 _test_rate_limiter_block(true, 1); 10188 _test_rate_limiter_block(true, 2) 10189 } 10190 10191 #[test] 10192 fn test_rate_limiter_group_block_bandwidth() { 10193 _test_rate_limiter_group_block(true, 1, 1); 10194 _test_rate_limiter_group_block(true, 2, 1); 10195 _test_rate_limiter_group_block(true, 1, 2); 10196 _test_rate_limiter_group_block(true, 2, 2); 10197 } 10198 10199 #[test] 10200 fn test_rate_limiter_block_iops() { 10201 _test_rate_limiter_block(false, 1); 10202 _test_rate_limiter_block(false, 2); 10203 } 10204 10205 #[test] 10206 fn test_rate_limiter_group_block_iops() { 10207 _test_rate_limiter_group_block(false, 1, 1); 10208 _test_rate_limiter_group_block(false, 2, 1); 10209 _test_rate_limiter_group_block(false, 1, 2); 10210 _test_rate_limiter_group_block(false, 2, 2); 10211 } 10212 } 10213